Compare commits

..
13 Commits
Author SHA1 Message Date
CircleCI dec9d063a6 Bump version to 0.1.0-RC.7 [skip ci] 2026-10-15 02:45:16 +00:00
james.maesandClaude Opus 5.5 89e0e5915b Merge develop into release/0.1 for 0.1.0-RC.7 (#24 help bundle)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 21:35:47 -05:00
CircleCI cf2e9fca77 Bump version to 0.1.0-RC.6 [skip ci] 2026-10-14 23:07:52 +00:00
james.maesandClaude Opus 5.5 cd26342cfd Merge develop into release/0.1 for 0.1.0-RC.6 (#18 ConfigSync session fix, #19, #20, #23)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:58:53 -05:00
CircleCI 93c116ff01 Bump version to 0.1.0-RC.5 [skip ci] 2026-10-14 22:34:39 +00:00
james.maesandClaude Opus 5.5 9f2d7fc202 Merge develop into release/0.1 for 0.1.0-RC.5 (#21 setup card, seeds, help)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:25:24 -05:00
CircleCI 5c8f2f595b Bump version to 0.1.0-RC.4 [skip ci] 2026-10-14 12:37:00 +00:00
james.maesandClaude Opus 5.5 1ea40241f7 Merge develop into release/0.1 for 0.1.0-RC.4 (#13 maxUploadBytes, #14 QQQ RC.6 and guardrails)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 07:28:10 -05:00
CircleCI 8692857891 Bump version to 0.1.0-RC.3 [skip ci] 2026-10-14 01:02:56 +00:00
james.maesandClaude Opus 5.5 b6db9aff8b Merge develop into release/0.1 for 0.1.0-RC.3 (ConnectionRuntime/S3ClientFactory consolidation, #11)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 19:55:34 -05:00
CircleCI 07275bce3f Bump version to 0.1.0-RC.2 [skip ci] 2026-10-13 15:51:56 +00:00
james.maesandClaude Opus 5.5 9eefc0b071 Merge develop into release/0.1 for 0.1.0-RC.2
AWS SDK for Java v2 2.41.10 -> 2.41.34 (GHSA-443w-3rq3-5m5h, #8).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 10:44:08 -05:00
CircleCI f1fb2462f9 Bump version to 0.1.0-RC.1 [skip ci] 2026-10-12 23:02:00 +00:00
37 changed files with 219 additions and 1203 deletions

No files matched your search

+57 -11
View File
@@ -1,14 +1,60 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.8
qqq-orb: kingsrook/qqq-orb@0.6.9
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
workflows:
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/^v\d+\.\d+\.\d+$/]
-7
View File
@@ -1,7 +0,0 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
-23
View File
@@ -1,23 +0,0 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+13 -62
View File
@@ -1,74 +1,25 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
'on':
on:
workflow_dispatch:
inputs:
dry_run:
description: Plan a cut without changing release state
description: Print the plan without merging or creating branches
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
types: [closed]
branches: [develop]
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
-16
View File
@@ -1,16 +0,0 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
-8
View File
@@ -26,18 +26,10 @@ All notable changes to this project are documented here. The format follows Keep
- Foreign keys on every relation between the qbit tables (`fk_<table>_<column>`), generated by `ObjectStorageSchema`: owned observed state (capabilities, folders, objects, versions, usage) cascades from its bucket or object, `object.folder_id` and `object_reference.object_id` are set null, and a bucket's connection, header and protection rules and operation log restrict. Each key is added by an idempotent block that first records rows whose parent is gone in `qqq_fk_quarantine` and clears (nullable column) or deletes (required column) them, so it also upgrades an existing schema. New indexes `ix_object_storage_bucket_1` (connection_id), `ix_object_storage_header_rule_1` and `ix_object_storage_protection_rule_1` (bucket_id) back the keys. Upgrade: run the generated statements again (or the host's copy of them).
- Conformance suite against MinIO and Garage (Testcontainers), service, index and action tests, and the index and actions on PostgreSQL.
### Fixed
- The service's own writes are no longer refused in a host that enforces permissions inside the core actions (`QInstance.enforcePermissionsInCoreActions`). `Db.asSystem` starts a new context, so each of its writes is an outermost action there and core asks `ObjectStoragePermissionChecker`, which refuses every write of a table nobody may write directly, to every session. The `object_storage_operation` row of every stored object was lost (logged as "Could not write a storage operation log row", the object itself was stored), and so was each other system write of those tables. `Db.insert`, `Db.update` and `Db.delete` now mark their input with core's per-call permission bypass (`withPermissionBypass`, reason `Db.OWN_WRITE_REASON`), so core does not ask about the service's own writes. The checker is unchanged in what it refuses: a write of those tables that does not carry a bypass is refused to a person, to `object_storage.admin` and to the core system user alike, so server code running as the system user cannot edit or delete operation log rows through an enforced action.
- An operation log row that could not be written is no longer only a warning. It is logged at ERROR, raises the `storage.operation_log.failed` alert (`StorageAlertSink`, severity `error`, dedupe key `bucket:<alias>`, resolved by the next row written for that bucket) and is counted in `OperationLog.failedWriteCount()` (per node, since start). A row the insert action hands back with a record error (a refused value, a record lock) is now detected as well; before, it was dropped without any log line from the qbit. The operation the row describes still succeeds.
### Removed
- The QRun-IO/qqq#1165 stand-ins: `SessionCredentialsClient` (session tokens now go through core `S3BackendMetaData.sessionToken` / `S3ClientFactory`, including for the presigner) and the file reading in `SecretRefs` (core `QMetaDataVariableInterpreter` resolves `${file.PATH}`). **Behavior change:** core reads `${file.PATH}` only inside its allowed roots, which are `/var/run/secrets` and `/run/secrets` by default. A host that mounts storage secrets elsewhere must set `qqq.fileReference.roots` or `QQQ_FILE_REFERENCE_ROOTS`. The qbit's default allow-list (`file./var/run/secrets/storage/*`) is already inside the default roots.
- None for the health migration; both Test connection record actions, including on-demand write tests, are retained.
### Changed
- Multi-node deployments: the runtime caches are registered with QQQ `ClusterCaches`. Shared (cleared on every node): the bucket rows' presign limits (`qbit-object-storage.bucket-row-limits`, on a bucket row insert or update), the enabled header rules (`qbit-object-storage.header-rules`, on a header rule insert, update or delete), both through the new `ConfigRowChangeCustomizer`, and the capability statuses (`qbit-object-storage.capabilities`: a probe result recorded on one node clears that capability on the others, which reload it from the capability row; before, other nodes kept the old status until restart). The 15 s TTLs remain the safety net. Node-local, with reasons: the alias and connection row-id maps, the S3 client map, the process-permission map, the provider defaults, the running-scan guard and the per-pod presign rate limiter (`StorageBucket.PRESIGN_COUNTS`; per-node rate limits are accepted for now). `ObjectStorageClusterCachesTest` audits the package for unregistered static caches and covers remote and local invalidation.
- Builds against QQQ `4.1.0-SNAPSHOT` (`qqq.snapshot.version`) for `ClusterCaches` until the next QQQ RC ships; a release must pin that RC.
- Read the qbit metadata version from the Maven-filtered artifact version so release metadata matches the build and help bundle.
- Rechecked the released RC.8 JARs: core still has no S3 session-token/provider hook or file-reference interpreter. Retained both compatibility shims and moved their review deadline to RC.9 under issue #10.
- Raised the QQQ BOM baseline to released `4.1.0-RC.8` for the central health API.
- Raised the QQQ BOM baseline to released `4.1.0-RC.7` for the core help-bundle registry and coverage assertions.
-1
View File
@@ -125,7 +125,6 @@ It is insert-only and inserts a **disabled** generic header rule, `default-mutab
- **Previews** are short-lived (5 minute) presigned GETs, so the Next dashboard's `img-src` must allow the storage host (it allows `https:` by default). Video, audio and PDF previews need same-origin media (the dashboard's `media-src` and `frame-src` are `'self'`); a same-origin download route is the planned follow-up. Until then those files show the file card with download and open links.
- **Scheduler.** Set `withSchedulerName(...)` to run the nightly scan, HEAD backlog, reference scan, usage snapshot and trash purge. Without it, run Rescan bucket by hand.
- **Alerts.** With the Alerts qbit installed, bind a `StorageAlertSink` adapter (`ObjectStorageRuntime.setAlertSink`); without it every condition is still visible on the home and in the tables.
- **A missing audit row.** A storage operation succeeds even when its `object_storage_operation` row cannot be written, and that row is then the one condition no table shows. It is logged at ERROR ("Could not write a storage operation log row"), raised as the `storage.operation_log.failed` alert (severity `error`, dedupe key `bucket:<alias>`, resolved by the next row written for that bucket) and counted in `OperationLog.failedWriteCount()`, per node since it started. A host without an alert sink should export that count (a gauge, or a health check of its own) and alert when it is above zero.
## Conformance
+10 -15
View File
@@ -26,21 +26,16 @@ The public qbit configuration API is unchanged.
## Remaining local code
- Core has no presigner factory. The local presigner keeps its separate public
endpoint and least-privilege keys; its credentials (including the session
token) come from core `S3ClientFactory.buildCredentialsProvider`.
- `SecretRefs` keeps only the qbit's allow-list; resolution, including
`${file.PATH}`, is core `QMetaDataVariableInterpreter`.
## Stand-ins removed (QRun-IO/qqq#1165)
- `SessionCredentialsClient` is gone: core `S3BackendMetaData.sessionToken`
makes `S3ClientFactory` sign with `AwsSessionCredentials`.
- `SecretRefs` no longer reads files itself. Core reads `${file.PATH}`
read-only, and only inside its allowed roots: `/var/run/secrets` and
`/run/secrets` by default, or `qqq.fileReference.roots` /
`QQQ_FILE_REFERENCE_ROOTS` (comma-separated). The qbit's default allow-list
(`file./var/run/secrets/storage/*`) is inside the default roots.
- Core RC.5 has no session-token field or credentials-provider customization
in `S3BackendMetaData` / `S3ClientFactory.buildClient`. The narrow
`SessionCredentialsClient` bridge adds session credentials to requests while
retaining their overrides; core still creates the client and HTTP transport.
Core needs session-token metadata (including enrichment and cache-key support)
and `AwsSessionCredentials` construction, or a caller-provided credentials hook.
Marker: `STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.7)`.
- Core has no presigner factory. The local presigner retains its separate public
endpoint and least-privilege keys, along with existing session-token handling.
- `SecretRefs` file resolution is unchanged and carries the same RC.7 marker.
## Resolved dependency diff
+4 -32
View File
@@ -20,7 +20,7 @@
<properties>
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
<revision>0.2.0-SNAPSHOT</revision>
<revision>0.1.0-RC.7</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<jacoco.version>0.8.15</jacoco.version>
@@ -29,7 +29,7 @@
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.2.1</version>
<version>2.1.0</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -172,35 +172,7 @@
</dependencies>
<build>
<resources>
<!-- help bundles carry the artifact version (QQQ help bundle provenance) -->
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
<includes>
<include>help/*.yaml</include>
<include>qbit-object-storage.properties</include>
</includes>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
<excludes>
<exclude>help/*.yaml</exclude>
<exclude>qbit-object-storage.properties</exclude>
</excludes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables combine.children="append">
<qbit.artifactVersion>${project.version}</qbit.artifactVersion>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@@ -283,14 +255,14 @@
<profiles>
<!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent -->
<!-- imports the released 4.1 BOM, this profile is active by default and -->
<!-- imports qqq-bom-pom 4.1.0-RC.15 ahead of the parent's 4.0.0 BOM. -->
<!-- imports qqq-bom-pom 4.1.0-RC.7 ahead of the parent's 4.0.0 BOM. -->
<profile>
<id>qqq-snapshot</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.25</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.8</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -62,7 +62,6 @@ public class ObjectStorageQBitConfig implements QBitConfig
private String appName = "objectStorageAdmin";
private String appLabel = "Storage";
private String parentAppName;
private String tablePathPrefix;
private List<String> allowedSecretRefPatterns = new ArrayList<>(SecretRefs.DEFAULT_ALLOWED_PATTERNS);
private Integer bulkCap = 10_000;
private Boolean bucketScoping = true;
@@ -426,39 +425,6 @@ public class ObjectStorageQBitConfig implements QBitConfig
/********************************************************************************
** Getter for tablePathPrefix: the front of the links the Storage home makes to
** tables and records. Defaults to /app, the route every QQQ frontend serves
** tables at (/app/<tableName>).
*******************************************************************************/
public String getTablePathPrefix()
{
return (this.tablePathPrefix == null ? "/app" : this.tablePathPrefix);
}
/********************************************************************************
** Setter for tablePathPrefix.
*******************************************************************************/
public void setTablePathPrefix(String tablePathPrefix)
{
this.tablePathPrefix = tablePathPrefix;
}
/********************************************************************************
** Fluent setter for tablePathPrefix.
*******************************************************************************/
public ObjectStorageQBitConfig withTablePathPrefix(String tablePathPrefix)
{
this.tablePathPrefix = tablePathPrefix;
return (this);
}
/********************************************************************************
** Getter for allowedSecretRefPatterns.
*******************************************************************************/
@@ -21,11 +21,8 @@
package com.kingsrook.qbits.objectstorage;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import com.kingsrook.qbits.objectstorage.app.StorageWidgets;
import com.kingsrook.qbits.objectstorage.health.ObjectStorageConnectionChecks;
import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
@@ -63,41 +60,13 @@ public class ObjectStorageQBitProducer implements QBitMetaDataProducer<ObjectSto
{
public static final String GROUP_ID = "com.kingsrook.qbits";
public static final String ARTIFACT_ID = "qbit-object-storage";
public static final String VERSION = loadVersion();
public static final String VERSION = "0.1.0";
private ObjectStorageQBitConfig qBitConfig;
private String namespace;
/*******************************************************************************
** Read the artifact version from the Maven-filtered resource.
*******************************************************************************/
private static String loadVersion()
{
try(InputStream inputStream = ObjectStorageQBitProducer.class.getClassLoader().getResourceAsStream("qbit-object-storage.properties"))
{
if(inputStream == null)
{
throw new IllegalStateException("Missing qbit-object-storage.properties; build with Maven to generate the artifact version");
}
Properties properties = new Properties();
properties.load(inputStream);
String version = properties.getProperty("version");
if(version == null || version.isBlank() || version.contains("${"))
{
throw new IllegalStateException("Missing or unfiltered version in qbit-object-storage.properties; build with Maven");
}
return (version);
}
catch(IOException e)
{
throw new IllegalStateException("Could not read artifact version from qbit-object-storage.properties", e);
}
}
/*******************************************************************************
** Fluent setter for the config.
*******************************************************************************/
@@ -46,7 +46,6 @@ import com.kingsrook.qbits.objectstorage.internal.PostPolicySigner;
import com.kingsrook.qbits.objectstorage.internal.ProtectionRules;
import com.kingsrook.qbits.objectstorage.internal.RateLimiter;
import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -97,8 +96,7 @@ public final class StorageBucket
{
private static final QLogger LOG = QLogger.getLogger(StorageBucket.class);
private static final Map<String, long[]> PRESIGN_COUNTS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"per-pod presign rate limiter; per-node rate limits are accepted for now (cluster HA spec, owner decision 4)");
private static final Map<String, long[]> PRESIGN_COUNTS = new ConcurrentHashMap<>();
private final StorageBucketConfig bucket;
private final ConnectionRuntime connection;
@@ -106,12 +106,16 @@ public final class StorageWidgets
/*******************************************************************************
** The prefix of table and record links: /app, so a link is /app/<tableName>
** (the frontend serves every table, in any app, at that one flat route).
** The prefix of table and record links.
*******************************************************************************/
public static String tablePathPrefix()
{
return (ObjectStorageRuntime.isInstalled() ? ObjectStorageRuntime.getConfig().getTablePathPrefix() : "/app");
if(!ObjectStorageRuntime.isInstalled())
{
return ("/objectStorageAdmin");
}
ObjectStorageQBitConfig config = ObjectStorageRuntime.getConfig();
return (config.getParentAppName() == null ? "/" + config.getAppName() : "/" + config.getParentAppName() + "/" + config.getAppName());
}
@@ -43,7 +43,6 @@ import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
import com.kingsrook.qbits.objectstorage.internal.Db;
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -79,8 +78,7 @@ public final class ObjectIndex
public static final int PAGE_SIZE = 1000;
private static final Set<String> RUNNING = ClusterCaches.nodeLocal(java.util.concurrent.ConcurrentHashMap.newKeySet(),
"the buckets this node is scanning now (an in-flight guard, not cached data)");
private static final Set<String> RUNNING = java.util.concurrent.ConcurrentHashMap.newKeySet();
/** Test seam: simulate a crash after this many pages of a scan (so resume can be tested). */
static volatile Integer failAfterPages = null;
@@ -30,7 +30,6 @@ import com.kingsrook.qbits.objectstorage.api.Capability;
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -48,11 +47,7 @@ public final class CapabilityStore
public static final Set<Capability> CONNECTION_SCOPE = EnumSet.of(Capability.LIST_V2, Capability.PRESIGNED_GET, Capability.PRESIGNED_PUT,
Capability.PRESIGNED_POST, Capability.USAGE_ADMIN_API, Capability.QUOTA_ADMIN_API);
/** Cluster cache name of the capability statuses (a probe result recorded on one node clears that key on every node). */
public static final String CACHE_NAME = "qbit-object-storage.capabilities";
private static final Map<String, CapabilityStatus> CACHE = ClusterCaches.shared(CACHE_NAME, new ConcurrentHashMap<>(),
CapabilityStore::clearCache, CapabilityStore::clearKey);
private static final Map<String, CapabilityStatus> CACHE = new ConcurrentHashMap<>();
private static final Set<String> LOADED = ConcurrentHashMap.newKeySet();
@@ -67,7 +62,7 @@ public final class CapabilityStore
/*******************************************************************************
** Drop everything cached on this node.
** Drop everything cached.
*******************************************************************************/
public static void clearCache()
{
@@ -77,28 +72,6 @@ public final class CapabilityStore
/*******************************************************************************
** Drop one cached status on this node; the next read reloads the missing
** statuses from the capability rows (cached ones are kept).
*******************************************************************************/
static void clearKey(String key)
{
CACHE.remove(key);
LOADED.clear();
}
/*******************************************************************************
** Whether a status is cached on this node (tests).
*******************************************************************************/
static boolean isCached(StorageBucketConfig bucket, Capability capability)
{
return (CACHE.containsKey(scopeKey(bucket, capability)));
}
/*******************************************************************************
** The status of a capability for a bucket alias.
*******************************************************************************/
@@ -139,24 +112,10 @@ public final class CapabilityStore
/*******************************************************************************
** Record one probe result (cache and, when the tables are there, the row).
** The row is written first, then the key is cleared on every node (they
** reload it from the row), then cached here.
*******************************************************************************/
public static void record(StorageBucketConfig bucket, Capability capability, CapabilityStatus status, String detail)
{
String key = scopeKey(bucket, capability);
persist(bucket, capability, status, detail);
ClusterCaches.invalidateKey(CACHE_NAME, key);
CACHE.put(key, status);
}
/*******************************************************************************
** Write one probe result to its capability row; never throws.
*******************************************************************************/
private static void persist(StorageBucketConfig bucket, Capability capability, CapabilityStatus status, String detail)
{
CACHE.put(scopeKey(bucket, capability), status);
if(!Db.available(ObjectStorageTables.BUCKET_CAPABILITY))
{
return;
@@ -1,95 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage.internal;
import java.util.List;
import java.util.Optional;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Post insert, update and delete of the editable config rows that the runtime
** caches: a bucket row (presign limits, ConfigSync) or a header rule
** (DefaultObjectHeaderPolicy). Clears that cache on every node, so an admin's
** edit takes effect everywhere at once; the short TTLs remain the safety net.
*******************************************************************************/
public class ConfigRowChangeCustomizer implements TableCustomizerInterface
{
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
{
invalidate(insertInput.getTableName());
return (records);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
{
invalidate(updateInput.getTableName());
return (records);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
{
invalidate(deleteInput.getTableName());
return (records);
}
/*******************************************************************************
** Clear the cache that holds the table's rows, on every node.
*******************************************************************************/
static void invalidate(String tableName)
{
if(ObjectStorageTables.BUCKET.equals(tableName))
{
ClusterCaches.invalidate(ConfigSync.ROW_LIMITS_CACHE_NAME);
}
else if(ObjectStorageTables.HEADER_RULE.equals(tableName))
{
ClusterCaches.invalidate(DefaultObjectHeaderPolicy.CACHE_NAME);
}
}
}
@@ -30,7 +30,6 @@ import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -49,16 +48,10 @@ public final class ConfigSync
{
private static final QLogger LOG = QLogger.getLogger(ConfigSync.class);
/** Cluster cache name of the bucket rows' presign limits (cleared on every node when a bucket row changes). */
public static final String ROW_LIMITS_CACHE_NAME = "qbit-object-storage.bucket-row-limits";
private static final Map<String, Integer> BUCKET_IDS = new ConcurrentHashMap<>();
private static final Map<String, Integer> CONNECTION_IDS = new ConcurrentHashMap<>();
private static final Map<String, Integer> BUCKET_IDS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"bucket alias to row id, written by the config sync; bucket rows cannot be inserted or deleted through the UI or API, so the ids are the same on every node");
private static final Map<String, Integer> CONNECTION_IDS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"connection name to row id, written by the config sync; connection rows cannot be inserted or deleted through the UI or API, so the ids are the same on every node");
private static final Map<String, RowLimits> ROW_LIMITS = ClusterCaches.shared(ROW_LIMITS_CACHE_NAME, new ConcurrentHashMap<>(),
ConfigSync::clearRowLimits, alias -> ConfigSync.ROW_LIMITS.remove(alias));
private static final Map<String, RowLimits> ROW_LIMITS = new ConcurrentHashMap<>();
private static final long ROW_LIMIT_CACHE_MILLIS = 15_000L;
private static volatile boolean synced = false;
@@ -320,8 +313,7 @@ public final class ConfigSync
/*******************************************************************************
** Forget the cached row limits on this node (a bucket row edit clears them on
** every node through ConfigRowChangeCustomizer; they also expire).
** Forget the cached row limits (after a row edit in a test; they also expire).
*******************************************************************************/
public static void clearRowLimits()
{
@@ -22,6 +22,7 @@ package com.kingsrook.qbits.objectstorage.internal;
import java.net.URI;
import java.util.function.Function;
import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
@@ -29,10 +30,17 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3ClientFactory;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.DelegatingS3Client;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.S3Request;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
@@ -190,7 +198,16 @@ public class ConnectionRuntime
*******************************************************************************/
private S3Client buildClient()
{
return (S3ClientFactory.buildClient(backendMetaData()));
S3BackendMetaData backend = backendMetaData();
String token = config.getCredentialsMode() == CredentialsMode.STATIC_REFS ? SecretRefs.resolve(config.getSessionTokenRef()) : null;
AwsCredentialsProvider sessionCredentials = null;
if(token != null && backend.getAccessKey() != null && backend.getSecretKey() != null)
{
sessionCredentials = StaticCredentialsProvider.create(AwsSessionCredentials.create(backend.getAccessKey(), backend.getSecretKey(), token));
}
S3Client coreClient = S3ClientFactory.buildClient(backend);
return (sessionCredentials == null ? coreClient : new SessionCredentialsClient(coreClient, sessionCredentials));
}
@@ -212,7 +229,8 @@ public class ConnectionRuntime
.withSocketTimeoutMillis(config.getReadTimeoutMs());
if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
{
withStaticCredentials(backend, config.getAccessKeyRef(), config.getSecretKeyRef());
backend.withAccessKey(SecretRefs.resolve(config.getAccessKeyRef()))
.withSecretKey(SecretRefs.resolve(config.getSecretKeyRef()));
}
return (backend);
}
@@ -252,33 +270,63 @@ public class ConnectionRuntime
/*******************************************************************************
** Static keys (with the session token, when set) from the references, or the
** SDK default chain; core builds the provider.
** Static keys from the references, or the SDK default chain.
*******************************************************************************/
private AwsCredentialsProvider credentials(String accessRef, String secretRef)
{
S3BackendMetaData credentials = new S3BackendMetaData();
if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
{
withStaticCredentials(credentials, accessRef, secretRef);
String accessKey = SecretRefs.resolve(accessRef);
String secretKey = SecretRefs.resolve(secretRef);
if(accessKey != null && secretKey != null)
{
String token = SecretRefs.resolve(config.getSessionTokenRef());
return (StaticCredentialsProvider.create(token == null ? AwsBasicCredentials.create(accessKey, secretKey)
: AwsSessionCredentials.create(accessKey, secretKey, token)));
}
}
return (S3ClientFactory.buildCredentialsProvider(credentials));
return (DefaultCredentialsProvider.builder().build());
}
/*******************************************************************************
** Resolve static keys onto core meta data. The session token applies only
** with both keys, as before (core rejects a token without keys).
** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.9)
** RC.8 S3BackendMetaData and S3ClientFactory still lack a session token or
** credentials-provider hook (verified against the released JAR).
** Override request credentials only; core still owns the client and transport.
*******************************************************************************/
private void withStaticCredentials(S3BackendMetaData backend, String accessRef, String secretRef)
private static class SessionCredentialsClient extends DelegatingS3Client
{
String accessKey = SecretRefs.resolve(accessRef);
String secretKey = SecretRefs.resolve(secretRef);
backend.withAccessKey(accessKey).withSecretKey(secretKey);
if(accessKey != null && secretKey != null)
private final AwsCredentialsProvider credentialsProvider;
/*******************************************************************************
** Wrap a core-built client with the connection's session credentials.
*******************************************************************************/
private SessionCredentialsClient(S3Client delegate, AwsCredentialsProvider credentialsProvider)
{
backend.withSessionToken(SecretRefs.resolve(config.getSessionTokenRef()));
super(delegate);
this.credentialsProvider = credentialsProvider;
}
/*******************************************************************************
** Preserve request overrides and the concrete request type while adding the
** session token to signing, including streaming and paginated operations.
*******************************************************************************/
@Override
@SuppressWarnings("unchecked") // toBuilder().build() retains the concrete request type T.
protected <T extends S3Request, ReturnT> ReturnT invokeOperation(T request, Function<T, ReturnT> operation)
{
AwsRequestOverrideConfiguration.Builder overrides = request.overrideConfiguration()
.map(AwsRequestOverrideConfiguration::toBuilder).orElseGet(AwsRequestOverrideConfiguration::builder);
T signedRequest = (T) request.toBuilder()
.overrideConfiguration(overrides.credentialsProvider(credentialsProvider).build())
.build();
return (operation.apply(signedRequest));
}
}
}
@@ -48,21 +48,9 @@ import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
** Small helpers for reading and writing the qbit's own tables as the system
** user (the service writes its audit log and index regardless of the calling
** user's per-bucket scope; the calling user's permissions were checked first).
**
** Every write made here carries core's per-call permission bypass
** (AbstractActionInput.withPermissionBypass). That mark, not the session, is
** what lets the service write tables nobody may write directly: a host that
** enforces permissions inside the core actions asks ObjectStoragePermissionChecker
** about every outermost write that does not carry a bypass, and the checker
** refuses those tables to every session, the system user included.
*******************************************************************************/
public final class Db
{
/** The reason recorded on (and logged by core for) each write the service makes of its own tables. */
public static final String OWN_WRITE_REASON = "The object storage service writes its own tables (operation log, index, capabilities, usage); the caller's storage permission was checked before the storage call.";
/*******************************************************************************
** Something that may throw.
*******************************************************************************/
@@ -249,9 +237,7 @@ public final class Db
*******************************************************************************/
public static List<QRecord> insert(String table, List<QRecord> records) throws QException
{
InsertInput input = new InsertInput(table).withRecords(records);
input.withPermissionBypass(OWN_WRITE_REASON);
return (new InsertAction().execute(input).getRecords());
return (new InsertAction().execute(new InsertInput(table).withRecords(records)).getRecords());
}
@@ -273,9 +259,7 @@ public final class Db
*******************************************************************************/
public static List<QRecord> update(String table, List<QRecord> records) throws QException
{
UpdateInput input = new UpdateInput(table).withRecords(records);
input.withPermissionBypass(OWN_WRITE_REASON);
return (new UpdateAction().execute(input).getRecords());
return (new UpdateAction().execute(new UpdateInput(table).withRecords(records)).getRecords());
}
@@ -287,9 +271,7 @@ public final class Db
{
if(!ids.isEmpty())
{
DeleteInput input = new DeleteInput(table).withPrimaryKeys(ids);
input.withPermissionBypass(OWN_WRITE_REASON);
new DeleteAction().execute(input);
new DeleteAction().execute(new DeleteInput(table).withPrimaryKeys(ids));
}
}
@@ -23,14 +23,12 @@ package com.kingsrook.qbits.objectstorage.internal;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
import com.kingsrook.qbits.objectstorage.api.ObjectHeaderPolicy;
import com.kingsrook.qbits.objectstorage.api.ObjectHeaders;
import com.kingsrook.qbits.objectstorage.config.BucketRole;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -46,31 +44,19 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
{
private static final QLogger LOG = QLogger.getLogger(DefaultObjectHeaderPolicy.class);
/** Cluster cache name of the enabled header rules (cleared on every node when a rule changes). */
public static final String CACHE_NAME = "qbit-object-storage.header-rules";
private static final long CACHE_MILLIS = 15_000L;
private static final AtomicReference<CachedRules> CACHED = ClusterCaches.shared(CACHE_NAME, new AtomicReference<>(), DefaultObjectHeaderPolicy::clearCache, null);
private static volatile List<QRecord> cachedRules;
private static volatile long cachedAt;
/*******************************************************************************
** The enabled rules and when they were read.
*******************************************************************************/
private record CachedRules(List<QRecord> rules, long readAt)
{
}
/*******************************************************************************
** Drop the cached rules on this node (a rule edit clears them on every node
** through ConfigRowChangeCustomizer).
** Drop cached rules (after the rules change).
*******************************************************************************/
public static void clearCache()
{
CACHED.set(null);
cachedRules = null;
}
@@ -133,12 +119,11 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
*******************************************************************************/
private static List<QRecord> rules()
{
CachedRules cached = CACHED.get();
if(cached != null && System.currentTimeMillis() - cached.readAt() < CACHE_MILLIS)
List<QRecord> rules = cachedRules;
if(rules != null && System.currentTimeMillis() - cachedAt < CACHE_MILLIS)
{
return (cached.rules());
return (rules);
}
List<QRecord> rules;
if(!Db.available(ObjectStorageTables.HEADER_RULE))
{
return (List.of());
@@ -148,7 +133,8 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
rules = Db.asSystem(() -> Db.query(ObjectStorageTables.HEADER_RULE, "enabled", true));
rules.sort(Comparator.comparing((QRecord r) -> r.getValueInteger("priority") == null ? 100 : r.getValueInteger("priority"))
.thenComparing(r -> r.getValueInteger("id") == null ? 0 : r.getValueInteger("id")));
CACHED.set(new CachedRules(rules, System.currentTimeMillis()));
cachedRules = rules;
cachedAt = System.currentTimeMillis();
return (rules);
}
catch(Exception e)
@@ -33,7 +33,6 @@ import com.kingsrook.qbits.objectstorage.api.StorageAlertSink;
import com.kingsrook.qbits.objectstorage.api.StorageChangeListener;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
/*******************************************************************************
@@ -44,11 +43,9 @@ import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
*******************************************************************************/
public final class ObjectStorageRuntime
{
private static final Map<String, ConnectionRuntime> CONNECTIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"this node's S3 clients, built from the code config at first use; row overrides apply when a client is built, the same on every node");
private static final Map<String, ConnectionRuntime> CONNECTIONS = new ConcurrentHashMap<>();
private static final List<StorageChangeListener> LISTENERS = new CopyOnWriteArrayList<>();
private static final Map<String, String> PROCESS_PERMISSIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"built from the qbit config at startup, the same on every node");
private static final Map<String, String> PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
private static volatile ObjectStorageQBitConfig config;
private static volatile ObjectHeaderPolicy headerPolicy;
@@ -99,7 +96,6 @@ public final class ObjectStorageRuntime
clock = Clock.systemUTC();
indexHook = null;
CapabilityStore.clearCache();
OperationLog.reset();
}
@@ -21,17 +21,10 @@
package com.kingsrook.qbits.objectstorage.internal;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import com.kingsrook.qbits.objectstorage.api.StorageCaller;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -40,22 +33,12 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** every caller (UI action, app code, scheduled job) lands here. Presigned URLs
** are logged without the signature (key, method, expiry and issuer only),
** because the URL itself is a bearer token. A logging failure never fails the
** operation it describes, so it is reported three ways instead: an ERROR log
** line, the storage.operation_log.failed alert (StorageAlertSink, resolved by
** the next row written for that bucket), and failedWriteCount().
** operation it describes.
*******************************************************************************/
public final class OperationLog
{
private static final QLogger LOG = QLogger.getLogger(OperationLog.class);
/** The alert raised (severity error, dedupe key "bucket:" + alias) when a row could not be written. */
public static final String ALERT_TYPE = "storage.operation_log.failed";
private static final AtomicLong FAILED_WRITES = new AtomicLong();
private static final Set<String> UNRESOLVED = ClusterCaches.nodeLocal(ConcurrentHashMap.newKeySet(),
"the buckets whose operation log alert this node raised and has not resolved (alert state, not cached data)");
/*******************************************************************************
@@ -200,31 +183,6 @@ public final class OperationLog
/*******************************************************************************
** How many operation log rows this node could not write since it started (or
** since the runtime was reset). The operations themselves were carried out, so
** anything above zero means storage changes that the audit log does not show.
** A host that has no alert sink reads this (a metrics gauge, a health check of
** its own) to alert on a missing audit row.
*******************************************************************************/
public static long failedWriteCount()
{
return (FAILED_WRITES.get());
}
/*******************************************************************************
** Forget the failures counted and the alerts raised (ObjectStorageRuntime.reset).
*******************************************************************************/
static void reset()
{
FAILED_WRITES.set(0);
UNRESOLVED.clear();
}
/*******************************************************************************
** Insert the row (as the system user), never throwing.
*******************************************************************************/
@@ -238,7 +196,7 @@ public final class OperationLog
try
{
Integer bucketId = ConfigSync.bucketId(e.bucketAlias);
QRecord written = Db.asSystem(() -> Db.insert(ObjectStorageTables.OPERATION, new QRecord()
Db.asSystem(() -> Db.insert(ObjectStorageTables.OPERATION, new QRecord()
.withValue("occurredAt", ObjectStorageRuntime.clock().instant())
.withValue("actor", e.caller.describe())
.withValue("action", e.action)
@@ -252,66 +210,10 @@ public final class OperationLog
.withValue("providerRequestId", e.requestId)
.withValue("processRunId", e.processRunId)
.withValue("detail", e.detail)));
////////////////////////////////////////////////////////////////////////////////
// the insert action does not throw for a record it refused (a missing value, //
// a value too long, a record lock): it hands the record back with its errors //
////////////////////////////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(written.getErrors()))
{
throw (new QException("The operation log row was refused: " + written.getErrors()));
}
written(e);
}
catch(Exception ex)
{
failed(e, ex);
}
}
/*******************************************************************************
** A row was written: resolve the alert of an earlier failure for its bucket.
*******************************************************************************/
private static void written(Entry e)
{
String dedupeKey = "bucket:" + e.bucketAlias;
if(UNRESOLVED.remove(dedupeKey))
{
try
{
ObjectStorageRuntime.alertSink().resolve(ALERT_TYPE, dedupeKey);
}
catch(RuntimeException sinkFailure)
{
LOG.warn("Could not resolve the storage operation log alert", sinkFailure, logPair("bucket", e.bucketAlias));
}
}
}
/*******************************************************************************
** A row could not be written: the operation happened and the audit log does
** not show it. Say so where an operator sees it, without failing the operation.
*******************************************************************************/
private static void failed(Entry e, Exception cause)
{
long count = FAILED_WRITES.incrementAndGet();
LOG.error("Could not write a storage operation log row: the operation is missing from the audit log", cause,
logPair("action", e.action), logPair("bucket", e.bucketAlias), logPair("key", e.key), logPair("result", e.result), logPair("failedWriteCount", count));
String dedupeKey = "bucket:" + e.bucketAlias;
UNRESOLVED.add(dedupeKey);
try
{
ObjectStorageRuntime.alertSink().raise(ALERT_TYPE, dedupeKey, "error", "A storage operation on " + e.bucketAlias + " was not recorded in the operation log",
Map.of("bucket", String.valueOf(e.bucketAlias), "action", String.valueOf(e.action), "count", count));
}
catch(RuntimeException sinkFailure)
{
LOG.warn("Could not raise the storage operation log alert", sinkFailure, logPair("bucket", e.bucketAlias));
LOG.warn("Could not write a storage operation log row", ex, logPair("action", e.action), logPair("key", e.key));
}
}
}
@@ -26,7 +26,6 @@ import java.util.Map;
import com.kingsrook.qbits.objectstorage.api.Capability;
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
import com.kingsrook.qbits.objectstorage.config.ProviderType;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.PARTIAL;
import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.SUPPORTED;
import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.UNKNOWN;
@@ -40,8 +39,7 @@ import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.UNSUPPORTED
*******************************************************************************/
public final class ProviderDefaults
{
private static final Map<ProviderType, Map<Capability, CapabilityStatus>> TABLE = ClusterCaches.nodeLocal(new EnumMap<>(ProviderType.class),
"provider defaults from code, the same on every node");
private static final Map<ProviderType, Map<Capability, CapabilityStatus>> TABLE = new EnumMap<>(ProviderType.class);
static
{
@@ -21,6 +21,10 @@
package com.kingsrook.qbits.objectstorage.internal;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.regex.Pattern;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
@@ -28,12 +32,14 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Enforces the qbit's secret-reference allow-list and resolves references
** (${env.X}, ${prop.X}, ${file.PATH}) through core QMetaDataVariableInterpreter.
** Core reads ${file.PATH} read-only from its allowed roots (default
** /var/run/secrets and /run/secrets; qqq.fileReference.roots or
** QQQ_FILE_REFERENCE_ROOTS). Resolved values live only in memory, never in a
** record, a log line or a process value.
** Resolves secret references (${env.X}, ${prop.X}, ${file.PATH}) and enforces
** the allow-list. Resolved values live only in memory, never in a record, a log
** line or a process value.
**
** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.9)
** RC.8 QMetaDataVariableInterpreter still supports env/prop, not file references.
** ${file.PATH} moves to QMetaDataVariableInterpreter in QQQ core
** (spec section 8.2); then this class delegates entirely.
*******************************************************************************/
public final class SecretRefs
{
@@ -87,7 +93,19 @@ public final class SecretRefs
{
return (null);
}
String value = new QMetaDataVariableInterpreter().interpret(ref.trim());
String trimmed = ref.trim();
if(trimmed.startsWith("${file.") && trimmed.endsWith("}"))
{
try
{
return (Files.readString(Path.of(trimmed.substring("${file.".length(), trimmed.length() - 1)), StandardCharsets.UTF_8).trim());
}
catch(IOException e)
{
return (null);
}
}
String value = new QMetaDataVariableInterpreter().interpret(trimmed);
return (value == null || value.startsWith("${") ? null : value);
}
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.internal.ConfigRowChangeCustomizer;
import com.kingsrook.qbits.objectstorage.permissions.ObjectStoragePermissionChecker;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
@@ -565,9 +564,7 @@ public final class ObjectStorageTables
.withField(ro("orphanedCount", QFieldType.LONG))
.withField(ro("orphanedBytes", QFieldType.LONG))
.withField(ro("missingCount", QFieldType.LONG))
.withField(ro("lastReferenceScanDate", QFieldType.DATE_TIME)), KEY_TYPE_BUCKET, "id", false)
.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class)));
.withField(ro("lastReferenceScanDate", QFieldType.DATE_TIME)), KEY_TYPE_BUCKET, "id", false));
}
@@ -704,10 +701,7 @@ public final class ObjectStorageTables
.withField(str("cacheControl", 300))
.withField(str("contentDisposition", 300))
.withField(new QFieldMetaData("priority", QFieldType.INTEGER).withDefaultValue(100))
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)), KEY_TYPE_BUCKET, "bucketId", true)
.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
.withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class)));
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)), KEY_TYPE_BUCKET, "bucketId", true));
}
@@ -46,9 +46,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
** - writes to the editable config tables (connection operational fields, header
** rules, protection rules, bucket thresholds) need object_storage.admin;
** observed and history tables are never written through the UI or API (they
** change only through actions and the service, whose own writes carry
** core's per-call permission bypass, see internal.Db; no session is let
** through, the core system user included);
** change only through actions and the service);
** - each process needs the permission registered for it (default admin).
*******************************************************************************/
public class ObjectStoragePermissionChecker implements CustomPermissionChecker
@@ -67,12 +65,6 @@ public class ObjectStoragePermissionChecker implements CustomPermissionChecker
String needed = required(actionInput, metaData);
if(needed == null)
{
////////////////////////////////////////////////////////////////////////////////////////
// nobody writes these tables directly, whatever the session: a person, the storage //
// administrator and the core system user are all refused. the service's own writes //
// (internal.Db) carry core's per-call permission bypass, so core does not ask about //
// them; a write that reaches this point did not come from the service //
////////////////////////////////////////////////////////////////////////////////////////
throw (new QPermissionDeniedException("This storage table changes only through storage actions."));
}
if(!has(needed))
@@ -1,5 +1,5 @@
bundle: qbit-object-storage
version: ${project.version}
version: 0.1.0-SNAPSHOT
layer: QBIT
entries:
- key: app:objectStorageAdmin;slot:help
@@ -1 +0,0 @@
version=${project.version}
@@ -59,13 +59,7 @@ class ObjectStorageHelpBundleTest
assertThat(instance.getTable("helpContent")).isNull();
assertThat(HelpBundles.of(instance).getBundles()).singleElement()
.satisfies(bundle ->
{
assertThat(bundle.getId()).isEqualTo("qbit-object-storage");
assertThat(System.getProperty("qbit.artifactVersion")).isNotBlank();
assertThat(bundle.getVersion()).isEqualTo(System.getProperty("qbit.artifactVersion"))
.isEqualTo(ObjectStorageQBitProducer.VERSION);
});
.satisfies(bundle -> assertThat(bundle.getId()).isEqualTo("qbit-object-storage"));
HelpBundles.syncOnStartup(instance);
Set<String> keys = HelpBundles.of(instance).getBundles().get(0).getEntries().stream()
.map(entry -> entry.getKey()).collect(Collectors.toSet());
@@ -44,10 +44,6 @@ class ObjectStorageQBitProducerTest
ObjectStorageQBitConfig config = new ObjectStorageQBitConfig().withBackendName("rdbms");
ObjectStorageQBitProducer producer = new ObjectStorageQBitProducer().withQBitConfig(config).withNamespace("object_storage");
String artifactVersion = System.getProperty("qbit.artifactVersion");
assertThat(artifactVersion).isNotBlank();
assertThat(ObjectStorageQBitProducer.VERSION).isEqualTo(artifactVersion);
assertThat(producer.getQBitMetaData().getVersion()).isEqualTo(artifactVersion);
assertThat(producer.isEnabled()).isTrue();
assertThat(producer.getQBitConfig()).isSameAs(config);
assertThat(producer.getNamespace()).isEqualTo("object_storage");
@@ -574,40 +574,4 @@ class MinioIndexTest
.contains("Usage trend").contains("collecting since")
.contains("Connections").contains("Never tested");
}
/*******************************************************************************
** Every link the home emits is a /app/<table>[...] route that the Next frontend
** resolves, never an /<appName>/<table> path, with data and without.
*******************************************************************************/
@Test
void everyHomeLinkUsesTheAppRoute() throws Exception
{
String prefix = com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix();
assertThat(prefix).isEqualTo("/app");
java.util.List<String> hrefs = new java.util.ArrayList<>(hrefs(com.kingsrook.qbits.objectstorage.app.StorageHome.render(prefix, Instant.now())));
rawPut(target.indexBucket, "media/1/a.jpg", 2048);
rawPut(target.indexBucket, "media/9/orphan.jpg", 1024);
ObjectIndex.scan("indexed");
ObjectIndex.headBacklog("indexed", 10);
ReferenceScan.run("indexed");
ObjectStorage.bucket("indexed").put(PutRequest.of("media/1/c.jpg", new byte[3]));
hrefs.addAll(hrefs(com.kingsrook.qbits.objectstorage.app.StorageHome.render(prefix, Instant.now())));
assertThat(hrefs).isNotEmpty().allSatisfy(href -> assertThat(href).startsWith("/app/"));
assertThat(hrefs).noneMatch(href -> href.contains("objectStorageAdmin"));
}
private static java.util.List<String> hrefs(String html)
{
java.util.List<String> hrefs = new java.util.ArrayList<>();
java.util.regex.Matcher m = java.util.regex.Pattern.compile("href=\"([^\"]*)\"").matcher(html);
while(m.find())
{
hrefs.add(m.group(1).replace("&amp;", "&"));
}
return (hrefs);
}
}
@@ -38,7 +38,6 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMe
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpServer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -59,21 +58,9 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
*******************************************************************************/
class ConnectionRuntimeTest
{
/*******************************************************************************
** Core reads ${file.PATH} only inside its allowed roots; reference() points
** them at each test's temporary directory.
*******************************************************************************/
@AfterEach
void afterEach()
{
System.clearProperty("qqq.fileReference.roots");
}
/*******************************************************************************
** Every core-supported setting is mapped after resolving references. Session
** tokens and presign settings are exercised below.
** tokens and presign settings have no core fields and are exercised below.
*******************************************************************************/
@Test
void testBackendMetaDataCarriesEverySetting(@TempDir Path dir) throws Exception
@@ -212,8 +199,8 @@ class ConnectionRuntimeTest
/*******************************************************************************
** Session credentials (core sessionToken) sign streaming reads/writes and
** paginator requests; request overrides are untouched.
** Session credentials sign streaming reads/writes and paginator requests.
** Existing request headers survive the bridge's credentials override.
*******************************************************************************/
@Test
void testSessionSigningAndRequestOverrides(@TempDir Path dir) throws Exception
@@ -326,7 +313,6 @@ class ConnectionRuntimeTest
*******************************************************************************/
private static String reference(Path dir, String name, String value) throws Exception
{
System.setProperty("qqq.fileReference.roots", dir.toString());
Path file = dir.resolve(name);
Files.writeString(file, value);
return ("${file." + file + "}");
@@ -21,50 +21,25 @@
package com.kingsrook.qbits.objectstorage.internal;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.api.StorageCaller;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qbits.objectstorage.permissions.ObjectStoragePermissions;
import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Db.asSystem runs work as the core system user. Token-based authentication
** modules (OAuth2, Auth0) accept a QSystemUserSession without a token and
** reject any other tokenless session, so the class matters, not just the
** permissions. The system session is not what lets the service write its own
** tables, though: each Db write carries core's per-call permission bypass, and
** a write that does not is refused to every session.
** permissions.
*******************************************************************************/
class DbSystemSessionTest
{
private static final String REFUSAL = "This storage table changes only through storage actions.";
/*******************************************************************************
** Reset the runtime and context after each test.
*******************************************************************************/
@@ -94,140 +69,4 @@ class DbSystemSessionTest
assertThat(inside.hasSecurityKeyValue("tenantAllAccess", true)).isTrue();
assertThat(QContext.getQSession()).isSameAs(caller);
}
/*******************************************************************************
** A host that enforces permissions inside the core actions
** (QInstance.enforcePermissionsInCoreActions) still gets its operation log
** row, and the row names the person who acted: the caller is captured before
** the write moves to the system user. The other tables the service keeps
** (here the object index) are written, changed and removed the same way.
*******************************************************************************/
@Test
void testServiceWritesItsOwnTablesWhenCoreActionsEnforcePermissions() throws Exception
{
enforcingInstance();
QContext.setQSession(new QSession().withUser(new QUser().withIdReference("pat@example.com").withFullName("Pat Example")).withPermissions(ObjectStoragePermissions.WRITE));
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("stored/by/the/service").write();
assertThat(operations()).as("the operation log row of a stored object").singleElement().satisfies(row ->
{
assertThat(row.getValueString("objectKey")).isEqualTo("stored/by/the/service");
assertThat(row.getValueString("actor")).as("the person who acted, not the system user that wrote the row").isEqualTo("pat@example.com");
});
assertThat(OperationLog.failedWriteCount()).isZero();
Serializable id = Db.asSystem(() -> Db.insert(ObjectStorageTables.OBJECT, new QRecord().withValue("objectKey", "a/b.txt").withValue("size", 1L))).getValue("id");
Db.asSystem(() -> Db.update(ObjectStorageTables.OBJECT, List.of(new QRecord().withValue("id", id).withValue("size", 2L))));
assertThat(Db.asSystem(() -> Db.first(ObjectStorageTables.OBJECT, "id", id)).getValueLong("size")).isEqualTo(2L);
Db.asSystem(() ->
{
Db.delete(ObjectStorageTables.OBJECT, List.of(id));
return (null);
});
assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.OBJECT))).isEmpty();
}
/*******************************************************************************
** Being the system user is not enough to write the operation log. Server code
** outside the service's Db (a host, another qbit) that runs as the core system
** user is refused the insert, the update and the delete, also when it borrows
** Db.asSystem for its session.
*******************************************************************************/
@Test
void testSystemSessionOutsideTheServiceCannotWriteTheOperationLog() throws Exception
{
QInstance instance = enforcingInstance();
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("the/audited/key").write();
Serializable id = operations().get(0).getValue("id");
QContext.setQSession(new QSystemUserSession(instance));
assertRefused("insert as the system user", () -> new InsertAction().execute(new InsertInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("action", "PUT")))));
assertRefused("update as the system user", () -> new UpdateAction().execute(new UpdateInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("id", id).withValue("actor", "nobody")))));
assertRefused("delete as the system user", () -> new DeleteAction().execute(new DeleteInput(ObjectStorageTables.OPERATION).withPrimaryKey(id)));
QContext.setQSession(new QSession());
assertRefused("insert inside Db.asSystem, not through Db", () -> Db.asSystem(() -> new InsertAction().execute(new InsertInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("action", "PUT"))))));
assertRefused("update inside Db.asSystem, not through Db", () -> Db.asSystem(() -> new UpdateAction().execute(new UpdateInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("id", id).withValue("actor", "nobody"))))));
assertRefused("delete inside Db.asSystem, not through Db", () -> Db.asSystem(() -> new DeleteAction().execute(new DeleteInput(ObjectStorageTables.OPERATION).withPrimaryKey(id))));
assertUnchanged();
}
/*******************************************************************************
** An operation row is not updated or deleted by an update or delete action
** that does not carry the bypass, whoever asks: a person, the storage
** administrator or the system user, by key or by filter.
*******************************************************************************/
@Test
void testOperationRowIsNeverUpdatedOrDeletedWithoutTheBypass() throws Exception
{
QInstance instance = enforcingInstance();
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("the/audited/key").write();
Serializable id = operations().get(0).getValue("id");
List<QSession> sessions = List.of(new QSession(), new QSession().withPermissions(ObjectStoragePermissions.ADMIN), new QSystemUserSession(instance));
for(QSession session : sessions)
{
QContext.setQSession(session);
String who = session.getClass().getSimpleName() + (session.hasPermission(ObjectStoragePermissions.ADMIN) ? " (admin)" : "");
assertRefused("update by " + who, () -> new UpdateAction().execute(new UpdateInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("id", id).withValue("result", "FAILED")))));
assertRefused("delete by key by " + who, () -> new DeleteAction().execute(new DeleteInput(ObjectStorageTables.OPERATION).withPrimaryKey(id)));
assertRefused("delete by filter by " + who, () -> new DeleteAction().execute(new DeleteInput(ObjectStorageTables.OPERATION).withQueryFilter(new QQueryFilter(Db.eq("action", "PUT")))));
}
assertUnchanged();
}
/*******************************************************************************
** An instance that enforces permissions inside the core actions.
*******************************************************************************/
private static QInstance enforcingInstance() throws QException
{
ObjectStorageQBitConfig config = new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false);
return (StorageTestSupport.setUp(config, instance -> instance.setEnforcePermissionsInCoreActions(true)));
}
/*******************************************************************************
** The operation log, read by the service.
*******************************************************************************/
private static List<QRecord> operations() throws QException
{
return (Db.asSystem(() -> Db.query(ObjectStorageTables.OPERATION)));
}
/*******************************************************************************
** The qbit's checker refused the write.
*******************************************************************************/
private static void assertRefused(String what, ThrowingCallable write)
{
assertThatThrownBy(write).as(what).isInstanceOf(QPermissionDeniedException.class).hasMessage(REFUSAL);
}
/*******************************************************************************
** The one row the service wrote is still there, as it was written.
*******************************************************************************/
private static void assertUnchanged() throws QException
{
assertThat(operations()).singleElement().satisfies(row ->
{
assertThat(row.getValueString("objectKey")).isEqualTo("the/audited/key");
assertThat(row.getValueString("result")).isEqualTo("OK");
assertThat(row.getValueString("actor")).isEqualTo("system (no user)");
});
}
}
@@ -83,23 +83,10 @@ class InternalsTest
assertThat(SecretRefs.resolve(null)).isNull();
assertThat(SecretRefs.resolveSetting("")).isNull();
//////////////////////////////////////////////////////////////////////
// core reads ${file.PATH} only inside its allowed roots (by default //
// /var/run/secrets and /run/secrets); point them at the temp dir. //
//////////////////////////////////////////////////////////////////////
Path file = dir.resolve("key");
Files.writeString(file, " from-file\n");
assertThat(SecretRefs.resolve("${file." + file + "}")).as("outside the allowed roots").isNull();
System.setProperty("qqq.fileReference.roots", dir.toString());
try
{
assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
}
finally
{
System.clearProperty("qqq.fileReference.roots");
}
assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
}
@@ -1,240 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage.internal;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.api.Capability;
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
import com.kingsrook.qbits.objectstorage.config.BucketRole;
import com.kingsrook.qbits.objectstorage.config.ProviderType;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
import com.kingsrook.qqq.backend.core.cluster.ClusterBus;
import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface;
import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData;
import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit;
import com.kingsrook.qqq.backend.core.cluster.ClusterEvent;
import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** The Object Storage caches are correct on several nodes: every static cache
** is registered with ClusterCaches, a probe result recorded on one node clears
** that capability on the others, and an edited bucket row or header rule
** clears its cache on every node.
*******************************************************************************/
class ObjectStorageClusterCachesTest
{
/** the "wire": events every node's transport sees */
static final List<Consumer<ClusterEvent>> WIRE = new CopyOnWriteArrayList<>();
/*******************************************************************************
** A transport over the static WIRE, so a "remote" publish reaches us.
*******************************************************************************/
public static class WireBus implements ClusterBusInterface
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void publish(ClusterEvent event)
{
WIRE.forEach(listener -> listener.accept(event));
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public AutoCloseable subscribe(Consumer<ClusterEvent> listener)
{
WIRE.add(listener);
return (() -> WIRE.remove(listener));
}
}
/*******************************************************************************
** A synced instance whose buckets never touch a real store, on the wire as
** node-a.
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
System.setProperty("os.unit.ak", "ak");
System.setProperty("os.unit.sk", "sk");
StorageTestSupport.setUp(new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false).withOpenWhenAnonymous(true).withAdminApp(true)
.withAllowedSecretRefPatterns(List.of("prop.os.unit.*"))
.withConnection(new StorageConnectionConfig("main").withProviderType(ProviderType.MINIO).withEndpoint("http://localhost:1").withRegion("us-east-1")
.withAccessKeyRef("${prop.os.unit.ak}").withSecretKeyRef("${prop.os.unit.sk}"))
.withBucket(new StorageBucketConfig("media").withConnection("main").withBucketName("media-bucket").withRole(BucketRole.PRIVATE).withIndexed(false)),
instance -> instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class))));
QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a"));
ClusterBus.connect(QContext.getQInstance());
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
void tearDown()
{
ClusterBus.close(QContext.getQInstance());
WIRE.clear();
QNodeRoles.set(null);
StorageTestSupport.tearDown();
}
/*******************************************************************************
** Every static Memoization in the QBit is registered (shared or node-local).
*******************************************************************************/
@Test
void testNoUnregisteredStaticCaches() throws Exception
{
assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.objectstorage")).isEmpty();
}
/*******************************************************************************
** Another node's probe result clears the capability here; the next read
** reloads it from the capability row.
*******************************************************************************/
@Test
void testRemoteProbeResultClearsCapability() throws Exception
{
StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
CapabilityStore.record(bucket, Capability.VERSIONING, CapabilityStatus.UNSUPPORTED, "probe here");
assertThat(CapabilityStore.isCached(bucket, Capability.VERSIONING)).isTrue();
////////////////////////////////////////////////////////////
// node-b probes again and writes SUPPORTED to the row //
////////////////////////////////////////////////////////////
Integer bucketId = ConfigSync.bucketId("media");
Db.asSystem(() ->
{
QRecord row = Db.first(ObjectStorageTables.BUCKET_CAPABILITY, "bucketId", bucketId, "capability", Capability.VERSIONING.name());
Db.update(ObjectStorageTables.BUCKET_CAPABILITY, List.of(new QRecord().withValue("id", row.getValue("id")).withValue("status", CapabilityStatus.SUPPORTED.name())));
return (null);
});
ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, CapabilityStore.CACHE_NAME, "b:media:" + Capability.VERSIONING);
remote.setOriginNodeId("node-b");
WIRE.forEach(listener -> listener.accept(remote));
assertThat(CapabilityStore.isCached(bucket, Capability.VERSIONING)).isFalse();
assertThat(CapabilityStore.status("media", Capability.VERSIONING)).isEqualTo(CapabilityStatus.SUPPORTED);
}
/*******************************************************************************
** A probe result recorded here tells the other nodes, and stays cached here.
*******************************************************************************/
@Test
void testProbeResultPublishesInvalidation()
{
List<ClusterEvent> sent = sentEvents();
StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
CapabilityStore.record(bucket, Capability.VERSIONING, CapabilityStatus.SUPPORTED, "probe here");
assertThat(CapabilityStore.status("media", Capability.VERSIONING)).isEqualTo(CapabilityStatus.SUPPORTED);
assertThat(sent).anySatisfy(event -> assertInvalidation(event, CapabilityStore.CACHE_NAME, "b:media:VERSIONING"));
}
/*******************************************************************************
** An edited bucket row takes effect at once and tells the other nodes.
*******************************************************************************/
@Test
void testBucketRowEditInvalidatesRowLimitsEverywhere() throws Exception
{
StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
assertThat(ConfigSync.effectiveMaxPresignGetSeconds(bucket)).isEqualTo(3600);
List<ClusterEvent> sent = sentEvents();
Db.update(ObjectStorageTables.BUCKET, List.of(new QRecord().withValue("id", ConfigSync.bucketId("media")).withValue("maxPresignGetSeconds", 60)));
assertThat(ConfigSync.effectiveMaxPresignGetSeconds(bucket)).as("no wait for the 15 s cache").isEqualTo(60);
assertThat(sent).anySatisfy(event -> assertInvalidation(event, ConfigSync.ROW_LIMITS_CACHE_NAME, null));
}
/*******************************************************************************
** An edited header rule tells the other nodes.
*******************************************************************************/
@Test
void testHeaderRuleEditInvalidatesRulesEverywhere() throws Exception
{
List<ClusterEvent> sent = sentEvents();
StorageTestSupport.insert(ObjectStorageTables.HEADER_RULE, new QRecord().withValue("name", "all").withValue("keyGlob", "**").withValue("cacheControl", "no-store"));
assertThat(sent).anySatisfy(event -> assertInvalidation(event, DefaultObjectHeaderPolicy.CACHE_NAME, null));
}
/*******************************************************************************
** Collect what this node puts on the wire.
*******************************************************************************/
private static List<ClusterEvent> sentEvents()
{
List<ClusterEvent> sent = new CopyOnWriteArrayList<>();
WIRE.add(sent::add);
return (sent);
}
/*******************************************************************************
** A memo invalidation from node-a for a cache (and key).
*******************************************************************************/
private static void assertInvalidation(ClusterEvent event, String name, String key)
{
assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE);
assertThat(event.getName()).isEqualTo(name);
assertThat(event.getKey()).isEqualTo(key);
assertThat(event.getOriginNodeId()).isEqualTo("node-a");
}
}
@@ -1,125 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage.internal;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.api.StorageAlertSink;
import com.kingsrook.qbits.objectstorage.api.StorageCaller;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
import com.kingsrook.qqq.backend.core.logging.CollectedLogMessage;
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import org.apache.logging.log4j.Level;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** An operation log row that cannot be written does not fail the operation it
** describes, so an operator has to be able to see it: an ERROR log line, the
** storage.operation_log.failed alert and OperationLog.failedWriteCount().
*******************************************************************************/
class OperationLogTest
{
private final List<String> alerts = new ArrayList<>();
/*******************************************************************************
** Reset the runtime, the context and the logger after each test.
*******************************************************************************/
@AfterEach
void tearDown()
{
QLogger.deactivateCollectingLoggerForClass(OperationLog.class);
StorageTestSupport.tearDown();
}
/*******************************************************************************
** A row the insert action refuses (it hands the record back with an error, it
** does not throw) and a row whose insert throws are both counted, logged at
** ERROR and raised as an alert; the caller is not failed. The next row written
** for the bucket resolves the alert and the count stays.
*******************************************************************************/
@Test
void testFailedRowIsAnErrorAnAlertAndACount() throws Exception
{
QInstance instance = StorageTestSupport.setUp(new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false));
ObjectStorageRuntime.setAlertSink(new StorageAlertSink()
{
/*******************************************************************************
** Record the alert.
*******************************************************************************/
@Override
public void raise(String alertType, String dedupeKey, String severity, String message, Map<String, Object> details)
{
alerts.add("raise " + alertType + " " + dedupeKey + " " + severity + " count=" + details.get("count") + " action=" + details.get("action"));
}
/*******************************************************************************
** Record the resolution.
*******************************************************************************/
@Override
public void resolve(String alertType, String dedupeKey)
{
alerts.add("resolve " + alertType + " " + dedupeKey);
}
});
QCollectingLogger logger = QLogger.activateCollectingLoggerForClass(OperationLog.class);
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("a").write();
assertThat(OperationLog.failedWriteCount()).isZero();
assertThat(alerts).isEmpty();
instance.getTable(ObjectStorageTables.OPERATION).getField("detail").setIsRequired(true);
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("refused").write();
assertThat(OperationLog.failedWriteCount()).isEqualTo(1);
assertThat(alerts).containsExactly("raise storage.operation_log.failed bucket:files error count=1 action=PUT");
instance.getTable(ObjectStorageTables.OPERATION).getField("detail").setIsRequired(false);
String backendName = instance.getTable(ObjectStorageTables.OPERATION).getBackendName();
instance.getTable(ObjectStorageTables.OPERATION).setBackendName("no-such-backend");
new OperationLog.Entry(StorageCaller.user(), "DELETE", "files").key("thrown").write();
assertThat(OperationLog.failedWriteCount()).isEqualTo(2);
assertThat(alerts).last().isEqualTo("raise storage.operation_log.failed bucket:files error count=2 action=DELETE");
List<CollectedLogMessage> errors = logger.getCollectedMessages().stream().filter(message -> Level.ERROR.equals(message.getLevel())).toList();
assertThat(errors).hasSize(2).allSatisfy(message -> assertThat(message.getMessage()).contains("Could not write a storage operation log row"));
instance.getTable(ObjectStorageTables.OPERATION).setBackendName(backendName);
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("b").write();
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("c").write();
assertThat(alerts).hasSize(3).last().isEqualTo("resolve storage.operation_log.failed bucket:files");
assertThat(OperationLog.failedWriteCount()).as("the count is of this node since it started, a later row does not lower it").isEqualTo(2);
assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.OPERATION))).extracting(row -> row.getValueString("objectKey")).containsExactly("a", "b", "c");
}
}
@@ -205,7 +205,7 @@ class TablesAndConfigTest
new com.kingsrook.qbits.objectstorage.ObjectStorageQBitProducer().withQBitConfig(config).produce(instance);
assertThat(instance.getApp("objectStorageAdmin").getParentAppName()).isEqualTo("admin");
assertThat(instance.getApp("admin").getChildren()).extracting(c -> c.getName()).contains("objectStorageAdmin");
assertThat(com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix()).isEqualTo("/app");
assertThat(com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix()).isEqualTo("/admin/objectStorageAdmin");
}