Compare commits

..
11 Commits
Author SHA1 Message Date
CircleCI cf2e9fca77 Bump version to 0.1.0-RC.6 [skip ci] 2026-10-14 23:07:52 +00:00
james.maesandClaude Opus 5.5 cd26342cfd Merge develop into release/0.1 for 0.1.0-RC.6 (#18 ConfigSync session fix, #19, #20, #23)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:58:53 -05:00
CircleCI 93c116ff01 Bump version to 0.1.0-RC.5 [skip ci] 2026-10-14 22:34:39 +00:00
james.maesandClaude Opus 5.5 9f2d7fc202 Merge develop into release/0.1 for 0.1.0-RC.5 (#21 setup card, seeds, help)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:25:24 -05:00
CircleCI 5c8f2f595b Bump version to 0.1.0-RC.4 [skip ci] 2026-10-14 12:37:00 +00:00
james.maesandClaude Opus 5.5 1ea40241f7 Merge develop into release/0.1 for 0.1.0-RC.4 (#13 maxUploadBytes, #14 QQQ RC.6 and guardrails)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 07:28:10 -05:00
CircleCI 8692857891 Bump version to 0.1.0-RC.3 [skip ci] 2026-10-14 01:02:56 +00:00
james.maesandClaude Opus 5.5 b6db9aff8b Merge develop into release/0.1 for 0.1.0-RC.3 (ConnectionRuntime/S3ClientFactory consolidation, #11)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 19:55:34 -05:00
CircleCI 07275bce3f Bump version to 0.1.0-RC.2 [skip ci] 2026-10-13 15:51:56 +00:00
james.maesandClaude Opus 5.5 9eefc0b071 Merge develop into release/0.1 for 0.1.0-RC.2
AWS SDK for Java v2 2.41.10 -> 2.41.34 (GHSA-443w-3rq3-5m5h, #8).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 10:44:08 -05:00
CircleCI f1fb2462f9 Bump version to 0.1.0-RC.1 [skip ci] 2026-10-12 23:02:00 +00:00
43 changed files with 314 additions and 3084 deletions

No files matched your search

+57 -11
View File
@@ -1,14 +1,60 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.8
qqq-orb: kingsrook/qqq-orb@0.6.9
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
workflows:
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/^v\d+\.\d+\.\d+$/]
-7
View File
@@ -1,7 +0,0 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
-23
View File
@@ -1,23 +0,0 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+13 -62
View File
@@ -1,74 +1,25 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
'on':
on:
workflow_dispatch:
inputs:
dry_run:
description: Plan a cut without changing release state
description: Print the plan without merging or creating branches
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
types: [closed]
branches: [develop]
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
-16
View File
@@ -1,16 +0,0 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
+2 -19
View File
@@ -5,10 +5,7 @@ All notable changes to this project are documented here. The format follows Keep
## [0.1.0] - Unreleased
### Added
- Central health supplier `object_storage.connection.<id>` (DEPENDENCY/STORAGE, 300s interval, 20000ms timeout), reusing bounded read-only capability probes with safe problem kinds and clock-skew details. Turn a connection's check off in the central `health_check` table.
- In-memory health tests for registration, refresh, UP/DOWN/DEGRADED/UNKNOWN outcomes, deadlines, read-only execution and sentinel-secret exclusion.
- Expanded the host-neutral in-app help bundle to cover storage fields, actions, widgets and possible values, with live references and a `HelpBundleAssertions` coverage gate. Registered it for QQQ startup sync and HOST overrides; retained inline table/app help for hosts without help-content tables.
- Every table explains itself: a help bundle (`help/qbit-object-storage.yaml`, QQQ help-bundle format) says what each `object_storage_*` table is for and how its rows get created; it shows on the table screen and when the table is empty. Applied inline by `ObjectStorageHelp` for hosts without help-content tables; a host's own help is kept.
- Every table explains itself: a help bundle (`help/qbit-object-storage.yaml`, QQQ help-bundle format) says what each `object_storage_*` table is for and how its rows get created; it shows on the table screen and when the table is empty. Applied by `ObjectStorageHelp` (STAND-IN for QRun-IO/qqq#1198); a host's own help is kept.
- A Setup card on the Storage home (connections, buckets, header rules, protection rules) with the state and next step of each, resolved the way the runtime resolves them.
- A starter changelog `db/liquibase/qbit-object-storage/object-storage-defaults.changelog.sql`: a disabled `default-mutable` header rule. Insert-only, run through real Liquibase in `DefaultsChangelogTest`.
- `ObjectStorageTablesGuardrailTest` (each table is used at runtime, seeded or explained, and has help).
@@ -26,24 +23,10 @@ All notable changes to this project are documented here. The format follows Keep
- Foreign keys on every relation between the qbit tables (`fk_<table>_<column>`), generated by `ObjectStorageSchema`: owned observed state (capabilities, folders, objects, versions, usage) cascades from its bucket or object, `object.folder_id` and `object_reference.object_id` are set null, and a bucket's connection, header and protection rules and operation log restrict. Each key is added by an idempotent block that first records rows whose parent is gone in `qqq_fk_quarantine` and clears (nullable column) or deletes (required column) them, so it also upgrades an existing schema. New indexes `ix_object_storage_bucket_1` (connection_id), `ix_object_storage_header_rule_1` and `ix_object_storage_protection_rule_1` (bucket_id) back the keys. Upgrade: run the generated statements again (or the host's copy of them).
- Conformance suite against MinIO and Garage (Testcontainers), service, index and action tests, and the index and actions on PostgreSQL.
### Fixed
- The service's own writes are no longer refused in a host that enforces permissions inside the core actions (`QInstance.enforcePermissionsInCoreActions`). `Db.asSystem` starts a new context, so each of its writes is an outermost action there and core asks `ObjectStoragePermissionChecker`, which refuses every write of a table nobody may write directly, to every session. The `object_storage_operation` row of every stored object was lost (logged as "Could not write a storage operation log row", the object itself was stored), and so was each other system write of those tables. `Db.insert`, `Db.update` and `Db.delete` now mark their input with core's per-call permission bypass (`withPermissionBypass`, reason `Db.OWN_WRITE_REASON`), so core does not ask about the service's own writes. The checker is unchanged in what it refuses: a write of those tables that does not carry a bypass is refused to a person, to `object_storage.admin` and to the core system user alike, so server code running as the system user cannot edit or delete operation log rows through an enforced action.
- An operation log row that could not be written is no longer only a warning. It is logged at ERROR, raises the `storage.operation_log.failed` alert (`StorageAlertSink`, severity `error`, dedupe key `bucket:<alias>`, resolved by the next row written for that bucket) and is counted in `OperationLog.failedWriteCount()` (per node, since start). A row the insert action hands back with a record error (a refused value, a record lock) is now detected as well; before, it was dropped without any log line from the qbit. The operation the row describes still succeeds.
### Removed
- The QRun-IO/qqq#1165 stand-ins: `SessionCredentialsClient` (session tokens now go through core `S3BackendMetaData.sessionToken` / `S3ClientFactory`, including for the presigner) and the file reading in `SecretRefs` (core `QMetaDataVariableInterpreter` resolves `${file.PATH}`). **Behavior change:** core reads `${file.PATH}` only inside its allowed roots, which are `/var/run/secrets` and `/run/secrets` by default. A host that mounts storage secrets elsewhere must set `qqq.fileReference.roots` or `QQQ_FILE_REFERENCE_ROOTS`. The qbit's default allow-list (`file./var/run/secrets/storage/*`) is already inside the default roots.
- None for the health migration; both Test connection record actions, including on-demand write tests, are retained.
### Changed
- Multi-node deployments: the runtime caches are registered with QQQ `ClusterCaches`. Shared (cleared on every node): the bucket rows' presign limits (`qbit-object-storage.bucket-row-limits`, on a bucket row insert or update), the enabled header rules (`qbit-object-storage.header-rules`, on a header rule insert, update or delete), both through the new `ConfigRowChangeCustomizer`, and the capability statuses (`qbit-object-storage.capabilities`: a probe result recorded on one node clears that capability on the others, which reload it from the capability row; before, other nodes kept the old status until restart). The 15 s TTLs remain the safety net. Node-local, with reasons: the alias and connection row-id maps, the S3 client map, the process-permission map, the provider defaults, the running-scan guard and the per-pod presign rate limiter (`StorageBucket.PRESIGN_COUNTS`; per-node rate limits are accepted for now). `ObjectStorageClusterCachesTest` audits the package for unregistered static caches and covers remote and local invalidation.
- Builds against QQQ `4.1.0-SNAPSHOT` (`qqq.snapshot.version`) for `ClusterCaches` until the next QQQ RC ships; a release must pin that RC.
- Read the qbit metadata version from the Maven-filtered artifact version so release metadata matches the build and help bundle.
- Rechecked the released RC.8 JARs: core still has no S3 session-token/provider hook or file-reference interpreter. Retained both compatibility shims and moved their review deadline to RC.9 under issue #10.
- Raised the QQQ BOM baseline to released `4.1.0-RC.8` for the central health API.
- Raised the QQQ BOM baseline to released `4.1.0-RC.7` for the core help-bundle registry and coverage assertions.
- Raised the QQQ BOM baseline to released `4.1.0-RC.5` and delegated S3 client construction to core `S3ClientFactory.buildClient`. Removed local HTTP client and trust-manager implementations; existing endpoint, region, addressing, checksum, TLS/CA, credentials and timeout settings remain supported.
- Clients remain owned by each connection runtime and are rebuilt on reset/reinstall. Retry behavior now follows core/SDK defaults instead of forcing `ADAPTIVE_V2`; hosts can select the SDK retry mode through `AWS_RETRY_MODE` or `aws.retryMode`.
- Retained local presigning (including a separate endpoint/key). A request-credentials bridge preserves static session tokens because RC.5 core metadata/factory has no session-token field or credentials-provider hook. This bridge and the existing `${file.PATH}` resolver remain necessary in released RC.7 (no session-token/provider hook or file-reference interpreter); their review deadline is now RC.8 under issue #10.
- Retained local presigning (including a separate endpoint/key). A request-credentials bridge preserves static session tokens because RC.5 core metadata/factory has no session-token field or credentials-provider hook. This bridge and the existing `${file.PATH}` resolver are marked for removal by RC.7 under issue #10.
- Added the filesystem module with its unused SFTP/SSH stack excluded. Resolved dependencies otherwise only change QQQ RC.4 to RC.5 and jsoup 1.23.1 to 1.23.2; see [dependency details](docs/S3-FACTORY-MIGRATION.md).
### Notes
+1 -31
View File
@@ -15,7 +15,7 @@ S3-compatible storage as a capability for any QQQ app, and an admin app to run i
## Requirements
- Java 21.
- QQQ `4.1.0-RC.8` (released BOM), including `qqq-backend-module-filesystem`. `ConnectionRuntime` maps connection settings to core's `S3BackendMetaData` and uses `S3ClientFactory.buildClient`; core owns HTTP, TLS and checksum setup. The unused SFTP/SSH dependency is excluded. See the [dependency diff](docs/S3-FACTORY-MIGRATION.md).
- QQQ `4.1.0-RC.5` (released BOM), including `qqq-backend-module-filesystem`. `ConnectionRuntime` maps connection settings to core's `S3BackendMetaData` and uses `S3ClientFactory.buildClient`; core owns HTTP, TLS and checksum setup. The unused SFTP/SSH dependency is excluded. See the [dependency diff](docs/S3-FACTORY-MIGRATION.md).
- A QQQ backend for the qbit's tables (PostgreSQL is the intended store). `ObjectStorageSchema.postgresScript(config)` generates the DDL from the same metadata the app runs on, for your Liquibase or Flyway migration.
## Install
@@ -47,20 +47,6 @@ Connections and buckets come from this code only. Secrets are **references** (`$
Client settings take effect when the qbit configuration is reinstalled or the runtime is reset. Presigning keeps its separate public endpoint and credentials. Retry behavior now follows core/SDK defaults; hosts requiring adaptive retries can set `AWS_RETRY_MODE=adaptive` or `aws.retryMode=adaptive`. Static session tokens remain supported through a small local bridge pending core support.
## In-app help
The producer registers [help/qbit-object-storage.yaml](src/main/resources/help/qbit-object-storage.yaml)
as a `QBIT` bundle. QQQ syncs it when the host has the help-content tables installed;
registration is safe without them, and inline table/app help still explains empty lists.
The bundle version is the literal POM version; update both when releasing.
Hosts can register their own `layer: HOST` bundle with `HelpBundles.register(qInstance, "help/my-host.yaml")`.
Use the same entry key and role to override qbit text, keeping live `{{...}}` references where useful.
Hosts that rename the optional Storage app should provide an app entry for their configured name.
`ObjectStorageHelpBundleTest` checks all owned tables, fields, processes, apps, widgets and
possible-value sources using `HelpBundleAssertions`, including valid keys, live references and
plain summaries of at most 120 characters. It runs with `./mvnw verify`.
## Use
```java
@@ -92,21 +78,6 @@ Every write is a standard streamed ETL: **Review** (the dry run, with summary li
Permissions: `object_storage.view`, `.write`, `.delete`, `.admin` (admin implies the rest). Per-bucket scoping uses record security locks on the `object_storage_bucket` and `object_storage_connection` security key types (switch off with `withBucketScoping(false)` for a single-administrator app).
## Health checks
The producer registers `object_storage.connection.<id>` for each configured
`object_storage_connection` row: **DEPENDENCY**, **STORAGE**, every **300s**,
with a **20000ms** timeout and component `object_storage`. The central health runner
refreshes the supplier as rows change. The existing `CapabilityProbe` runs only
read-only calls within the deadline; it never runs write or presign tests or stores
free-form capability diagnostics. Connection failures are DOWN, clock skew above
900 seconds is DEGRADED, and connections without buckets are UNKNOWN. Details
contain only problem kind, clock skew seconds and bucket count.
Turn a connection's check off or change its interval in the central `health_check`
table. Both **Test connection** record actions remain available, including
on-demand write tests.
## Setup and starter defaults
The Storage home has a **Setup** card: connections and buckets (declared in code, synced into their tables), header rules, and protection rules, each with its state and next step. Every table has help (`help/qbit-object-storage.yaml`) saying what it is for and how its rows get created, which also shows when the table is empty.
@@ -125,7 +96,6 @@ It is insert-only and inserts a **disabled** generic header rule, `default-mutab
- **Previews** are short-lived (5 minute) presigned GETs, so the Next dashboard's `img-src` must allow the storage host (it allows `https:` by default). Video, audio and PDF previews need same-origin media (the dashboard's `media-src` and `frame-src` are `'self'`); a same-origin download route is the planned follow-up. Until then those files show the file card with download and open links.
- **Scheduler.** Set `withSchedulerName(...)` to run the nightly scan, HEAD backlog, reference scan, usage snapshot and trash purge. Without it, run Rescan bucket by hand.
- **Alerts.** With the Alerts qbit installed, bind a `StorageAlertSink` adapter (`ObjectStorageRuntime.setAlertSink`); without it every condition is still visible on the home and in the tables.
- **A missing audit row.** A storage operation succeeds even when its `object_storage_operation` row cannot be written, and that row is then the one condition no table shows. It is logged at ERROR ("Could not write a storage operation log row"), raised as the `storage.operation_log.failed` alert (severity `error`, dedupe key `bucket:<alias>`, resolved by the next row written for that bucket) and counted in `OperationLog.failedWriteCount()`, per node since it started. A host without an alert sink should export that count (a gauge, or a health check of its own) and alert when it is above zero.
## Conformance
+10 -15
View File
@@ -26,21 +26,16 @@ The public qbit configuration API is unchanged.
## Remaining local code
- Core has no presigner factory. The local presigner keeps its separate public
endpoint and least-privilege keys; its credentials (including the session
token) come from core `S3ClientFactory.buildCredentialsProvider`.
- `SecretRefs` keeps only the qbit's allow-list; resolution, including
`${file.PATH}`, is core `QMetaDataVariableInterpreter`.
## Stand-ins removed (QRun-IO/qqq#1165)
- `SessionCredentialsClient` is gone: core `S3BackendMetaData.sessionToken`
makes `S3ClientFactory` sign with `AwsSessionCredentials`.
- `SecretRefs` no longer reads files itself. Core reads `${file.PATH}`
read-only, and only inside its allowed roots: `/var/run/secrets` and
`/run/secrets` by default, or `qqq.fileReference.roots` /
`QQQ_FILE_REFERENCE_ROOTS` (comma-separated). The qbit's default allow-list
(`file./var/run/secrets/storage/*`) is inside the default roots.
- Core RC.5 has no session-token field or credentials-provider customization
in `S3BackendMetaData` / `S3ClientFactory.buildClient`. The narrow
`SessionCredentialsClient` bridge adds session credentials to requests while
retaining their overrides; core still creates the client and HTTP transport.
Core needs session-token metadata (including enrichment and cache-key support)
and `AwsSessionCredentials` construction, or a caller-provided credentials hook.
Marker: `STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.7)`.
- Core has no presigner factory. The local presigner retains its separate public
endpoint and least-privilege keys, along with existing session-token handling.
- `SecretRefs` file resolution is unchanged and carries the same RC.7 marker.
## Resolved dependency diff
+4 -32
View File
@@ -20,7 +20,7 @@
<properties>
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
<revision>0.2.0-SNAPSHOT</revision>
<revision>0.1.0-RC.6</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<jacoco.version>0.8.15</jacoco.version>
@@ -29,7 +29,7 @@
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.2.1</version>
<version>2.1.0</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -172,35 +172,7 @@
</dependencies>
<build>
<resources>
<!-- help bundles carry the artifact version (QQQ help bundle provenance) -->
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
<includes>
<include>help/*.yaml</include>
<include>qbit-object-storage.properties</include>
</includes>
</resource>
<resource>
<directory>src/main/resources</directory>
<filtering>false</filtering>
<excludes>
<exclude>help/*.yaml</exclude>
<exclude>qbit-object-storage.properties</exclude>
</excludes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables combine.children="append">
<qbit.artifactVersion>${project.version}</qbit.artifactVersion>
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
@@ -283,14 +255,14 @@
<profiles>
<!-- Built against qqq 4.1 (QBitMetaDataProducer). Until qbit-build-parent -->
<!-- imports the released 4.1 BOM, this profile is active by default and -->
<!-- imports qqq-bom-pom 4.1.0-RC.15 ahead of the parent's 4.0.0 BOM. -->
<!-- imports qqq-bom-pom 4.1.0-RC.5 ahead of the parent's 4.0.0 BOM. -->
<profile>
<id>qqq-snapshot</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.25</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.6</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -62,7 +62,6 @@ public class ObjectStorageQBitConfig implements QBitConfig
private String appName = "objectStorageAdmin";
private String appLabel = "Storage";
private String parentAppName;
private String tablePathPrefix;
private List<String> allowedSecretRefPatterns = new ArrayList<>(SecretRefs.DEFAULT_ALLOWED_PATTERNS);
private Integer bulkCap = 10_000;
private Boolean bucketScoping = true;
@@ -426,39 +425,6 @@ public class ObjectStorageQBitConfig implements QBitConfig
/********************************************************************************
** Getter for tablePathPrefix: the front of the links the Storage home makes to
** tables and records. Defaults to /app, the route every QQQ frontend serves
** tables at (/app/<tableName>).
*******************************************************************************/
public String getTablePathPrefix()
{
return (this.tablePathPrefix == null ? "/app" : this.tablePathPrefix);
}
/********************************************************************************
** Setter for tablePathPrefix.
*******************************************************************************/
public void setTablePathPrefix(String tablePathPrefix)
{
this.tablePathPrefix = tablePathPrefix;
}
/********************************************************************************
** Fluent setter for tablePathPrefix.
*******************************************************************************/
public ObjectStorageQBitConfig withTablePathPrefix(String tablePathPrefix)
{
this.tablePathPrefix = tablePathPrefix;
return (this);
}
/********************************************************************************
** Getter for allowedSecretRefPatterns.
*******************************************************************************/
@@ -21,13 +21,9 @@
package com.kingsrook.qbits.objectstorage;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import com.kingsrook.qbits.objectstorage.app.StorageWidgets;
import com.kingsrook.qbits.objectstorage.health.ObjectStorageConnectionChecks;
import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
import com.kingsrook.qbits.objectstorage.internal.DefaultObjectHeaderPolicy;
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
@@ -35,13 +31,9 @@ import com.kingsrook.qbits.objectstorage.model.ObjectStorageHelp;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qbits.objectstorage.processes.ObjectStorageProcesses;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
@@ -63,41 +55,13 @@ public class ObjectStorageQBitProducer implements QBitMetaDataProducer<ObjectSto
{
public static final String GROUP_ID = "com.kingsrook.qbits";
public static final String ARTIFACT_ID = "qbit-object-storage";
public static final String VERSION = loadVersion();
public static final String VERSION = "0.1.0";
private ObjectStorageQBitConfig qBitConfig;
private String namespace;
/*******************************************************************************
** Read the artifact version from the Maven-filtered resource.
*******************************************************************************/
private static String loadVersion()
{
try(InputStream inputStream = ObjectStorageQBitProducer.class.getClassLoader().getResourceAsStream("qbit-object-storage.properties"))
{
if(inputStream == null)
{
throw new IllegalStateException("Missing qbit-object-storage.properties; build with Maven to generate the artifact version");
}
Properties properties = new Properties();
properties.load(inputStream);
String version = properties.getProperty("version");
if(version == null || version.isBlank() || version.contains("${"))
{
throw new IllegalStateException("Missing or unfiltered version in qbit-object-storage.properties; build with Maven");
}
return (version);
}
catch(IOException e)
{
throw new IllegalStateException("Could not read artifact version from qbit-object-storage.properties", e);
}
}
/*******************************************************************************
** Fluent setter for the config.
*******************************************************************************/
@@ -223,9 +187,6 @@ public class ObjectStorageQBitProducer implements QBitMetaDataProducer<ObjectSto
output.add(app);
}
output.addSelfToInstance(qInstance);
HealthChecksMetaData.ofOrWithNew(qInstance).withSupplier(new HealthCheckSupplierMetaData()
.withNamePrefix("object_storage.connection").withComponent("object_storage")
.withSupplier(new QCodeReference(ObjectStorageConnectionChecks.class)));
if(Boolean.TRUE.equals(qBitConfig.getAdminApp()) && qBitConfig.getParentAppName() != null && qInstance.getApp(qBitConfig.getParentAppName()) != null)
{
///////////////////////////////////////////////////////////////////////////////
@@ -233,8 +194,6 @@ public class ObjectStorageQBitProducer implements QBitMetaDataProducer<ObjectSto
///////////////////////////////////////////////////////////////////////////////
qInstance.getApp(qBitConfig.getParentAppName()).withChild(qInstance.getApp(qBitConfig.getAppName()));
}
// Registration is metadata-only; core skips startup sync without a helpContent table.
HelpBundles.register(qInstance, "help/qbit-object-storage.yaml");
ObjectStorageHelp.apply(qInstance, qBitConfig);
return (new MetaDataProducerMultiOutput());
}
@@ -46,7 +46,6 @@ import com.kingsrook.qbits.objectstorage.internal.PostPolicySigner;
import com.kingsrook.qbits.objectstorage.internal.ProtectionRules;
import com.kingsrook.qbits.objectstorage.internal.RateLimiter;
import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -97,8 +96,7 @@ public final class StorageBucket
{
private static final QLogger LOG = QLogger.getLogger(StorageBucket.class);
private static final Map<String, long[]> PRESIGN_COUNTS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"per-pod presign rate limiter; per-node rate limits are accepted for now (cluster HA spec, owner decision 4)");
private static final Map<String, long[]> PRESIGN_COUNTS = new ConcurrentHashMap<>();
private final StorageBucketConfig bucket;
private final ConnectionRuntime connection;
@@ -106,12 +106,16 @@ public final class StorageWidgets
/*******************************************************************************
** The prefix of table and record links: /app, so a link is /app/<tableName>
** (the frontend serves every table, in any app, at that one flat route).
** The prefix of table and record links.
*******************************************************************************/
public static String tablePathPrefix()
{
return (ObjectStorageRuntime.isInstalled() ? ObjectStorageRuntime.getConfig().getTablePathPrefix() : "/app");
if(!ObjectStorageRuntime.isInstalled())
{
return ("/objectStorageAdmin");
}
ObjectStorageQBitConfig config = ObjectStorageRuntime.getConfig();
return (config.getParentAppName() == null ? "/" + config.getAppName() : "/" + config.getParentAppName() + "/" + config.getAppName());
}
@@ -1,121 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage.health;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import com.kingsrook.qbits.objectstorage.internal.CapabilityProbe;
import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
import com.kingsrook.qbits.objectstorage.internal.Db;
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
import com.kingsrook.qqq.backend.core.health.HealthDetails;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
/*******************************************************************************
** Dynamic, read-only checks for configured connection rows. Diagnostic text and
** resolved settings never cross the health boundary.
*******************************************************************************/
public class ObjectStorageConnectionChecks implements HealthCheckSupplier
{
/*******************************************************************************
** Resolve current rows on every refresh; removed configuration is not probed.
*******************************************************************************/
@Override
public List<HealthCheckMetaData> supply(QInstance qInstance) throws QException
{
ConfigSync.ensureSynced();
List<HealthCheckMetaData> checks = new ArrayList<>();
for(QRecord row : Db.asSystem(() -> Db.query(ObjectStorageTables.CONNECTION)))
{
if(ObjectStorageRuntime.getConfig().findConnection(row.getValueString("name")) == null)
{
continue;
}
Integer id = row.getValueInteger("id");
String name = "object_storage.connection." + id.toString().toLowerCase(Locale.ROOT);
checks.add(new HealthCheckMetaData().withName(name).withLabel("Storage connection " + id)
.withComponent("object_storage").withCategory(HealthCategory.STORAGE).withKind(HealthCheckKind.DEPENDENCY)
.withIntervalSeconds(300).withTimeoutMs(20000).withSeverity(AlertSeverity.ERROR)
.withCheckInstance(input -> run(id, input)));
}
return (checks);
}
/*******************************************************************************
** Check the current row, reuse the bounded capability probe, and expose only
** numeric facts and a fixed problem vocabulary (never exception messages).
*******************************************************************************/
private static HealthCheckOutcome run(Integer id, HealthCheckInput input) throws QException
{
if(!Instant.now().isBefore(input.getDeadline()) || Thread.currentThread().isInterrupted())
{
return (HealthCheckOutcome.down("Storage probe timed out").withDetails(new HealthDetails().withValue("problemKind", "timeout")));
}
QRecord row = Db.asSystem(() -> Db.first(ObjectStorageTables.CONNECTION, "id", id));
if(row == null || ObjectStorageRuntime.getConfig().findConnection(row.getValueString("name")) == null)
{
return (HealthCheckOutcome.unknown("Storage connection is disabled or no longer configured"));
}
try
{
CapabilityProbe.Report report = CapabilityProbe.probeConnection(row.getValueString("name"), input.getDeadline());
HealthDetails details = new HealthDetails().withValue("bucketsProbed", report.getByBucket().size());
if(report.getClockSkewSeconds() != null)
{
details.withValue("clockSkewSeconds", report.getClockSkewSeconds());
}
if(report.getProblemKind() != null)
{
return (HealthCheckOutcome.down("Storage connection probe failed").withDetails(details.withValue("problemKind", report.getProblemKind())));
}
if(report.getClockSkewSeconds() != null && report.getClockSkewSeconds() > CapabilityProbe.CLOCK_SKEW_THRESHOLD_SECONDS)
{
return (HealthCheckOutcome.degraded("Storage clock skew exceeds the signing threshold").withDetails(details));
}
if(report.getByBucket().isEmpty())
{
return (HealthCheckOutcome.unknown("No storage buckets configured to probe").withDetails(details));
}
return (HealthCheckOutcome.up("Storage connection is reachable").withDetails(details));
}
catch(RuntimeException e)
{
return (HealthCheckOutcome.down("Storage connection probe failed")
.withDetails(new HealthDetails().withValue("problemKind", CapabilityProbe.problemKind(e))));
}
}
}
@@ -43,7 +43,6 @@ import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
import com.kingsrook.qbits.objectstorage.internal.Db;
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
@@ -79,8 +78,7 @@ public final class ObjectIndex
public static final int PAGE_SIZE = 1000;
private static final Set<String> RUNNING = ClusterCaches.nodeLocal(java.util.concurrent.ConcurrentHashMap.newKeySet(),
"the buckets this node is scanning now (an in-flight guard, not cached data)");
private static final Set<String> RUNNING = java.util.concurrent.ConcurrentHashMap.newKeySet();
/** Test seam: simulate a crash after this many pages of a scan (so resume can be tested). */
static volatile Integer failAfterPages = null;
@@ -21,9 +21,7 @@
package com.kingsrook.qbits.objectstorage.internal;
import javax.net.ssl.SSLException;
import java.net.URI;
import java.net.UnknownHostException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
@@ -37,7 +35,6 @@ import java.util.EnumMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Function;
import com.kingsrook.qbits.objectstorage.api.Capability;
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
import com.kingsrook.qbits.objectstorage.api.PresignedPost;
@@ -48,9 +45,7 @@ import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.services.s3.DelegatingS3Client;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
@@ -66,7 +61,6 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Request;
import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest;
@@ -85,7 +79,6 @@ public final class CapabilityProbe
private static final QLogger LOG = QLogger.getLogger(CapabilityProbe.class);
public static final String PROBE_PREFIX = ".qqq-probe/";
public static final int CLOCK_SKEW_THRESHOLD_SECONDS = 900;
private static final int MIN_PART_BYTES = 5 * 1024 * 1024;
@@ -101,17 +94,6 @@ public final class CapabilityProbe
private final List<String> problems = new ArrayList<>();
private Long clockSkewSeconds;
private Boolean presignHostReachable;
private String problemKind;
/*******************************************************************************
** Safe connection failure category, without provider messages or identifiers.
*******************************************************************************/
public String getProblemKind()
{
return (problemKind);
}
@@ -227,7 +209,7 @@ public final class CapabilityProbe
*******************************************************************************/
public static Report probeConnection(String connectionName, boolean includeWrite)
{
details.get().clear();
details.clear();
Report report = new Report();
ConnectionRuntime connection = ObjectStorageRuntime.connection(connectionName);
StorageConnectionConfig config = connection.getConfig();
@@ -248,7 +230,7 @@ public final class CapabilityProbe
List<String> notes = new ArrayList<>();
report.notes.put(bucket.getAlias(), notes);
readProbes(connection, bucket, results, report, null);
readProbes(connection, bucket, results, report);
recordVersioningState(bucket);
if(includeWrite)
@@ -269,7 +251,7 @@ public final class CapabilityProbe
}
for(Map.Entry<Capability, CapabilityStatus> e : results.entrySet())
{
CapabilityStore.record(bucket, e.getKey(), e.getValue(), details.get().getOrDefault(bucket.getAlias() + ":" + e.getKey(), null));
CapabilityStore.record(bucket, e.getKey(), e.getValue(), details.getOrDefault(bucket.getAlias() + ":" + e.getKey(), null));
}
}
@@ -280,7 +262,7 @@ public final class CapabilityProbe
report.byBucket.get(writable.getAlias()).putAll(presign);
for(Map.Entry<Capability, CapabilityStatus> e : presign.entrySet())
{
CapabilityStore.record(writable, e.getKey(), e.getValue(), details.get().get(writable.getAlias() + ":" + e.getKey()));
CapabilityStore.record(writable, e.getKey(), e.getValue(), details.get(writable.getAlias() + ":" + e.getKey()));
}
}
@@ -299,116 +281,7 @@ public final class CapabilityProbe
return (report);
}
private static final ThreadLocal<Map<String, String>> details = ThreadLocal.withInitial(java.util.HashMap::new);
/*******************************************************************************
** Reuse the read probes within a health deadline, without storing diagnostics
** or running write/presign probes. Only the returned report is consumed.
*******************************************************************************/
public static Report probeConnection(String connectionName, Instant deadline)
{
try
{
return (probeReadOnly(connectionName, deadline));
}
finally
{
details.remove();
}
}
/*******************************************************************************
** Run the shared read-only calls without updating capability or bucket rows.
*******************************************************************************/
private static Report probeReadOnly(String connectionName, Instant deadline)
{
Report report = new Report();
ConnectionRuntime connection = ObjectStorageRuntime.connection(connectionName);
for(StorageBucketConfig bucket : ObjectStorageRuntime.getConfig().getBuckets())
{
if(!connectionName.equals(bucket.getConnection()))
{
continue;
}
if(!Instant.now().isBefore(deadline) || Thread.currentThread().isInterrupted())
{
report.problemKind = "timeout";
break;
}
Map<Capability, CapabilityStatus> results = new EnumMap<>(Capability.class);
report.byBucket.put(bucket.getAlias(), results);
readProbes(connection, bucket, results, report, deadline);
}
if(!Instant.now().isBefore(deadline))
{
report.problemKind = "timeout";
}
return (report);
}
/*******************************************************************************
** Classify failures using types and known protocol codes, never message text.
*******************************************************************************/
public static String problemKind(Throwable error)
{
for(Throwable cause = error; cause != null; cause = cause.getCause())
{
if(cause instanceof UnknownHostException)
{
return ("dns");
}
if(cause instanceof SSLException)
{
return ("tls");
}
String code = ErrorMapper.errorCode(cause);
if("RequestTimeTooSkewed".equals(code) || "RequestExpired".equals(code))
{
return ("clock");
}
if(cause instanceof S3Exception s3 && (s3.statusCode() == 401 || s3.statusCode() == 403
|| "InvalidAccessKeyId".equals(code) || "SignatureDoesNotMatch".equals(code)))
{
return ("auth");
}
}
return ("connection");
}
/*******************************************************************************
** Bound each SDK call (including retries) by the remaining probe budget.
*******************************************************************************/
private static S3Client boundedClient(S3Client client, Instant deadline)
{
return (new DelegatingS3Client(client)
{
/*******************************************************************************
** Preserve credentials and other overrides while limiting the request.
*******************************************************************************/
@Override
@SuppressWarnings("unchecked") // toBuilder retains the concrete request type.
protected <T extends S3Request, ReturnT> ReturnT invokeOperation(T request, Function<T, ReturnT> operation)
{
Duration remaining = Duration.between(Instant.now(), deadline);
if(remaining.isNegative() || remaining.isZero() || Thread.currentThread().isInterrupted())
{
throw (new IllegalStateException("Storage probe deadline exceeded"));
}
AwsRequestOverrideConfiguration.Builder overrides = request.overrideConfiguration()
.map(AwsRequestOverrideConfiguration::toBuilder).orElseGet(AwsRequestOverrideConfiguration::builder);
T bounded = (T) request.toBuilder().overrideConfiguration(overrides.apiCallTimeout(remaining).apiCallAttemptTimeout(remaining).build()).build();
return (operation.apply(bounded));
}
});
}
private static final Map<String, String> details = new java.util.concurrent.ConcurrentHashMap<>();
@@ -418,7 +291,7 @@ public final class CapabilityProbe
*******************************************************************************/
private static void recordVersioningState(StorageBucketConfig bucket)
{
String seen = details.get().get(bucket.getAlias() + ":" + Capability.VERSIONING);
String seen = details.get(bucket.getAlias() + ":" + Capability.VERSIONING);
if(seen == null || !Db.available(ObjectStorageTables.BUCKET))
{
return;
@@ -444,14 +317,14 @@ public final class CapabilityProbe
/*******************************************************************************
** The read-only probes of one bucket.
*******************************************************************************/
private static void readProbes(ConnectionRuntime connection, StorageBucketConfig bucket, Map<Capability, CapabilityStatus> results, Report report, Instant deadline)
private static void readProbes(ConnectionRuntime connection, StorageBucketConfig bucket, Map<Capability, CapabilityStatus> results, Report report)
{
S3Client client = deadline == null ? connection.client() : boundedClient(connection.client(), deadline);
S3Client client = connection.client();
String name = name(bucket);
String alias = bucket.getAlias();
String sampleKey = null;
try(RateLimiter.Permit ignored = deadline == null ? connection.getRateLimiter().acquire() : connection.getRateLimiter().acquire(deadline))
try(RateLimiter.Permit ignored = connection.getRateLimiter().acquire())
{
ListObjectsV2Response r = client.listObjectsV2(ListObjectsV2Request.builder().bucket(name).maxKeys(1).build());
results.put(Capability.LIST_V2, CapabilityStatus.SUPPORTED);
@@ -479,7 +352,6 @@ public final class CapabilityProbe
{
results.put(Capability.LIST_V2, classify(alias, Capability.LIST_V2, e));
report.problems.add(alias + ": listing failed (" + shortMessage(e) + ")");
report.problemKind = problemKind(e);
}
probe(results, alias, Capability.VERSIONING, () ->
@@ -487,10 +359,10 @@ public final class CapabilityProbe
String status = client.getBucketVersioning(b -> b.bucket(name)).statusAsString();
if(status == null && ProviderDefaults.defaultStatus(connection.getConfig().getProviderType(), Capability.VERSIONING) == CapabilityStatus.UNSUPPORTED)
{
details.get().put(alias + ":" + Capability.VERSIONING, "the provider reports versioning off and documents it as a stub");
details.put(alias + ":" + Capability.VERSIONING, "the provider reports versioning off and documents it as a stub");
return (CapabilityStatus.UNSUPPORTED);
}
details.get().put(alias + ":" + Capability.VERSIONING, status == null ? "not enabled" : status);
details.put(alias + ":" + Capability.VERSIONING, status == null ? "not enabled" : status);
return (CapabilityStatus.SUPPORTED);
});
probe(results, alias, Capability.OBJECT_LOCK, () ->
@@ -504,7 +376,7 @@ public final class CapabilityProbe
{
if("ObjectLockConfigurationNotFoundError".equals(ErrorMapper.errorCode(e)))
{
details.get().put(alias + ":" + Capability.OBJECT_LOCK, "object lock is not enabled on this bucket");
details.put(alias + ":" + Capability.OBJECT_LOCK, "object lock is not enabled on this bucket");
return (CapabilityStatus.UNSUPPORTED);
}
throw (e);
@@ -519,7 +391,7 @@ public final class CapabilityProbe
if(key == null)
{
results.put(Capability.OBJECT_TAGGING, CapabilityStatus.UNKNOWN);
details.get().put(alias + ":" + Capability.OBJECT_TAGGING, "no object in the bucket to test");
details.put(alias + ":" + Capability.OBJECT_TAGGING, "no object in the bucket to test");
}
else
{
@@ -633,7 +505,7 @@ public final class CapabilityProbe
try
{
client.putObject(PutObjectRequest.builder().bucket(name).key(key).ifNoneMatch("*").build(), RequestBody.fromString("2"));
details.get().put(alias + ":" + Capability.CONDITIONAL_WRITE, "If-None-Match was ignored");
details.put(alias + ":" + Capability.CONDITIONAL_WRITE, "If-None-Match was ignored");
return (CapabilityStatus.UNSUPPORTED);
}
catch(S3Exception e)
@@ -692,7 +564,7 @@ public final class CapabilityProbe
report.presignHostReachable = true;
if(r.statusCode() == 200 && !"probe".equals(r.body()))
{
details.get().put(alias + ":" + Capability.PRESIGNED_GET, "the presigned GET returned unexpected content (does it reach the same store?)");
details.put(alias + ":" + Capability.PRESIGNED_GET, "the presigned GET returned unexpected content (does it reach the same store?)");
return (CapabilityStatus.UNKNOWN);
}
return (httpStatus(alias, Capability.PRESIGNED_GET, r.statusCode(), r.body()));
@@ -746,7 +618,7 @@ public final class CapabilityProbe
{
return (CapabilityStatus.SUPPORTED);
}
details.get().put(alias + ":" + capability, "HTTP " + status + (body == null ? "" : " " + body.replaceAll("\\s+", " ").substring(0, Math.min(200, body.length()))));
details.put(alias + ":" + capability, "HTTP " + status + (body == null ? "" : " " + body.replaceAll("\\s+", " ").substring(0, Math.min(200, body.length()))));
if(status == 403)
{
return (CapabilityStatus.NOT_PERMITTED);
@@ -797,7 +669,7 @@ public final class CapabilityProbe
static CapabilityStatus classify(String alias, Capability capability, Exception e)
{
String detail = shortMessage(e);
details.get().put(alias + ":" + capability, detail);
details.put(alias + ":" + capability, detail);
if(e instanceof S3Exception s3)
{
String code = ErrorMapper.errorCode(s3);
@@ -30,7 +30,6 @@ import com.kingsrook.qbits.objectstorage.api.Capability;
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -48,11 +47,7 @@ public final class CapabilityStore
public static final Set<Capability> CONNECTION_SCOPE = EnumSet.of(Capability.LIST_V2, Capability.PRESIGNED_GET, Capability.PRESIGNED_PUT,
Capability.PRESIGNED_POST, Capability.USAGE_ADMIN_API, Capability.QUOTA_ADMIN_API);
/** Cluster cache name of the capability statuses (a probe result recorded on one node clears that key on every node). */
public static final String CACHE_NAME = "qbit-object-storage.capabilities";
private static final Map<String, CapabilityStatus> CACHE = ClusterCaches.shared(CACHE_NAME, new ConcurrentHashMap<>(),
CapabilityStore::clearCache, CapabilityStore::clearKey);
private static final Map<String, CapabilityStatus> CACHE = new ConcurrentHashMap<>();
private static final Set<String> LOADED = ConcurrentHashMap.newKeySet();
@@ -67,7 +62,7 @@ public final class CapabilityStore
/*******************************************************************************
** Drop everything cached on this node.
** Drop everything cached.
*******************************************************************************/
public static void clearCache()
{
@@ -77,28 +72,6 @@ public final class CapabilityStore
/*******************************************************************************
** Drop one cached status on this node; the next read reloads the missing
** statuses from the capability rows (cached ones are kept).
*******************************************************************************/
static void clearKey(String key)
{
CACHE.remove(key);
LOADED.clear();
}
/*******************************************************************************
** Whether a status is cached on this node (tests).
*******************************************************************************/
static boolean isCached(StorageBucketConfig bucket, Capability capability)
{
return (CACHE.containsKey(scopeKey(bucket, capability)));
}
/*******************************************************************************
** The status of a capability for a bucket alias.
*******************************************************************************/
@@ -139,24 +112,10 @@ public final class CapabilityStore
/*******************************************************************************
** Record one probe result (cache and, when the tables are there, the row).
** The row is written first, then the key is cleared on every node (they
** reload it from the row), then cached here.
*******************************************************************************/
public static void record(StorageBucketConfig bucket, Capability capability, CapabilityStatus status, String detail)
{
String key = scopeKey(bucket, capability);
persist(bucket, capability, status, detail);
ClusterCaches.invalidateKey(CACHE_NAME, key);
CACHE.put(key, status);
}
/*******************************************************************************
** Write one probe result to its capability row; never throws.
*******************************************************************************/
private static void persist(StorageBucketConfig bucket, Capability capability, CapabilityStatus status, String detail)
{
CACHE.put(scopeKey(bucket, capability), status);
if(!Db.available(ObjectStorageTables.BUCKET_CAPABILITY))
{
return;
@@ -1,95 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage.internal;
import java.util.List;
import java.util.Optional;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Post insert, update and delete of the editable config rows that the runtime
** caches: a bucket row (presign limits, ConfigSync) or a header rule
** (DefaultObjectHeaderPolicy). Clears that cache on every node, so an admin's
** edit takes effect everywhere at once; the short TTLs remain the safety net.
*******************************************************************************/
public class ConfigRowChangeCustomizer implements TableCustomizerInterface
{
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public List<QRecord> postInsert(InsertInput insertInput, List<QRecord> records) throws QException
{
invalidate(insertInput.getTableName());
return (records);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public List<QRecord> postUpdate(UpdateInput updateInput, List<QRecord> records, Optional<List<QRecord>> oldRecordList) throws QException
{
invalidate(updateInput.getTableName());
return (records);
}
/*******************************************************************************
** {@inheritDoc}
*******************************************************************************/
@Override
public List<QRecord> postDelete(DeleteInput deleteInput, List<QRecord> records) throws QException
{
invalidate(deleteInput.getTableName());
return (records);
}
/*******************************************************************************
** Clear the cache that holds the table's rows, on every node.
*******************************************************************************/
static void invalidate(String tableName)
{
if(ObjectStorageTables.BUCKET.equals(tableName))
{
ClusterCaches.invalidate(ConfigSync.ROW_LIMITS_CACHE_NAME);
}
else if(ObjectStorageTables.HEADER_RULE.equals(tableName))
{
ClusterCaches.invalidate(DefaultObjectHeaderPolicy.CACHE_NAME);
}
}
}
@@ -30,7 +30,6 @@ import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
@@ -49,16 +48,10 @@ public final class ConfigSync
{
private static final QLogger LOG = QLogger.getLogger(ConfigSync.class);
/** Cluster cache name of the bucket rows' presign limits (cleared on every node when a bucket row changes). */
public static final String ROW_LIMITS_CACHE_NAME = "qbit-object-storage.bucket-row-limits";
private static final Map<String, Integer> BUCKET_IDS = new ConcurrentHashMap<>();
private static final Map<String, Integer> CONNECTION_IDS = new ConcurrentHashMap<>();
private static final Map<String, Integer> BUCKET_IDS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"bucket alias to row id, written by the config sync; bucket rows cannot be inserted or deleted through the UI or API, so the ids are the same on every node");
private static final Map<String, Integer> CONNECTION_IDS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"connection name to row id, written by the config sync; connection rows cannot be inserted or deleted through the UI or API, so the ids are the same on every node");
private static final Map<String, RowLimits> ROW_LIMITS = ClusterCaches.shared(ROW_LIMITS_CACHE_NAME, new ConcurrentHashMap<>(),
ConfigSync::clearRowLimits, alias -> ConfigSync.ROW_LIMITS.remove(alias));
private static final Map<String, RowLimits> ROW_LIMITS = new ConcurrentHashMap<>();
private static final long ROW_LIMIT_CACHE_MILLIS = 15_000L;
private static volatile boolean synced = false;
@@ -320,8 +313,7 @@ public final class ConfigSync
/*******************************************************************************
** Forget the cached row limits on this node (a bucket row edit clears them on
** every node through ConfigRowChangeCustomizer; they also expire).
** Forget the cached row limits (after a row edit in a test; they also expire).
*******************************************************************************/
public static void clearRowLimits()
{
@@ -22,6 +22,7 @@ package com.kingsrook.qbits.objectstorage.internal;
import java.net.URI;
import java.util.function.Function;
import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
import com.kingsrook.qbits.objectstorage.config.CredentialsMode;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
@@ -29,10 +30,17 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMetaData;
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
import com.kingsrook.qqq.backend.module.filesystem.s3.utils.S3ClientFactory;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.awscore.AwsRequestOverrideConfiguration;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.DelegatingS3Client;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
import software.amazon.awssdk.services.s3.model.S3Request;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
@@ -190,7 +198,16 @@ public class ConnectionRuntime
*******************************************************************************/
private S3Client buildClient()
{
return (S3ClientFactory.buildClient(backendMetaData()));
S3BackendMetaData backend = backendMetaData();
String token = config.getCredentialsMode() == CredentialsMode.STATIC_REFS ? SecretRefs.resolve(config.getSessionTokenRef()) : null;
AwsCredentialsProvider sessionCredentials = null;
if(token != null && backend.getAccessKey() != null && backend.getSecretKey() != null)
{
sessionCredentials = StaticCredentialsProvider.create(AwsSessionCredentials.create(backend.getAccessKey(), backend.getSecretKey(), token));
}
S3Client coreClient = S3ClientFactory.buildClient(backend);
return (sessionCredentials == null ? coreClient : new SessionCredentialsClient(coreClient, sessionCredentials));
}
@@ -212,7 +229,8 @@ public class ConnectionRuntime
.withSocketTimeoutMillis(config.getReadTimeoutMs());
if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
{
withStaticCredentials(backend, config.getAccessKeyRef(), config.getSecretKeyRef());
backend.withAccessKey(SecretRefs.resolve(config.getAccessKeyRef()))
.withSecretKey(SecretRefs.resolve(config.getSecretKeyRef()));
}
return (backend);
}
@@ -252,33 +270,62 @@ public class ConnectionRuntime
/*******************************************************************************
** Static keys (with the session token, when set) from the references, or the
** SDK default chain; core builds the provider.
** Static keys from the references, or the SDK default chain.
*******************************************************************************/
private AwsCredentialsProvider credentials(String accessRef, String secretRef)
{
S3BackendMetaData credentials = new S3BackendMetaData();
if(config.getCredentialsMode() == CredentialsMode.STATIC_REFS)
{
withStaticCredentials(credentials, accessRef, secretRef);
String accessKey = SecretRefs.resolve(accessRef);
String secretKey = SecretRefs.resolve(secretRef);
if(accessKey != null && secretKey != null)
{
String token = SecretRefs.resolve(config.getSessionTokenRef());
return (StaticCredentialsProvider.create(token == null ? AwsBasicCredentials.create(accessKey, secretKey)
: AwsSessionCredentials.create(accessKey, secretKey, token)));
}
}
return (S3ClientFactory.buildCredentialsProvider(credentials));
return (DefaultCredentialsProvider.builder().build());
}
/*******************************************************************************
** Resolve static keys onto core meta data. The session token applies only
** with both keys, as before (core rejects a token without keys).
** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.7)
** RC.5 metadata/factory lacks a session token or credentials-provider hook.
** Override request credentials only; core still owns the client and transport.
*******************************************************************************/
private void withStaticCredentials(S3BackendMetaData backend, String accessRef, String secretRef)
private static class SessionCredentialsClient extends DelegatingS3Client
{
String accessKey = SecretRefs.resolve(accessRef);
String secretKey = SecretRefs.resolve(secretRef);
backend.withAccessKey(accessKey).withSecretKey(secretKey);
if(accessKey != null && secretKey != null)
private final AwsCredentialsProvider credentialsProvider;
/*******************************************************************************
** Wrap a core-built client with the connection's session credentials.
*******************************************************************************/
private SessionCredentialsClient(S3Client delegate, AwsCredentialsProvider credentialsProvider)
{
backend.withSessionToken(SecretRefs.resolve(config.getSessionTokenRef()));
super(delegate);
this.credentialsProvider = credentialsProvider;
}
/*******************************************************************************
** Preserve request overrides and the concrete request type while adding the
** session token to signing, including streaming and paginated operations.
*******************************************************************************/
@Override
@SuppressWarnings("unchecked") // toBuilder().build() retains the concrete request type T.
protected <T extends S3Request, ReturnT> ReturnT invokeOperation(T request, Function<T, ReturnT> operation)
{
AwsRequestOverrideConfiguration.Builder overrides = request.overrideConfiguration()
.map(AwsRequestOverrideConfiguration::toBuilder).orElseGet(AwsRequestOverrideConfiguration::builder);
T signedRequest = (T) request.toBuilder()
.overrideConfiguration(overrides.credentialsProvider(credentialsProvider).build())
.build();
return (operation.apply(signedRequest));
}
}
}
@@ -48,21 +48,9 @@ import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
** Small helpers for reading and writing the qbit's own tables as the system
** user (the service writes its audit log and index regardless of the calling
** user's per-bucket scope; the calling user's permissions were checked first).
**
** Every write made here carries core's per-call permission bypass
** (AbstractActionInput.withPermissionBypass). That mark, not the session, is
** what lets the service write tables nobody may write directly: a host that
** enforces permissions inside the core actions asks ObjectStoragePermissionChecker
** about every outermost write that does not carry a bypass, and the checker
** refuses those tables to every session, the system user included.
*******************************************************************************/
public final class Db
{
/** The reason recorded on (and logged by core for) each write the service makes of its own tables. */
public static final String OWN_WRITE_REASON = "The object storage service writes its own tables (operation log, index, capabilities, usage); the caller's storage permission was checked before the storage call.";
/*******************************************************************************
** Something that may throw.
*******************************************************************************/
@@ -249,9 +237,7 @@ public final class Db
*******************************************************************************/
public static List<QRecord> insert(String table, List<QRecord> records) throws QException
{
InsertInput input = new InsertInput(table).withRecords(records);
input.withPermissionBypass(OWN_WRITE_REASON);
return (new InsertAction().execute(input).getRecords());
return (new InsertAction().execute(new InsertInput(table).withRecords(records)).getRecords());
}
@@ -273,9 +259,7 @@ public final class Db
*******************************************************************************/
public static List<QRecord> update(String table, List<QRecord> records) throws QException
{
UpdateInput input = new UpdateInput(table).withRecords(records);
input.withPermissionBypass(OWN_WRITE_REASON);
return (new UpdateAction().execute(input).getRecords());
return (new UpdateAction().execute(new UpdateInput(table).withRecords(records)).getRecords());
}
@@ -287,9 +271,7 @@ public final class Db
{
if(!ids.isEmpty())
{
DeleteInput input = new DeleteInput(table).withPrimaryKeys(ids);
input.withPermissionBypass(OWN_WRITE_REASON);
new DeleteAction().execute(input);
new DeleteAction().execute(new DeleteInput(table).withPrimaryKeys(ids));
}
}
@@ -23,14 +23,12 @@ package com.kingsrook.qbits.objectstorage.internal;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
import com.kingsrook.qbits.objectstorage.api.ObjectHeaderPolicy;
import com.kingsrook.qbits.objectstorage.api.ObjectHeaders;
import com.kingsrook.qbits.objectstorage.config.BucketRole;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
@@ -46,31 +44,19 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
{
private static final QLogger LOG = QLogger.getLogger(DefaultObjectHeaderPolicy.class);
/** Cluster cache name of the enabled header rules (cleared on every node when a rule changes). */
public static final String CACHE_NAME = "qbit-object-storage.header-rules";
private static final long CACHE_MILLIS = 15_000L;
private static final AtomicReference<CachedRules> CACHED = ClusterCaches.shared(CACHE_NAME, new AtomicReference<>(), DefaultObjectHeaderPolicy::clearCache, null);
private static volatile List<QRecord> cachedRules;
private static volatile long cachedAt;
/*******************************************************************************
** The enabled rules and when they were read.
*******************************************************************************/
private record CachedRules(List<QRecord> rules, long readAt)
{
}
/*******************************************************************************
** Drop the cached rules on this node (a rule edit clears them on every node
** through ConfigRowChangeCustomizer).
** Drop cached rules (after the rules change).
*******************************************************************************/
public static void clearCache()
{
CACHED.set(null);
cachedRules = null;
}
@@ -133,12 +119,11 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
*******************************************************************************/
private static List<QRecord> rules()
{
CachedRules cached = CACHED.get();
if(cached != null && System.currentTimeMillis() - cached.readAt() < CACHE_MILLIS)
List<QRecord> rules = cachedRules;
if(rules != null && System.currentTimeMillis() - cachedAt < CACHE_MILLIS)
{
return (cached.rules());
return (rules);
}
List<QRecord> rules;
if(!Db.available(ObjectStorageTables.HEADER_RULE))
{
return (List.of());
@@ -148,7 +133,8 @@ public class DefaultObjectHeaderPolicy implements ObjectHeaderPolicy
rules = Db.asSystem(() -> Db.query(ObjectStorageTables.HEADER_RULE, "enabled", true));
rules.sort(Comparator.comparing((QRecord r) -> r.getValueInteger("priority") == null ? 100 : r.getValueInteger("priority"))
.thenComparing(r -> r.getValueInteger("id") == null ? 0 : r.getValueInteger("id")));
CACHED.set(new CachedRules(rules, System.currentTimeMillis()));
cachedRules = rules;
cachedAt = System.currentTimeMillis();
return (rules);
}
catch(Exception e)
@@ -33,7 +33,6 @@ import com.kingsrook.qbits.objectstorage.api.StorageAlertSink;
import com.kingsrook.qbits.objectstorage.api.StorageChangeListener;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
/*******************************************************************************
@@ -44,11 +43,9 @@ import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
*******************************************************************************/
public final class ObjectStorageRuntime
{
private static final Map<String, ConnectionRuntime> CONNECTIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"this node's S3 clients, built from the code config at first use; row overrides apply when a client is built, the same on every node");
private static final Map<String, ConnectionRuntime> CONNECTIONS = new ConcurrentHashMap<>();
private static final List<StorageChangeListener> LISTENERS = new CopyOnWriteArrayList<>();
private static final Map<String, String> PROCESS_PERMISSIONS = ClusterCaches.nodeLocal(new ConcurrentHashMap<>(),
"built from the qbit config at startup, the same on every node");
private static final Map<String, String> PROCESS_PERMISSIONS = new ConcurrentHashMap<>();
private static volatile ObjectStorageQBitConfig config;
private static volatile ObjectHeaderPolicy headerPolicy;
@@ -99,7 +96,6 @@ public final class ObjectStorageRuntime
clock = Clock.systemUTC();
indexHook = null;
CapabilityStore.clearCache();
OperationLog.reset();
}
@@ -21,17 +21,10 @@
package com.kingsrook.qbits.objectstorage.internal;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import com.kingsrook.qbits.objectstorage.api.StorageCaller;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
@@ -40,22 +33,12 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** every caller (UI action, app code, scheduled job) lands here. Presigned URLs
** are logged without the signature (key, method, expiry and issuer only),
** because the URL itself is a bearer token. A logging failure never fails the
** operation it describes, so it is reported three ways instead: an ERROR log
** line, the storage.operation_log.failed alert (StorageAlertSink, resolved by
** the next row written for that bucket), and failedWriteCount().
** operation it describes.
*******************************************************************************/
public final class OperationLog
{
private static final QLogger LOG = QLogger.getLogger(OperationLog.class);
/** The alert raised (severity error, dedupe key "bucket:" + alias) when a row could not be written. */
public static final String ALERT_TYPE = "storage.operation_log.failed";
private static final AtomicLong FAILED_WRITES = new AtomicLong();
private static final Set<String> UNRESOLVED = ClusterCaches.nodeLocal(ConcurrentHashMap.newKeySet(),
"the buckets whose operation log alert this node raised and has not resolved (alert state, not cached data)");
/*******************************************************************************
@@ -200,31 +183,6 @@ public final class OperationLog
/*******************************************************************************
** How many operation log rows this node could not write since it started (or
** since the runtime was reset). The operations themselves were carried out, so
** anything above zero means storage changes that the audit log does not show.
** A host that has no alert sink reads this (a metrics gauge, a health check of
** its own) to alert on a missing audit row.
*******************************************************************************/
public static long failedWriteCount()
{
return (FAILED_WRITES.get());
}
/*******************************************************************************
** Forget the failures counted and the alerts raised (ObjectStorageRuntime.reset).
*******************************************************************************/
static void reset()
{
FAILED_WRITES.set(0);
UNRESOLVED.clear();
}
/*******************************************************************************
** Insert the row (as the system user), never throwing.
*******************************************************************************/
@@ -238,7 +196,7 @@ public final class OperationLog
try
{
Integer bucketId = ConfigSync.bucketId(e.bucketAlias);
QRecord written = Db.asSystem(() -> Db.insert(ObjectStorageTables.OPERATION, new QRecord()
Db.asSystem(() -> Db.insert(ObjectStorageTables.OPERATION, new QRecord()
.withValue("occurredAt", ObjectStorageRuntime.clock().instant())
.withValue("actor", e.caller.describe())
.withValue("action", e.action)
@@ -252,66 +210,10 @@ public final class OperationLog
.withValue("providerRequestId", e.requestId)
.withValue("processRunId", e.processRunId)
.withValue("detail", e.detail)));
////////////////////////////////////////////////////////////////////////////////
// the insert action does not throw for a record it refused (a missing value, //
// a value too long, a record lock): it hands the record back with its errors //
////////////////////////////////////////////////////////////////////////////////
if(CollectionUtils.nullSafeHasContents(written.getErrors()))
{
throw (new QException("The operation log row was refused: " + written.getErrors()));
}
written(e);
}
catch(Exception ex)
{
failed(e, ex);
}
}
/*******************************************************************************
** A row was written: resolve the alert of an earlier failure for its bucket.
*******************************************************************************/
private static void written(Entry e)
{
String dedupeKey = "bucket:" + e.bucketAlias;
if(UNRESOLVED.remove(dedupeKey))
{
try
{
ObjectStorageRuntime.alertSink().resolve(ALERT_TYPE, dedupeKey);
}
catch(RuntimeException sinkFailure)
{
LOG.warn("Could not resolve the storage operation log alert", sinkFailure, logPair("bucket", e.bucketAlias));
}
}
}
/*******************************************************************************
** A row could not be written: the operation happened and the audit log does
** not show it. Say so where an operator sees it, without failing the operation.
*******************************************************************************/
private static void failed(Entry e, Exception cause)
{
long count = FAILED_WRITES.incrementAndGet();
LOG.error("Could not write a storage operation log row: the operation is missing from the audit log", cause,
logPair("action", e.action), logPair("bucket", e.bucketAlias), logPair("key", e.key), logPair("result", e.result), logPair("failedWriteCount", count));
String dedupeKey = "bucket:" + e.bucketAlias;
UNRESOLVED.add(dedupeKey);
try
{
ObjectStorageRuntime.alertSink().raise(ALERT_TYPE, dedupeKey, "error", "A storage operation on " + e.bucketAlias + " was not recorded in the operation log",
Map.of("bucket", String.valueOf(e.bucketAlias), "action", String.valueOf(e.action), "count", count));
}
catch(RuntimeException sinkFailure)
{
LOG.warn("Could not raise the storage operation log alert", sinkFailure, logPair("bucket", e.bucketAlias));
LOG.warn("Could not write a storage operation log row", ex, logPair("action", e.action), logPair("key", e.key));
}
}
}
@@ -26,7 +26,6 @@ import java.util.Map;
import com.kingsrook.qbits.objectstorage.api.Capability;
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
import com.kingsrook.qbits.objectstorage.config.ProviderType;
import com.kingsrook.qqq.backend.core.cluster.ClusterCaches;
import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.PARTIAL;
import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.SUPPORTED;
import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.UNKNOWN;
@@ -40,8 +39,7 @@ import static com.kingsrook.qbits.objectstorage.api.CapabilityStatus.UNSUPPORTED
*******************************************************************************/
public final class ProviderDefaults
{
private static final Map<ProviderType, Map<Capability, CapabilityStatus>> TABLE = ClusterCaches.nodeLocal(new EnumMap<>(ProviderType.class),
"provider defaults from code, the same on every node");
private static final Map<ProviderType, Map<Capability, CapabilityStatus>> TABLE = new EnumMap<>(ProviderType.class);
static
{
@@ -21,10 +21,7 @@
package com.kingsrook.qbits.objectstorage.internal;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
/*******************************************************************************
@@ -60,52 +57,17 @@ public class RateLimiter
*******************************************************************************/
public Permit acquire()
{
return (acquire(null));
}
/*******************************************************************************
** Wait no later than a caller's deadline, releasing the slot on interruption.
*******************************************************************************/
public Permit acquire(Instant deadline)
{
boolean acquired = false;
try
{
if(deadline == null)
{
concurrency.acquire();
acquired = true;
}
else
{
acquired = concurrency.tryAcquire(Math.max(0, Duration.between(Instant.now(), deadline).toNanos()), TimeUnit.NANOSECONDS);
if(!acquired)
{
throw (new IllegalStateException("Storage probe deadline exceeded"));
}
}
waitForToken(deadline);
concurrency.acquire();
waitForToken();
return (new Permit(concurrency));
}
catch(InterruptedException e)
{
if(acquired)
{
concurrency.release();
}
Thread.currentThread().interrupt();
throw (new IllegalStateException("Interrupted while waiting for the storage rate limiter", e));
}
catch(RuntimeException e)
{
if(acquired)
{
concurrency.release();
}
throw (e);
}
}
@@ -113,14 +75,10 @@ public class RateLimiter
/*******************************************************************************
** Block until a token is available.
*******************************************************************************/
private void waitForToken(Instant deadline) throws InterruptedException
private void waitForToken() throws InterruptedException
{
while(true)
{
if(deadline != null && !Instant.now().isBefore(deadline))
{
throw (new IllegalStateException("Storage probe deadline exceeded"));
}
long sleepMillis;
synchronized(this)
{
@@ -135,7 +93,7 @@ public class RateLimiter
throttledWaits++;
sleepMillis = Math.max(1, (long) ((1.0 - tokens) / opsPerSecond * 1000));
}
Thread.sleep(deadline == null ? sleepMillis : Math.min(sleepMillis, Math.max(1, Duration.between(Instant.now(), deadline).toMillis())));
Thread.sleep(sleepMillis);
}
}
@@ -21,6 +21,10 @@
package com.kingsrook.qbits.objectstorage.internal;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.regex.Pattern;
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
@@ -28,12 +32,13 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** Enforces the qbit's secret-reference allow-list and resolves references
** (${env.X}, ${prop.X}, ${file.PATH}) through core QMetaDataVariableInterpreter.
** Core reads ${file.PATH} read-only from its allowed roots (default
** /var/run/secrets and /run/secrets; qqq.fileReference.roots or
** QQQ_FILE_REFERENCE_ROOTS). Resolved values live only in memory, never in a
** record, a log line or a process value.
** Resolves secret references (${env.X}, ${prop.X}, ${file.PATH}) and enforces
** the allow-list. Resolved values live only in memory, never in a record, a log
** line or a process value.
**
** STAND-IN(core=QRun-IO/qqq#1165, removeBy=4.1.0-RC.7)
** ${file.PATH} moves to QMetaDataVariableInterpreter in QQQ core
** (spec section 8.2); then this class delegates entirely.
*******************************************************************************/
public final class SecretRefs
{
@@ -87,7 +92,19 @@ public final class SecretRefs
{
return (null);
}
String value = new QMetaDataVariableInterpreter().interpret(ref.trim());
String trimmed = ref.trim();
if(trimmed.startsWith("${file.") && trimmed.endsWith("}"))
{
try
{
return (Files.readString(Path.of(trimmed.substring("${file.".length(), trimmed.length() - 1)), StandardCharsets.UTF_8).trim());
}
catch(IOException e)
{
return (null);
}
}
String value = new QMetaDataVariableInterpreter().interpret(trimmed);
return (value == null || value.startsWith("${") ? null : value);
}
@@ -40,8 +40,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
** Applies help/qbit-object-storage.yaml (the QQQ help-bundle format) to the
** Storage app and its object_storage_* tables: what each table is for and how
** its rows get created, shown on the table screen and when it is empty.
** Inline defaults preserve empty-state help for hosts without helpContent.
** The producer also registers the bundle for core sync and HOST overrides.
** STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.8): replace with
** HelpBundles.register once the pinned QQQ release includes it; the bundle file
** stays the same.
*******************************************************************************/
public final class ObjectStorageHelp
{
@@ -80,7 +81,7 @@ public final class ObjectStorageHelp
{
String key = entry.path("key").asText();
String summary = entry.path("summary").asText();
if(key.isBlank() || summary.isBlank() || summary.length() > 120 || entries.put(key, entry) != null)
if(key.isBlank() || summary.isBlank() || summary.length() > 120 || entry.path("content").asText().isBlank() || entries.put(key, entry) != null)
{
throw (new IOException("invalid or duplicate entry: " + key));
}
@@ -25,7 +25,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.internal.ConfigRowChangeCustomizer;
import com.kingsrook.qbits.objectstorage.permissions.ObjectStoragePermissionChecker;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
@@ -565,9 +564,7 @@ public final class ObjectStorageTables
.withField(ro("orphanedCount", QFieldType.LONG))
.withField(ro("orphanedBytes", QFieldType.LONG))
.withField(ro("missingCount", QFieldType.LONG))
.withField(ro("lastReferenceScanDate", QFieldType.DATE_TIME)), KEY_TYPE_BUCKET, "id", false)
.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class)));
.withField(ro("lastReferenceScanDate", QFieldType.DATE_TIME)), KEY_TYPE_BUCKET, "id", false));
}
@@ -704,10 +701,7 @@ public final class ObjectStorageTables
.withField(str("cacheControl", 300))
.withField(str("contentDisposition", 300))
.withField(new QFieldMetaData("priority", QFieldType.INTEGER).withDefaultValue(100))
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)), KEY_TYPE_BUCKET, "bucketId", true)
.withCustomizer(TableCustomizers.POST_INSERT_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
.withCustomizer(TableCustomizers.POST_UPDATE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class))
.withCustomizer(TableCustomizers.POST_DELETE_RECORD, new QCodeReference(ConfigRowChangeCustomizer.class)));
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)), KEY_TYPE_BUCKET, "bucketId", true));
}
@@ -46,9 +46,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSession;
** - writes to the editable config tables (connection operational fields, header
** rules, protection rules, bucket thresholds) need object_storage.admin;
** observed and history tables are never written through the UI or API (they
** change only through actions and the service, whose own writes carry
** core's per-call permission bypass, see internal.Db; no session is let
** through, the core system user included);
** change only through actions and the service);
** - each process needs the permission registered for it (default admin).
*******************************************************************************/
public class ObjectStoragePermissionChecker implements CustomPermissionChecker
@@ -67,12 +65,6 @@ public class ObjectStoragePermissionChecker implements CustomPermissionChecker
String needed = required(actionInput, metaData);
if(needed == null)
{
////////////////////////////////////////////////////////////////////////////////////////
// nobody writes these tables directly, whatever the session: a person, the storage //
// administrator and the core system user are all refused. the service's own writes //
// (internal.Db) carry core's per-call permission bypass, so core does not ask about //
// them; a write that reaches this point did not come from the service //
////////////////////////////////////////////////////////////////////////////////////////
throw (new QPermissionDeniedException("This storage table changes only through storage actions."));
}
if(!has(needed))
@@ -227,7 +227,7 @@ public final class TestConnectionProcess
}
if(report.getClockSkewSeconds() != null)
{
boolean skewed = report.getClockSkewSeconds() > CapabilityProbe.CLOCK_SKEW_THRESHOLD_SECONDS;
boolean skewed = report.getClockSkewSeconds() > 900;
ok = ok && !skewed;
lines.add(new ProcessSummaryLine(skewed ? Status.ERROR : Status.INFO, "Clock skew against the store: " + report.getClockSkewSeconds() + " seconds"
+ (skewed ? " (SigV4 fails beyond 15 minutes)" : "")));
File diff suppressed because it is too large. Load diff
@@ -1 +0,0 @@
version=${project.version}
@@ -1,82 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage;
import java.util.Set;
import java.util.stream.Collectors;
import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
import com.kingsrook.qqq.backend.core.testing.HelpBundleAssertions;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Coverage of every object owned by the storage qbit, including its optional app.
*******************************************************************************/
class ObjectStorageHelpBundleTest
{
/*******************************************************************************
** Release the installed runtime after each test.
*******************************************************************************/
@AfterEach
void tearDown()
{
StorageTestSupport.tearDown();
}
/*******************************************************************************
** Register safely without help tables and cover all owned metadata.
*******************************************************************************/
@Test
void bundleIsCompleteWithoutHelpTables() throws Exception
{
QInstance instance = StorageTestSupport.setUp(new ObjectStorageQBitConfig()
.withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false).withOpenWhenAnonymous(true).withAdminApp(true));
assertThat(instance.getTable("helpContent")).isNull();
assertThat(HelpBundles.of(instance).getBundles()).singleElement()
.satisfies(bundle ->
{
assertThat(bundle.getId()).isEqualTo("qbit-object-storage");
assertThat(System.getProperty("qbit.artifactVersion")).isNotBlank();
assertThat(bundle.getVersion()).isEqualTo(System.getProperty("qbit.artifactVersion"))
.isEqualTo(ObjectStorageQBitProducer.VERSION);
});
HelpBundles.syncOnStartup(instance);
Set<String> keys = HelpBundles.of(instance).getBundles().get(0).getEntries().stream()
.map(entry -> entry.getKey()).collect(Collectors.toSet());
// RC.7 validates these entries but only requires app/table/process/field coverage.
instance.getWidgets().keySet().stream().filter(name -> name.startsWith("object_storage_") || name.startsWith("objectStorage"))
.forEach(name -> assertThat(keys).contains("widget:" + name + ";slot:label"));
instance.getPossibleValueSources().keySet().stream().filter(name -> name.startsWith("object_storage_"))
.forEach(name -> assertThat(keys).contains("possibleValueSource:" + name + ";slot:help"));
HelpBundleAssertions.forInstance(instance)
.bundles("help/qbit-object-storage.yaml")
.owning(target -> target.name().startsWith("object_storage_") || target.name().startsWith("objectStorage"))
.assertComplete();
}
}
@@ -44,10 +44,6 @@ class ObjectStorageQBitProducerTest
ObjectStorageQBitConfig config = new ObjectStorageQBitConfig().withBackendName("rdbms");
ObjectStorageQBitProducer producer = new ObjectStorageQBitProducer().withQBitConfig(config).withNamespace("object_storage");
String artifactVersion = System.getProperty("qbit.artifactVersion");
assertThat(artifactVersion).isNotBlank();
assertThat(ObjectStorageQBitProducer.VERSION).isEqualTo(artifactVersion);
assertThat(producer.getQBitMetaData().getVersion()).isEqualTo(artifactVersion);
assertThat(producer.isEnabled()).isTrue();
assertThat(producer.getQBitConfig()).isSameAs(config);
assertThat(producer.getNamespace()).isEqualTo("object_storage");
@@ -1,364 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage.health;
import javax.net.ssl.SSLHandshakeException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
import com.kingsrook.qbits.objectstorage.internal.CapabilityProbe;
import com.kingsrook.qbits.objectstorage.internal.ConfigSync;
import com.kingsrook.qbits.objectstorage.internal.ConnectionRuntime;
import com.kingsrook.qbits.objectstorage.internal.Db;
import com.kingsrook.qbits.objectstorage.internal.ObjectStorageRuntime;
import com.kingsrook.qbits.objectstorage.internal.RateLimiter;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qbits.objectstorage.processes.TestConnectionProcess;
import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
import com.kingsrook.qqq.backend.core.health.HealthCategory;
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
import com.kingsrook.qqq.backend.core.health.HealthStatus;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
import software.amazon.awssdk.http.SdkHttpResponse;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.S3Exception;
import software.amazon.awssdk.services.s3.model.S3Request;
import software.amazon.awssdk.utils.builder.SdkBuilder;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Exercise the real probe using an in-memory S3 client and QQQ record store.
*******************************************************************************/
class ObjectStorageConnectionChecksTest
{
private static final String SECRET = "S3NT1NEL-secret";
private QInstance instance;
private ObjectStorageQBitConfig config;
private final List<String> calls = new ArrayList<>();
private RuntimeException failure;
private long skew;
private boolean expireAfterList;
private Instant deadline;
/*******************************************************************************
** Poison every credential reference; the fake prevents all network access.
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
System.setProperty("os.health.secret", SECRET);
String ref = "${prop.os.health.secret}";
config = new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY)
.withAllowedSecretRefPatterns(List.of("prop.os.health.*"))
.withConnection(new StorageConnectionConfig("PRIMARY").withEndpoint("https://user:" + SECRET + "@private.example")
.withAccessKeyRef(ref).withSecretKeyRef(ref).withSessionTokenRef(ref)
.withPresignAccessKeyRef(ref).withPresignSecretKeyRef(ref).withCaBundleRef(ref).withMaxConcurrency(1))
.withBucket(new StorageBucketConfig("files").withConnection("PRIMARY").withBucketName("private-bucket").withIndexed(false));
instance = StorageTestSupport.setUp(config);
S3Client client = (S3Client) Proxy.newProxyInstance(S3Client.class.getClassLoader(), new Class<?>[] { S3Client.class }, (proxy, method, args) ->
{
if(method.getName().equals("close"))
{
return (null);
}
if(args != null && args.length == 1 && args[0] instanceof java.util.function.Consumer<?>)
{
return (InvocationHandler.invokeDefault(proxy, method, args));
}
calls.add(method.getName());
assertThat(method.getName()).matches("listObjectsV2|getBucket.*|getObjectLockConfiguration|getObjectTagging");
S3Request request = (S3Request) args[0];
Duration timeout = request.overrideConfiguration().orElseThrow().apiCallTimeout().orElseThrow();
assertThat(timeout).isPositive().isLessThanOrEqualTo(Duration.ofSeconds(20));
assertThat(request.overrideConfiguration().orElseThrow().apiCallAttemptTimeout()).contains(timeout);
if(failure != null)
{
throw (failure);
}
if(method.getName().equals("listObjectsV2"))
{
if(expireAfterList)
{
Thread.sleep(Math.max(1, Duration.between(Instant.now(), deadline).toMillis() + 10));
}
return (ListObjectsV2Response.builder().sdkHttpResponse(SdkHttpResponse.builder().statusCode(200)
.putHeader("Date", DateTimeFormatter.RFC_1123_DATE_TIME.format(Instant.now().minusSeconds(skew).atZone(ZoneOffset.UTC))).build()).build());
}
return (((SdkBuilder<?, ?>) method.getReturnType().getMethod("builder").invoke(null)).build());
});
Field field = ConnectionRuntime.class.getDeclaredField("client");
field.setAccessible(true);
field.set(ObjectStorageRuntime.connection("PRIMARY"), client);
}
/*******************************************************************************
** Restore process-wide fixtures and the poisoned property.
*******************************************************************************/
@AfterEach
void tearDown()
{
StorageTestSupport.tearDown();
System.clearProperty("os.health.secret");
}
/*******************************************************************************
** Producer registration includes the required metadata and keeps both actions.
*******************************************************************************/
@Test
void registrationAndRefresh() throws Exception
{
var registry = HealthChecksMetaData.ofOrWithNew(instance);
assertThat(registry.getSuppliers()).singleElement().satisfies(supplier ->
{
assertThat(supplier.getNamePrefix()).isEqualTo("object_storage.connection");
assertThat(supplier.getComponent()).isEqualTo("object_storage");
});
HealthCheckMetaData check = check();
assertThat(check.getName()).isEqualTo("object_storage.connection." + ConfigSync.connectionId("PRIMARY")).matches("[a-z0-9]+([._-][a-z0-9]+)*");
assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
assertThat(check.getCategory()).isEqualTo(HealthCategory.STORAGE);
assertThat(check.getComponent()).isEqualTo("object_storage");
assertThat(check.getIntervalSeconds()).isEqualTo(300);
assertThat(check.getTimeoutMs()).isEqualTo(20000);
assertThat(instance.getProcess(TestConnectionProcess.READ_NAME)).isNotNull();
assertThat(instance.getProcess(TestConnectionProcess.WRITE_NAME)).isNotNull();
ConfigSync.sync();
assertThat(registry.resolveAll(instance)).hasSize(1);
config.withConnection(new StorageConnectionConfig("SECOND"));
ConfigSync.sync();
assertThat(registry.resolveAll(instance)).hasSize(2);
config.getConnections().clear();
assertThat(registry.resolveAll(instance)).isEmpty();
assertClean(check.getCheckInstance().run(input()), HealthStatus.UNKNOWN);
}
/*******************************************************************************
** Successful checks read buckets without persisting any capability diagnostics.
*******************************************************************************/
@Test
void upReadOnlyAndNoSecrets() throws Exception
{
HealthCheckOutcome outcome = run();
assertClean(outcome, HealthStatus.UP);
assertThat(outcome.getDetails().getValues()).containsEntry("bucketsProbed", 1);
assertThat(calls).contains("listObjectsV2", "getBucketVersioning", "getBucketAcl");
assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.CONNECTION_CAPABILITY))).isEmpty();
assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.BUCKET_CAPABILITY))).isEmpty();
assertThat(Db.asSystem(() -> Db.first(ObjectStorageTables.CONNECTION, "name", "PRIMARY")).getValue("lastTestedAt")).isNull();
}
/*******************************************************************************
** Clock skew uses the same threshold as the record action.
*******************************************************************************/
@Test
void clockSkew() throws Exception
{
skew = CapabilityProbe.CLOCK_SKEW_THRESHOLD_SECONDS - 2;
assertClean(run(), HealthStatus.UP);
skew = CapabilityProbe.CLOCK_SKEW_THRESHOLD_SECONDS + 10;
HealthCheckOutcome outcome = run();
assertClean(outcome, HealthStatus.DEGRADED);
assertThat((Long) outcome.getDetails().getValues().get("clockSkewSeconds")).isGreaterThan(900);
}
/*******************************************************************************
** Expected failures carry only a fixed kind even when errors contain secrets.
*******************************************************************************/
@ParameterizedTest
@ValueSource(strings = { "auth", "tls", "dns", "clock", "connection" })
void downAndNoSecrets(String kind) throws Exception
{
failure = switch(kind)
{
case "auth" -> (S3Exception) S3Exception.builder().statusCode(403).message(SECRET).build();
case "tls" -> new IllegalStateException(SECRET, new SSLHandshakeException(SECRET));
case "dns" -> new IllegalStateException(SECRET, new UnknownHostException(SECRET));
case "clock" -> (S3Exception) S3Exception.builder().statusCode(403).message(SECRET)
.awsErrorDetails(AwsErrorDetails.builder().errorCode("RequestTimeTooSkewed").build()).build();
default -> new IllegalStateException(SECRET);
};
skew = 1000;
HealthCheckOutcome outcome = run();
assertClean(outcome, HealthStatus.DOWN);
assertThat(outcome.getDetails().getValues()).containsEntry("problemKind", kind);
assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.CONNECTION_CAPABILITY))).isEmpty();
}
/*******************************************************************************
** An empty configuration cannot establish connectivity.
*******************************************************************************/
@Test
void noBuckets() throws Exception
{
config.getBuckets().clear();
assertClean(run(), HealthStatus.UNKNOWN);
assertThat(calls).isEmpty();
}
/*******************************************************************************
** Client initialization errors also cannot expose poisoned configuration.
*******************************************************************************/
@Test
@SuppressWarnings("unchecked")
void clientInitializationFailure() throws Exception
{
Field connections = ObjectStorageRuntime.class.getDeclaredField("CONNECTIONS");
connections.setAccessible(true);
((Map<String, ConnectionRuntime>) connections.get(null)).put("PRIMARY", new ConnectionRuntime(config.getConnections().getFirst())
{
/*******************************************************************************
** Simulate a credential or endpoint failure before any request is sent.
*******************************************************************************/
@Override
public synchronized S3Client client()
{
throw (new IllegalStateException(SECRET));
}
});
assertClean(run(), HealthStatus.DOWN);
assertThat(calls).isEmpty();
}
/*******************************************************************************
** Expired checks perform no I/O; a deadline reached mid-probe stops SDK calls.
*******************************************************************************/
@Test
void deadlines() throws Exception
{
assertClean(check().getCheckInstance().run(input().withDeadline(Instant.now().minusSeconds(1))), HealthStatus.DOWN);
assertThat(calls).isEmpty();
expireAfterList = true;
deadline = Instant.now().plusMillis(500);
HealthCheckOutcome outcome = check().getCheckInstance().run(input().withDeadline(deadline));
assertClean(outcome, HealthStatus.DOWN);
assertThat(outcome.getDetails().getValues()).containsEntry("problemKind", "timeout");
assertThat(calls).containsExactly("listObjectsV2");
}
/*******************************************************************************
** Rate limiting also respects the budget, both for slots and depleted tokens.
*******************************************************************************/
@Test
void rateLimiterDeadline() throws Exception
{
RateLimiter limiter = ObjectStorageRuntime.connection("PRIMARY").getRateLimiter();
try(var held = limiter.acquire())
{
assertClean(check().getCheckInstance().run(input().withDeadline(Instant.now().plusMillis(50))), HealthStatus.DOWN);
assertThat(calls).isEmpty();
}
RateLimiter onePerSecond = new RateLimiter(1, 1);
onePerSecond.acquire().close();
assertThatThrownBy(() -> onePerSecond.acquire(Instant.now().plusMillis(20))).isInstanceOf(IllegalStateException.class);
try(var permit = onePerSecond.acquire(Instant.now().plusSeconds(2)))
{
assertThat(permit).isNotNull();
}
}
/*******************************************************************************
** Fetch checks through the registry, exercising the supplier code reference.
*******************************************************************************/
private HealthCheckMetaData check() throws Exception
{
return (HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance).getFirst());
}
/*******************************************************************************
** Default bounded input.
*******************************************************************************/
private HealthCheckInput input()
{
return (new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(20)));
}
/*******************************************************************************
** Run the registered check.
*******************************************************************************/
private HealthCheckOutcome run() throws Exception
{
return (check().getCheckInstance().run(input()));
}
/*******************************************************************************
** HealthRunner.sanitize is package-private; enforce cleanliness on raw output.
*******************************************************************************/
private void assertClean(HealthCheckOutcome outcome, HealthStatus expected)
{
assertThat(outcome.getStatus()).isEqualTo(expected);
assertThat(outcome.getMessage() + outcome.getDetails().toJson()).doesNotContain(SECRET, "private.example", "private-bucket");
assertThat(outcome.getMessage()).doesNotContain("\n", "\r");
}
}
@@ -574,40 +574,4 @@ class MinioIndexTest
.contains("Usage trend").contains("collecting since")
.contains("Connections").contains("Never tested");
}
/*******************************************************************************
** Every link the home emits is a /app/<table>[...] route that the Next frontend
** resolves, never an /<appName>/<table> path, with data and without.
*******************************************************************************/
@Test
void everyHomeLinkUsesTheAppRoute() throws Exception
{
String prefix = com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix();
assertThat(prefix).isEqualTo("/app");
java.util.List<String> hrefs = new java.util.ArrayList<>(hrefs(com.kingsrook.qbits.objectstorage.app.StorageHome.render(prefix, Instant.now())));
rawPut(target.indexBucket, "media/1/a.jpg", 2048);
rawPut(target.indexBucket, "media/9/orphan.jpg", 1024);
ObjectIndex.scan("indexed");
ObjectIndex.headBacklog("indexed", 10);
ReferenceScan.run("indexed");
ObjectStorage.bucket("indexed").put(PutRequest.of("media/1/c.jpg", new byte[3]));
hrefs.addAll(hrefs(com.kingsrook.qbits.objectstorage.app.StorageHome.render(prefix, Instant.now())));
assertThat(hrefs).isNotEmpty().allSatisfy(href -> assertThat(href).startsWith("/app/"));
assertThat(hrefs).noneMatch(href -> href.contains("objectStorageAdmin"));
}
private static java.util.List<String> hrefs(String html)
{
java.util.List<String> hrefs = new java.util.ArrayList<>();
java.util.regex.Matcher m = java.util.regex.Pattern.compile("href=\"([^\"]*)\"").matcher(html);
while(m.find())
{
hrefs.add(m.group(1).replace("&amp;", "&"));
}
return (hrefs);
}
}
@@ -38,7 +38,6 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMe
import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3ChecksumMode;
import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpServer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -59,21 +58,9 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
*******************************************************************************/
class ConnectionRuntimeTest
{
/*******************************************************************************
** Core reads ${file.PATH} only inside its allowed roots; reference() points
** them at each test's temporary directory.
*******************************************************************************/
@AfterEach
void afterEach()
{
System.clearProperty("qqq.fileReference.roots");
}
/*******************************************************************************
** Every core-supported setting is mapped after resolving references. Session
** tokens and presign settings are exercised below.
** tokens and presign settings have no core fields and are exercised below.
*******************************************************************************/
@Test
void testBackendMetaDataCarriesEverySetting(@TempDir Path dir) throws Exception
@@ -212,8 +199,8 @@ class ConnectionRuntimeTest
/*******************************************************************************
** Session credentials (core sessionToken) sign streaming reads/writes and
** paginator requests; request overrides are untouched.
** Session credentials sign streaming reads/writes and paginator requests.
** Existing request headers survive the bridge's credentials override.
*******************************************************************************/
@Test
void testSessionSigningAndRequestOverrides(@TempDir Path dir) throws Exception
@@ -326,7 +313,6 @@ class ConnectionRuntimeTest
*******************************************************************************/
private static String reference(Path dir, String name, String value) throws Exception
{
System.setProperty("qqq.fileReference.roots", dir.toString());
Path file = dir.resolve(name);
Files.writeString(file, value);
return ("${file." + file + "}");
@@ -21,50 +21,25 @@
package com.kingsrook.qbits.objectstorage.internal;
import java.io.Serializable;
import java.util.List;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.api.StorageCaller;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qbits.objectstorage.permissions.ObjectStoragePermissions;
import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import org.assertj.core.api.ThrowableAssert.ThrowingCallable;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Db.asSystem runs work as the core system user. Token-based authentication
** modules (OAuth2, Auth0) accept a QSystemUserSession without a token and
** reject any other tokenless session, so the class matters, not just the
** permissions. The system session is not what lets the service write its own
** tables, though: each Db write carries core's per-call permission bypass, and
** a write that does not is refused to every session.
** permissions.
*******************************************************************************/
class DbSystemSessionTest
{
private static final String REFUSAL = "This storage table changes only through storage actions.";
/*******************************************************************************
** Reset the runtime and context after each test.
*******************************************************************************/
@@ -94,140 +69,4 @@ class DbSystemSessionTest
assertThat(inside.hasSecurityKeyValue("tenantAllAccess", true)).isTrue();
assertThat(QContext.getQSession()).isSameAs(caller);
}
/*******************************************************************************
** A host that enforces permissions inside the core actions
** (QInstance.enforcePermissionsInCoreActions) still gets its operation log
** row, and the row names the person who acted: the caller is captured before
** the write moves to the system user. The other tables the service keeps
** (here the object index) are written, changed and removed the same way.
*******************************************************************************/
@Test
void testServiceWritesItsOwnTablesWhenCoreActionsEnforcePermissions() throws Exception
{
enforcingInstance();
QContext.setQSession(new QSession().withUser(new QUser().withIdReference("pat@example.com").withFullName("Pat Example")).withPermissions(ObjectStoragePermissions.WRITE));
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("stored/by/the/service").write();
assertThat(operations()).as("the operation log row of a stored object").singleElement().satisfies(row ->
{
assertThat(row.getValueString("objectKey")).isEqualTo("stored/by/the/service");
assertThat(row.getValueString("actor")).as("the person who acted, not the system user that wrote the row").isEqualTo("pat@example.com");
});
assertThat(OperationLog.failedWriteCount()).isZero();
Serializable id = Db.asSystem(() -> Db.insert(ObjectStorageTables.OBJECT, new QRecord().withValue("objectKey", "a/b.txt").withValue("size", 1L))).getValue("id");
Db.asSystem(() -> Db.update(ObjectStorageTables.OBJECT, List.of(new QRecord().withValue("id", id).withValue("size", 2L))));
assertThat(Db.asSystem(() -> Db.first(ObjectStorageTables.OBJECT, "id", id)).getValueLong("size")).isEqualTo(2L);
Db.asSystem(() ->
{
Db.delete(ObjectStorageTables.OBJECT, List.of(id));
return (null);
});
assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.OBJECT))).isEmpty();
}
/*******************************************************************************
** Being the system user is not enough to write the operation log. Server code
** outside the service's Db (a host, another qbit) that runs as the core system
** user is refused the insert, the update and the delete, also when it borrows
** Db.asSystem for its session.
*******************************************************************************/
@Test
void testSystemSessionOutsideTheServiceCannotWriteTheOperationLog() throws Exception
{
QInstance instance = enforcingInstance();
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("the/audited/key").write();
Serializable id = operations().get(0).getValue("id");
QContext.setQSession(new QSystemUserSession(instance));
assertRefused("insert as the system user", () -> new InsertAction().execute(new InsertInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("action", "PUT")))));
assertRefused("update as the system user", () -> new UpdateAction().execute(new UpdateInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("id", id).withValue("actor", "nobody")))));
assertRefused("delete as the system user", () -> new DeleteAction().execute(new DeleteInput(ObjectStorageTables.OPERATION).withPrimaryKey(id)));
QContext.setQSession(new QSession());
assertRefused("insert inside Db.asSystem, not through Db", () -> Db.asSystem(() -> new InsertAction().execute(new InsertInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("action", "PUT"))))));
assertRefused("update inside Db.asSystem, not through Db", () -> Db.asSystem(() -> new UpdateAction().execute(new UpdateInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("id", id).withValue("actor", "nobody"))))));
assertRefused("delete inside Db.asSystem, not through Db", () -> Db.asSystem(() -> new DeleteAction().execute(new DeleteInput(ObjectStorageTables.OPERATION).withPrimaryKey(id))));
assertUnchanged();
}
/*******************************************************************************
** An operation row is not updated or deleted by an update or delete action
** that does not carry the bypass, whoever asks: a person, the storage
** administrator or the system user, by key or by filter.
*******************************************************************************/
@Test
void testOperationRowIsNeverUpdatedOrDeletedWithoutTheBypass() throws Exception
{
QInstance instance = enforcingInstance();
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("the/audited/key").write();
Serializable id = operations().get(0).getValue("id");
List<QSession> sessions = List.of(new QSession(), new QSession().withPermissions(ObjectStoragePermissions.ADMIN), new QSystemUserSession(instance));
for(QSession session : sessions)
{
QContext.setQSession(session);
String who = session.getClass().getSimpleName() + (session.hasPermission(ObjectStoragePermissions.ADMIN) ? " (admin)" : "");
assertRefused("update by " + who, () -> new UpdateAction().execute(new UpdateInput(ObjectStorageTables.OPERATION).withRecords(List.of(new QRecord().withValue("id", id).withValue("result", "FAILED")))));
assertRefused("delete by key by " + who, () -> new DeleteAction().execute(new DeleteInput(ObjectStorageTables.OPERATION).withPrimaryKey(id)));
assertRefused("delete by filter by " + who, () -> new DeleteAction().execute(new DeleteInput(ObjectStorageTables.OPERATION).withQueryFilter(new QQueryFilter(Db.eq("action", "PUT")))));
}
assertUnchanged();
}
/*******************************************************************************
** An instance that enforces permissions inside the core actions.
*******************************************************************************/
private static QInstance enforcingInstance() throws QException
{
ObjectStorageQBitConfig config = new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false);
return (StorageTestSupport.setUp(config, instance -> instance.setEnforcePermissionsInCoreActions(true)));
}
/*******************************************************************************
** The operation log, read by the service.
*******************************************************************************/
private static List<QRecord> operations() throws QException
{
return (Db.asSystem(() -> Db.query(ObjectStorageTables.OPERATION)));
}
/*******************************************************************************
** The qbit's checker refused the write.
*******************************************************************************/
private static void assertRefused(String what, ThrowingCallable write)
{
assertThatThrownBy(write).as(what).isInstanceOf(QPermissionDeniedException.class).hasMessage(REFUSAL);
}
/*******************************************************************************
** The one row the service wrote is still there, as it was written.
*******************************************************************************/
private static void assertUnchanged() throws QException
{
assertThat(operations()).singleElement().satisfies(row ->
{
assertThat(row.getValueString("objectKey")).isEqualTo("the/audited/key");
assertThat(row.getValueString("result")).isEqualTo("OK");
assertThat(row.getValueString("actor")).isEqualTo("system (no user)");
});
}
}
@@ -83,23 +83,10 @@ class InternalsTest
assertThat(SecretRefs.resolve(null)).isNull();
assertThat(SecretRefs.resolveSetting("")).isNull();
//////////////////////////////////////////////////////////////////////
// core reads ${file.PATH} only inside its allowed roots (by default //
// /var/run/secrets and /run/secrets); point them at the temp dir. //
//////////////////////////////////////////////////////////////////////
Path file = dir.resolve("key");
Files.writeString(file, " from-file\n");
assertThat(SecretRefs.resolve("${file." + file + "}")).as("outside the allowed roots").isNull();
System.setProperty("qqq.fileReference.roots", dir.toString());
try
{
assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
}
finally
{
System.clearProperty("qqq.fileReference.roots");
}
assertThat(SecretRefs.resolve("${file." + file + "}")).isEqualTo("from-file");
assertThat(SecretRefs.resolve("${file." + dir.resolve("missing") + "}")).isNull();
}
@@ -1,240 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage.internal;
import java.util.EnumSet;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.function.Consumer;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.api.Capability;
import com.kingsrook.qbits.objectstorage.api.CapabilityStatus;
import com.kingsrook.qbits.objectstorage.config.BucketRole;
import com.kingsrook.qbits.objectstorage.config.ProviderType;
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
import com.kingsrook.qqq.backend.core.cluster.ClusterBus;
import com.kingsrook.qqq.backend.core.cluster.ClusterBusInterface;
import com.kingsrook.qqq.backend.core.cluster.ClusterBusMetaData;
import com.kingsrook.qqq.backend.core.cluster.ClusterCacheAudit;
import com.kingsrook.qqq.backend.core.cluster.ClusterEvent;
import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** The Object Storage caches are correct on several nodes: every static cache
** is registered with ClusterCaches, a probe result recorded on one node clears
** that capability on the others, and an edited bucket row or header rule
** clears its cache on every node.
*******************************************************************************/
class ObjectStorageClusterCachesTest
{
/** the "wire": events every node's transport sees */
static final List<Consumer<ClusterEvent>> WIRE = new CopyOnWriteArrayList<>();
/*******************************************************************************
** A transport over the static WIRE, so a "remote" publish reaches us.
*******************************************************************************/
public static class WireBus implements ClusterBusInterface
{
/*******************************************************************************
**
*******************************************************************************/
@Override
public void publish(ClusterEvent event)
{
WIRE.forEach(listener -> listener.accept(event));
}
/*******************************************************************************
**
*******************************************************************************/
@Override
public AutoCloseable subscribe(Consumer<ClusterEvent> listener)
{
WIRE.add(listener);
return (() -> WIRE.remove(listener));
}
}
/*******************************************************************************
** A synced instance whose buckets never touch a real store, on the wire as
** node-a.
*******************************************************************************/
@BeforeEach
void setUp() throws Exception
{
System.setProperty("os.unit.ak", "ak");
System.setProperty("os.unit.sk", "sk");
StorageTestSupport.setUp(new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false).withOpenWhenAnonymous(true).withAdminApp(true)
.withAllowedSecretRefPatterns(List.of("prop.os.unit.*"))
.withConnection(new StorageConnectionConfig("main").withProviderType(ProviderType.MINIO).withEndpoint("http://localhost:1").withRegion("us-east-1")
.withAccessKeyRef("${prop.os.unit.ak}").withSecretKeyRef("${prop.os.unit.sk}"))
.withBucket(new StorageBucketConfig("media").withConnection("main").withBucketName("media-bucket").withRole(BucketRole.PRIVATE).withIndexed(false)),
instance -> instance.withSupplementalMetaData(new ClusterBusMetaData().withBus(new QCodeReference(WireBus.class))));
QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, null, "node-a"));
ClusterBus.connect(QContext.getQInstance());
}
/*******************************************************************************
**
*******************************************************************************/
@AfterEach
void tearDown()
{
ClusterBus.close(QContext.getQInstance());
WIRE.clear();
QNodeRoles.set(null);
StorageTestSupport.tearDown();
}
/*******************************************************************************
** Every static Memoization in the QBit is registered (shared or node-local).
*******************************************************************************/
@Test
void testNoUnregisteredStaticCaches() throws Exception
{
assertThat(ClusterCacheAudit.unregisteredStaticCaches("com.kingsrook.qbits.objectstorage")).isEmpty();
}
/*******************************************************************************
** Another node's probe result clears the capability here; the next read
** reloads it from the capability row.
*******************************************************************************/
@Test
void testRemoteProbeResultClearsCapability() throws Exception
{
StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
CapabilityStore.record(bucket, Capability.VERSIONING, CapabilityStatus.UNSUPPORTED, "probe here");
assertThat(CapabilityStore.isCached(bucket, Capability.VERSIONING)).isTrue();
////////////////////////////////////////////////////////////
// node-b probes again and writes SUPPORTED to the row //
////////////////////////////////////////////////////////////
Integer bucketId = ConfigSync.bucketId("media");
Db.asSystem(() ->
{
QRecord row = Db.first(ObjectStorageTables.BUCKET_CAPABILITY, "bucketId", bucketId, "capability", Capability.VERSIONING.name());
Db.update(ObjectStorageTables.BUCKET_CAPABILITY, List.of(new QRecord().withValue("id", row.getValue("id")).withValue("status", CapabilityStatus.SUPPORTED.name())));
return (null);
});
ClusterEvent remote = new ClusterEvent(ClusterEvent.TYPE_MEMO_INVALIDATE, CapabilityStore.CACHE_NAME, "b:media:" + Capability.VERSIONING);
remote.setOriginNodeId("node-b");
WIRE.forEach(listener -> listener.accept(remote));
assertThat(CapabilityStore.isCached(bucket, Capability.VERSIONING)).isFalse();
assertThat(CapabilityStore.status("media", Capability.VERSIONING)).isEqualTo(CapabilityStatus.SUPPORTED);
}
/*******************************************************************************
** A probe result recorded here tells the other nodes, and stays cached here.
*******************************************************************************/
@Test
void testProbeResultPublishesInvalidation()
{
List<ClusterEvent> sent = sentEvents();
StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
CapabilityStore.record(bucket, Capability.VERSIONING, CapabilityStatus.SUPPORTED, "probe here");
assertThat(CapabilityStore.status("media", Capability.VERSIONING)).isEqualTo(CapabilityStatus.SUPPORTED);
assertThat(sent).anySatisfy(event -> assertInvalidation(event, CapabilityStore.CACHE_NAME, "b:media:VERSIONING"));
}
/*******************************************************************************
** An edited bucket row takes effect at once and tells the other nodes.
*******************************************************************************/
@Test
void testBucketRowEditInvalidatesRowLimitsEverywhere() throws Exception
{
StorageBucketConfig bucket = ObjectStorageRuntime.bucketConfig("media");
assertThat(ConfigSync.effectiveMaxPresignGetSeconds(bucket)).isEqualTo(3600);
List<ClusterEvent> sent = sentEvents();
Db.update(ObjectStorageTables.BUCKET, List.of(new QRecord().withValue("id", ConfigSync.bucketId("media")).withValue("maxPresignGetSeconds", 60)));
assertThat(ConfigSync.effectiveMaxPresignGetSeconds(bucket)).as("no wait for the 15 s cache").isEqualTo(60);
assertThat(sent).anySatisfy(event -> assertInvalidation(event, ConfigSync.ROW_LIMITS_CACHE_NAME, null));
}
/*******************************************************************************
** An edited header rule tells the other nodes.
*******************************************************************************/
@Test
void testHeaderRuleEditInvalidatesRulesEverywhere() throws Exception
{
List<ClusterEvent> sent = sentEvents();
StorageTestSupport.insert(ObjectStorageTables.HEADER_RULE, new QRecord().withValue("name", "all").withValue("keyGlob", "**").withValue("cacheControl", "no-store"));
assertThat(sent).anySatisfy(event -> assertInvalidation(event, DefaultObjectHeaderPolicy.CACHE_NAME, null));
}
/*******************************************************************************
** Collect what this node puts on the wire.
*******************************************************************************/
private static List<ClusterEvent> sentEvents()
{
List<ClusterEvent> sent = new CopyOnWriteArrayList<>();
WIRE.add(sent::add);
return (sent);
}
/*******************************************************************************
** A memo invalidation from node-a for a cache (and key).
*******************************************************************************/
private static void assertInvalidation(ClusterEvent event, String name, String key)
{
assertThat(event.getType()).isEqualTo(ClusterEvent.TYPE_MEMO_INVALIDATE);
assertThat(event.getName()).isEqualTo(name);
assertThat(event.getKey()).isEqualTo(key);
assertThat(event.getOriginNodeId()).isEqualTo("node-a");
}
}
@@ -1,125 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.objectstorage.internal;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
import com.kingsrook.qbits.objectstorage.api.StorageAlertSink;
import com.kingsrook.qbits.objectstorage.api.StorageCaller;
import com.kingsrook.qbits.objectstorage.model.ObjectStorageTables;
import com.kingsrook.qbits.objectstorage.testsupport.StorageTestSupport;
import com.kingsrook.qqq.backend.core.logging.CollectedLogMessage;
import com.kingsrook.qqq.backend.core.logging.QCollectingLogger;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import org.apache.logging.log4j.Level;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** An operation log row that cannot be written does not fail the operation it
** describes, so an operator has to be able to see it: an ERROR log line, the
** storage.operation_log.failed alert and OperationLog.failedWriteCount().
*******************************************************************************/
class OperationLogTest
{
private final List<String> alerts = new ArrayList<>();
/*******************************************************************************
** Reset the runtime, the context and the logger after each test.
*******************************************************************************/
@AfterEach
void tearDown()
{
QLogger.deactivateCollectingLoggerForClass(OperationLog.class);
StorageTestSupport.tearDown();
}
/*******************************************************************************
** A row the insert action refuses (it hands the record back with an error, it
** does not throw) and a row whose insert throws are both counted, logged at
** ERROR and raised as an alert; the caller is not failed. The next row written
** for the bucket resolves the alert and the count stays.
*******************************************************************************/
@Test
void testFailedRowIsAnErrorAnAlertAndACount() throws Exception
{
QInstance instance = StorageTestSupport.setUp(new ObjectStorageQBitConfig().withBackendName(StorageTestSupport.MEMORY).withBucketScoping(false));
ObjectStorageRuntime.setAlertSink(new StorageAlertSink()
{
/*******************************************************************************
** Record the alert.
*******************************************************************************/
@Override
public void raise(String alertType, String dedupeKey, String severity, String message, Map<String, Object> details)
{
alerts.add("raise " + alertType + " " + dedupeKey + " " + severity + " count=" + details.get("count") + " action=" + details.get("action"));
}
/*******************************************************************************
** Record the resolution.
*******************************************************************************/
@Override
public void resolve(String alertType, String dedupeKey)
{
alerts.add("resolve " + alertType + " " + dedupeKey);
}
});
QCollectingLogger logger = QLogger.activateCollectingLoggerForClass(OperationLog.class);
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("a").write();
assertThat(OperationLog.failedWriteCount()).isZero();
assertThat(alerts).isEmpty();
instance.getTable(ObjectStorageTables.OPERATION).getField("detail").setIsRequired(true);
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("refused").write();
assertThat(OperationLog.failedWriteCount()).isEqualTo(1);
assertThat(alerts).containsExactly("raise storage.operation_log.failed bucket:files error count=1 action=PUT");
instance.getTable(ObjectStorageTables.OPERATION).getField("detail").setIsRequired(false);
String backendName = instance.getTable(ObjectStorageTables.OPERATION).getBackendName();
instance.getTable(ObjectStorageTables.OPERATION).setBackendName("no-such-backend");
new OperationLog.Entry(StorageCaller.user(), "DELETE", "files").key("thrown").write();
assertThat(OperationLog.failedWriteCount()).isEqualTo(2);
assertThat(alerts).last().isEqualTo("raise storage.operation_log.failed bucket:files error count=2 action=DELETE");
List<CollectedLogMessage> errors = logger.getCollectedMessages().stream().filter(message -> Level.ERROR.equals(message.getLevel())).toList();
assertThat(errors).hasSize(2).allSatisfy(message -> assertThat(message.getMessage()).contains("Could not write a storage operation log row"));
instance.getTable(ObjectStorageTables.OPERATION).setBackendName(backendName);
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("b").write();
new OperationLog.Entry(StorageCaller.user(), "PUT", "files").key("c").write();
assertThat(alerts).hasSize(3).last().isEqualTo("resolve storage.operation_log.failed bucket:files");
assertThat(OperationLog.failedWriteCount()).as("the count is of this node since it started, a later row does not lower it").isEqualTo(2);
assertThat(Db.asSystem(() -> Db.query(ObjectStorageTables.OPERATION))).extracting(row -> row.getValueString("objectKey")).containsExactly("a", "b", "c");
}
}
@@ -205,7 +205,7 @@ class TablesAndConfigTest
new com.kingsrook.qbits.objectstorage.ObjectStorageQBitProducer().withQBitConfig(config).produce(instance);
assertThat(instance.getApp("objectStorageAdmin").getParentAppName()).isEqualTo("admin");
assertThat(instance.getApp("admin").getChildren()).extracting(c -> c.getName()).contains("objectStorageAdmin");
assertThat(com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix()).isEqualTo("/app");
assertThat(com.kingsrook.qbits.objectstorage.app.StorageWidgets.tablePathPrefix()).isEqualTo("/admin/objectStorageAdmin");
}