Compare commits

..
Author SHA1 Message Date
james.maes 79d241e1f6 Merge pull request 'build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)' (#40) from deps/align-gitea into develop
build / maven (push) Successful in 2m41s
maven / build (push) Successful in 2m41s
2026-11-07 19:30:09 +00:00
James MaesandClaude Opus 5.5 c47d5a1c18 fix(upload): stop using status as the record-version field
build / maven (push) Successful in 2m14s
maven / build (push) Successful in 2m14s
QQQ 4.1.0-RC.26 (RecordVersionCheck.advance) advances a table's configured
record-version field on every update; a STRING version becomes a random
UUID. The upload-job table declared status as its version field, so every
update overwrote status with a UUID.

Drop the version field and compare the expected status inside the existing
account-lock transaction, which already serializes transitions across pods.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 13:26:54 -06:00
james.maesandClaude Opus 5.5 677aa54801 build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)
build / maven (push) Failing after 3m57s
maven / build (push) Failing after 3m57s
pom.xml: parent -> 2.2.1
pom.xml: qqq.snapshot.version: 4.1.0-RC.7 -> 4.1.0-RC.26
pom.xml: qbit-object-storage: 0.1.0-RC.2 -> 0.1.0-RC.10
pom.xml: qqq-frontend-next: 1.0.0-RC.14 -> 1.0.0-RC.38
pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-07 17:55:47 +00:00
james.maes 44a463e4d9 Merge pull request 'ci: pin shared workflow to v1' (#39) from ci/pin-v1-develop into develop
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1e25bb5a243dfdcd95d4b9a4f780200ff8c8778a] (push) Successful in 15s
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 13s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 10s
build / maven (push) Successful in 4m4s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (push) Skipped
maven / build (push) Successful in 4m4s
2026-11-06 16:07:29 +00:00
james.maesandClaude Opus 5.5 32f91b75c2 ci: pin shared workflow to v1
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Waiting to run
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
build / maven (push) Successful in 4m59s
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1e25bb5a243dfdcd95d4b9a4f780200ff8c8778a] (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Successful in 4m58s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 6s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-06 14:09:43 +00:00
james.maes 32239cd57d Merge pull request 'ci: use shared ci-workflows Maven workflow' (#38) from ci/ci-workflows into develop
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 5s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / original-voyage (push) Skipped
QRun-IO/ci-policy: ci / original-cli (push) Skipped
QRun-IO/ci-policy: ci / ci-result (push) Successful in 5s
build / maven (push) Successful in 3m17s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow d20a578c7d818bc7ca32a2ac2c675da4e4f27385] (push) Successful in 6s
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-business (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (push) Skipped
maven / build (push) Successful in 3m17s
2026-11-05 23:16:48 +00:00
james.maesandClaude Opus 5.5 950a8018c7 ci: use shared ci-workflows Maven workflow
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / Scoped workflow [workflow 1b3648d9421543350067e95e54959dfb471b26d5] (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / original-business (pull_request) Skipped
QRun-IO/ci-policy: ci / original-mcp (pull_request) Skipped
QRun-IO/ci-policy: ci / original-voyage (pull_request) Skipped
QRun-IO/ci-policy: ci / original-cli (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI / verify (pull_request) Successful in 2m39s
build / maven (push) Successful in 3m36s
QRun-IO/ci-policy: Shared Java Maven CI / secrets (pull_request) Failing after 2s
QRun-IO/ci-policy: Shared Java Maven CI / gate (pull_request) Failing after 2s
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Failing after 10m20s
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 2s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 7s
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / shared-pipeline (pull_request) Skipped
maven / build (push) Successful in 3m36s
Replace the platform-ci release callers with one caller of
KofTwentyTwo/ci-workflows maven.yaml (qqq-orb port).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 22:05:20 +00:00
forge-recovery 43bfe48feb ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: Shared Java Maven CI / verify (push) Successful in 3m0s
Shared Java Maven CI / verify (push) Successful in 3m31s
qrun-release / postpublish (push) Successful in 11m14s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 4s
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 6s
Shared Java Maven CI / secrets (push) Successful in 21s
QRun-IO/ci-policy: Shared Java Maven CI / secrets (push) Successful in 20s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI / gate (push) Successful in 13s
QRun-IO/ci-policy: ci / java-maven-6 (push) Successful in 46m47s
Shared Java Maven CI / gate (push) Successful in 14s
qrun-release / release (push) Successful in 47m5s
Shared QRun source version / version (push) Successful in 11m14s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
2026-11-02 22:16:24 +00:00
forge-recovery ead0c904ce ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI / secrets (push) Successful in 31s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: native / native-result (push) Successful in 7s
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI / verify (push) Successful in 3m2s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 11s
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI / gate (push) Successful in 23s
QRun-IO/ci-policy: ci / java-maven-6 (push) Successful in 15m34s
2026-11-02 21:33:26 +00:00
forge-recoveryandCodex 579946d62c ci: enroll shared Gitea release-candidate lifecycle
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI / secrets (pull_request) Successful in 17s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI / verify (pull_request) Successful in 2m24s
QRun-IO/ci-policy: Shared Java Maven CI / gate (pull_request) Successful in 12s
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Successful in 1h0m9s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 5s
Release candidate / release-rc (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 5s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / android-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 20:17:19 +00:00
4f53c29318 ci: add the isolated postpublish job to the QRun release caller
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
Shared Java Maven CI / secrets (push) Successful in 17s
QRun-IO/ci-policy: Shared Java Maven CI / secrets (pull_request) Successful in 17s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
Shared Java Maven CI / verify (push) Canceled after 0s
qrun-release / postpublish (push) Canceled after 0s
QRun-IO/ci-policy: Shared Java Maven CI / verify (pull_request) Successful in 2m2s
QRun-IO/ci-policy: Shared Java Maven CI / gate (pull_request) Successful in 8s
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Successful in 2h57m45s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 4s
Release candidate / release-rc (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 6s
Shared Java Maven CI / gate (push) Canceled after 0s
qrun-release / release (push) Canceled after 2h37m34s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 15:49:26 +00:00
cd3c0393a2 ci: move the QRun release caller to the thin v2 contract
QRun-IO/ci-policy: Shared Java Maven CI / gate (pull_request) Canceled after 0s
Shared Java Maven CI / verify (push) Canceled after 0s
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 4s
Shared Java Maven CI / secrets (push) Successful in 26s
QRun-IO/ci-policy: Shared Java Maven CI / secrets (pull_request) Successful in 24s
QRun-IO/ci-policy: Shared Java Maven CI / verify (pull_request) Successful in 3m40s
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Canceled after 1h2m7s
Shared Java Maven CI / gate (push) Canceled after 0s
qrun-release / release (push) Canceled after 1h2m33s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-7 (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 6s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 06:18:17 +00:00
forge-recoveryandCodex d9f74cb576 ci: add the identical QRun release caller
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI / verify (pull_request) Successful in 3m39s
QRun-IO/ci-policy: Shared Java Maven CI / secrets (pull_request) Successful in 26s
QRun-IO/ci-policy: Shared Java Maven CI / gate (pull_request) Failing after 0s
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 0s
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Failing after 7m20s
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 04:11:39 +00:00
james.maesandCodex d12a7777af ci: adopt managed orb 0.8 patch track (#31)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 16:02:30 -05:00
james.maesandCodex 5f6a5022ce ci: adopt current shared orb standard (#30)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 14:52:14 -05:00
james.maesandCodex 3c9a4d229b ci: use the shared branch-aware CodeQL worker (#28)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 09:44:05 -05:00
james.maesandCodex c5e1dd7e14 ci: adopt central orb 0.7.12 and stable gates (#26)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 03:32:34 -05:00
james.maesandCodex d6ebb60de6 ci: centralize rc release completion in qqq orb (#25)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 02:23:27 -05:00
james.maesandCodex e41bf1147b ci: adopt shared orb 0.7.11 (#24)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 00:54:13 -05:00
james.maes d4ca427445 Merge pull request #22 from QRun-IO/feature/central-ci
ci: adopt shared QQQ orb standard
2026-10-20 19:19:13 -05:00
james.maesandCodex 2542dc0f89 ci: adopt verified central QQQ orb 0.7.3
Co-Authored-By: Codex <noreply@openai.com>
2026-10-20 19:08:07 -05:00
james.maes c6954d86f1 Merge pull request #20 from QRun-IO/feature/permissions-in-catalog
feat(permissions): file_gateway.view/admin/connect in the permission catalog
2026-10-16 12:01:44 -05:00
james.maesandClaude Opus 5.5 05bfcd8d4b feat(permissions): list file_gateway.view/admin/connect in the permission catalog
The gateway checked these names only in code, so getAllAvailablePermissions
never listed them: the qbit-authentik mapper expanded them to nothing and
Identity policies could not grant them. FileGatewayPermissionChecker now
implements buildAvailablePermission, collapsing every gateway object onto
the three names (reads/widgets -> view, writes/processes -> admin, app ->
connect). Adds a test that all three are in the catalog.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 11:58:21 -05:00
james.maes d3cc594b0e Merge pull request #19 from QRun-IO/fix/file-gateway-increase-is-good
fix(home): File Gateway tile directions and reasoned empty tables
2026-10-16 10:19:54 -05:00
james.maesandClaude Opus 5.5 ff182b6d7f fix(home): File Gateway tile directions and reasoned empty tables
Failed jobs and Locked accounts now declare increaseIsGood=false (volume tiles
stay true). The four home tables (Recent failures, Top partners, Active
sessions, Server) set noRowsFoundHTML saying why they are empty.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 09:10:36 -05:00
james.maes 7482050234 Merge pull request #18 from QRun-IO/ci/release-rc-secrets-inherit
ci(release-rc): pass secrets so the shared workflow can sign the RC tag
2026-10-16 08:05:09 -05:00
james.maesandClaude Opus 5.5 27324a4159 ci(release-rc): pass secrets to the shared workflow so it can sign the RC tag
QRun-IO/.github#2 adds a tag-and-release job that signs vX with the
QRUN_TAG_SIGNING_KEY secret after Maven Central confirms the artifact.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 16:20:17 -05:00
james.maes 37ee9028f2 Merge pull request #17 from QRun-IO/fix/no-log-before-auth
fix(sftp): no connection-log row for connections that never sign in
2026-10-15 07:19:19 -05:00
james.maesandClaude Opus 5.5 c8a231f5f8 fix(sftp): no connection-log row for connections that never sign in
The row was inserted at TCP accept with a provisional AUTH_FAILED, so the
kubelet's TCP readiness probe on the SFTP port wrote a "failed sign-in"
row every 5 seconds per pod in kof22 prod (about 35k a day). The row is
now written at the first authentication attempt or a pre-auth refusal
(the connection rate limit); probes and port scans leave nothing. The
per-IP connection rate is still counted at accept.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-15 07:15:38 -05:00
james.maes 0a8c95e0ad Merge pull request #16 from QRun-IO/fix/home-tables-four-columns
fix(home): home tables have at most four columns
2026-10-14 17:40:24 -05:00
james.maesandClaude Opus 5.5 f0e9836a89 fix(home): home tables have at most four columns
The Active sessions table had five (the pod is on the connection record
and in the Server table). kof22's readability guard allows four, so
tables stay readable at phone widths; a test now checks every home table.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:37:48 -05:00
james.maes dd4643e60d Merge pull request #14 from QRun-IO/fix/home-table-no-widths
fix(home): home tables declare no column widths
2026-10-14 17:07:24 -05:00
james.maesandClaude Opus 5.5 483c856594 fix(home): home tables declare no column widths
Fixed 1fr widths made Next lay the home tables out at a minimum width
(the kof22 readability guard, docs/admin-dashboards.md). Columns now
declare no width; a test keeps it that way.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 17:00:03 -05:00
james.maes 275cc26b6b Merge pull request #13 from QRun-IO/fix/optional-saved-object-pickers
fix: endpoint pickers only for saved-object sources the host registers
2026-10-14 16:56:13 -05:00
james.maesandClaude Opus 5.5 3e5cec20d2 fix: endpoint pickers only for saved-object sources the host registers
A host without saved reports (kof22) failed instance validation:
file_gateway_alias.savedReportId named the unregistered possible-value
source savedReport (and its bulkEdit process inherited it). The producer
now drops the picker and its contingent filter for savedView, savedReport
and savedBulkLoadProfile when the host has not registered that source.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 16:45:02 -05:00
james.maes 1713d18fd7 Merge pull request #10 from QRun-IO/feature/preferences-file-transfer
feat(preferences): the File transfer section (SSH keys self-service)
2026-10-14 16:28:20 -05:00
james.maes 6073091434 Merge pull request #12 from QRun-IO/feature/pin-qqq-rc7
build: pin QQQ 4.1.0-RC.7 (first release with QRun-IO/qqq#1170)
2026-10-14 16:26:10 -05:00
james.maes db241b03bc Merge feature/pin-qqq-rc7 into feature/preferences-file-transfer
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

# Conflicts:
#	AGENTS.md
#	CHANGELOG.md
#	CLAUDE.md
#	README.md
#	docs/DESIGN.md
2026-10-14 16:25:24 -05:00
james.maesandClaude Opus 5.5 10eb8807ba Merge develop into feature/pin-qqq-rc7
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 16:20:51 -05:00
james.maes ac55e437bd Merge pull request #11 from QRun-IO/feature/orb-0.6.9
ci: qqq-orb 0.6.9 (refresh SNAPSHOT dependencies)
2026-10-14 10:46:06 -05:00
james.maesandClaude Opus 5.5 3ba219d668 ci: qqq-orb 0.6.9 (refresh SNAPSHOT dependencies)
Moves qqq-orb@0.6.8 to qqq-orb@0.6.9, which passes --update-snapshots so cached SNAPSHOT dependencies are re-checked on every build (QRun-IO/qqq-orb#22).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 10:07:05 -05:00
james.maesandClaude Opus 5.5 d0572402a9 ci: re-run against the develop SNAPSHOT that includes QRun-IO/qqq#1162
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:57:19 -05:00
james.maesandClaude Opus 5.5 e6ff62181b feat(preferences): the "File transfer" section (SSH keys self-service)
A user-preferences section (QRun-IO/qqq#1162) for anyone with
file_gateway.connect: the SFTP username, host and host-key fingerprint,
and the user's own keys. "Add key" creates the user's account on first use
and inserts the key through the table (validation and audits apply);
"Revoke" sets revoked_at; labels and expiry are editable. The provider
resolves the user only from the request context and touches only that
user's account and keys. Duplicate fingerprints are rejected up front.

99 tests (11 new); JaCoCo 92.6% instructions, 97.7% classes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:29:31 -05:00
james.maesandClaude Opus 5.5 dc8ac5f089 Merge develop into feature/preferences-file-transfer
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:29:30 -05:00
james.maesandClaude Opus 5.5 7a5e42495d build: pin QQQ 4.1.0-RC.7 (first release with QRun-IO/qqq#1170)
Replaces the 4.1.0-SNAPSHOT stand-in (guardrail G7) and drops the private
FG-SNAPSHOT build notes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:18:38 -05:00
james.maes 5098152460 Merge pull request #8 from QRun-IO/feature/phase-e-endpoints-home
feat: endpoints, outbox, PGP, app home and retention (phase e)
2026-10-14 09:17:42 -05:00
james.maes ab64cbf040 Merge pull request #7 from QRun-IO/feature/phase-c-downloads
feat: downloads of saved views and saved reports (phase c)
2026-10-14 09:17:29 -05:00
james.maes 6261e197d0 Merge pull request #6 from QRun-IO/feature/phase-b-upload
feat: upload to bulk load through object storage (phase b)
2026-10-14 09:17:14 -05:00
james.maes 3672bb513d Merge pull request #5 from QRun-IO/feature/phase-a-sftp-server
feat: SFTP server, virtual tree, SSH-key auth and listing (phase a)
2026-10-14 09:17:00 -05:00
james.maesandClaude Opus 5.5 182fb46a66 Merge feature/phase-c-downloads into feature/phase-e-endpoints-home
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:13:32 -05:00
james.maesandClaude Opus 5.5 516c250d4f Merge feature/phase-b-upload into feature/phase-c-downloads
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:13:29 -05:00
james.maesandClaude Opus 5.5 c0646db521 Merge feature/phase-c-downloads into feature/phase-e-endpoints-home
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:13:20 -05:00
james.maesandClaude Opus 5.5 5fcf0d1f8d Merge feature/phase-b-upload into feature/phase-c-downloads
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:13:20 -05:00
james.maesandClaude Opus 5.5 c1b78d3077 ci: re-run against the QQQ 4.1.0-SNAPSHOT that includes QRun-IO/qqq#1170
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:08:47 -05:00
james.maesandClaude Opus 5.5 00a20eb5c8 ci: re-run against the QQQ 4.1.0-SNAPSHOT that includes QRun-IO/qqq#1170
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 09:08:45 -05:00
james.maesandClaude Opus 5.5 24f1cbec33 docs(k8s): note the older io.cilium/lb-ipam-ips annotation (k8s-galaxy)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 23:02:59 -05:00
james.maesandClaude Opus 5.5 415d0bdd0a feat(ddl): indexes for the job queue, dedupe, live sessions and retention
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 22:56:01 -05:00
james.maesandClaude Opus 5.5 89e468ca45 feat: endpoints, outbox, PGP, app home and retention (phase e)
Phase (e) of docs/DESIGN.md:
- endpoints (aliases) with path validation and the ENDPOINTS_ONLY chroot;
  BULK_LOAD, INBOX (consumer by name), VIEW, REPORT and OUTBOX kinds;
- the outbox provider (FileGatewayOutbox.put) in every deployment, outbox
  files, pickup tracking and onDownloaded;
- optional per-account PGP: decrypt uploads, encrypt downloads (keys in
  memory, BouncyCastle);
- the typed app home (KPIs, recent failures, top partners, active
  sessions, gateway servers via file_gateway_server heartbeats);
- Unlock/Disable account, Revoke key and End session record actions;
- the daily retention process; FileGatewayIntegrations (ESB, Quick
  Search, API);
- docs/k8s example manifests (SFTP port 22 only, LB IP 10.100.22.65);
- plain labels for fields, enum values and child tabs; a test-scope demo
  server for screenshots.

88 tests; JaCoCo 92% instructions.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 22:52:41 -05:00
james.maesandClaude Opus 5.5 3296cb9ca4 feat: downloads of saved views and saved reports (phase c)
Phase (c) of docs/DESIGN.md:
- views/<slug>.<csv|xlsx|json>: the saved view's filter, columns and order
  exported as the user through /qqq/v1 table export, streamed;
- reports/<slug>.<fmt>: renderSavedReport run as the user, then the file
  streamed from /download (listed only when a report storage table exists);
- a generated read channel with a bounded in-memory read-ahead window (no
  disk), maxGeneratedFileMb and the daily download cap, DOWNLOAD transfer
  rows.

Masking is what /qqq/v1 returns for the user: tested with a
FieldSecurityLock DENY field, byte-compared with a direct export.
77 tests; JaCoCo 92.6% instructions, 97.2% classes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-13 22:17:13 -05:00
james.maes 852d474722 Merge pull request #4 from QRun-IO/feature/spec
docs(spec): qbit-file-gateway design
2026-10-13 19:43:04 -05:00
james.maes ec8092e468 Merge pull request #1 from QRun-IO/feature/scaffold
chore: repo-standard scaffold for qbit-file-gateway
2026-10-13 19:33:18 -05:00
57 changed files with 6150 additions and 209 deletions

No files matched your search

+10 -61
View File
@@ -1,65 +1,14 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.6.8
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
qqq-orb: kingsrook/qqq-orb@0.8
workflows:
test_only:
setup:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
filters:
branches:
ignore: /(develop|release/.*|integration.*)/
tags:
only: []
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: snapshot
filters:
branches:
only:
- develop
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: release_candidate
filters:
branches:
only:
- /release\/.*/
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [ /^v\d+\.\d+\.\d+$/ ]
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
+7
View File
@@ -0,0 +1,7 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
+23
View File
@@ -0,0 +1,23 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+62 -13
View File
@@ -1,25 +1,74 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
on:
'on':
workflow_dispatch:
inputs:
dry_run:
description: Print the plan without merging or creating branches
description: Plan a cut without changing release state
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types: [closed]
branches: [develop]
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
+16
View File
@@ -0,0 +1,16 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release-no-integration
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
+28 -11
View File
@@ -14,13 +14,12 @@ code and QInstance. Design: `docs/DESIGN.md`.
## Build commands
```bash
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT compile
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT test
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
./mvnw compile
./mvnw test
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
```
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd` and quote the `-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT` argument in PowerShell.
Keep the POM default at `4.1.0-SNAPSHOT` for CI; the private version prevents local artifact replacement.
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
## Architecture
@@ -51,12 +50,30 @@ Claims compare expected status under an account-row transaction; core's optimist
alone does not add a SQL WHERE guard. Fresh worker user sessions outlive SSH connections and are
revoked after delivery. Bulk process multipart field: `theFile`; mapping frontend step: `fileMapping`
(resumes `receiveFileMapping`). RC.2 object storage owns S3 part size at connection level. The
private invoker selects multipart only when a file part is present, so the mapping request
restreams the file with its values. See README adoption prerequisites and compatibility notes.
private invoker supports multipart process values without a file part, so the mapping request
sends only its values. See README adoption prerequisites and compatibility notes.
Saved-view/report downloads, endpoint routing/chroot, outbox delivery, Quick Search/ESB,
preferences, retention and the app home belong to later phases. The core-valid alert type is `filegateway.lockout`; the design's
underscore-only name is rejected by core validation.
Phase (c) adds `GeneratedDownloads` (user-scoped export/render/download API flow and durable
account-locked download quotas) and `GeneratedReadChannel` (fixed rolling window, no spool).
Saved-view JSON uses `queryFilter.orderBys` and `queryColumns.columns` visibility/order. Generated
stat size is zero; reads fill requests until real EOF to avoid MINA's short-read EOF inference.
Reports list only with a configured render-process storage table. FieldSecurityLock DENY tests
require `enforcePermissionsInCoreActions=true` and match direct same-user UI export bytes.
Core `RenderSavedReportExecuteStep` stages a rendered report in a JVM temp file and deletes it
in try-with-resources; that is the bounded temp area the owner rule allows (in-memory `/tmp`).
The core-valid alert type is `filegateway.lockout` (dotted names only).
Phase (e) adds endpoints and chroot (`AliasCustomizer` validates paths), `outbox/` (the
`FileGatewayOutboxProvider`, registered in every deployment), `pgp/` (BouncyCastle stream
decrypt/encrypt, keys in memory), `maintenance/` (daily retention process and record-action
effects), `FileGatewayHome` (typed STATISTICS and TABLE widgets only; guardrail G5),
`FileGatewayIntegrations` (ESB, Quick Search, API lists) and `file_gateway_server` heartbeats.
The `file-transfer` Preferences section requires only `file_gateway.connect`. Its provider uses
core's trusted scope with the original session, derives identity only from the request context,
and manages the caller's unrevoked USER-account keys through normal table actions. Add key
creates an account on first use; Revoke sets `revoked_at`; PATCH edits labels and expiry.
Optional `publicHostName` advertises the hostname; port/fingerprint use the latest heartbeat.
The k8s example is in `docs/k8s/`.
Tests run a strict UI API server, H2 from the adopter DDL and real MINA clients. NoDiskExtension
snapshots the working tree (excluding target) and isolated test home, and asserts the test temp
@@ -78,7 +95,7 @@ directory is empty. The host supplies RDBMS; `qqq-backend-module-rdbms` must rem
## Dependencies
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-SNAPSHOT` until 4.1.0-RC.6.
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.7` (first release with QRun-IO/qqq#1170); never a SNAPSHOT.
Only allow-listed qbit API artifacts (`qbit-object-storage` `api` package) may be depended on.
Notifications go through core `QAlerts.notify`, never qbit-alerts classes.
+37 -6
View File
@@ -4,8 +4,30 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
## [Unreleased]
### Fixed
- Upload-job status transitions no longer use `status` as the table's record-version field.
QQQ 4.1.0-RC.26 advances a configured version field on every update (a STRING version
becomes a random UUID), which overwrote upload status. The guarded transition now re-reads
status inside the existing cross-pod account-lock transaction. The upload-job table no longer
declares a record-version field; no schema or public API change.
- Endpoint pickers for saved views, saved reports and saved bulk-load profiles are dropped when
the host has not registered those possible-value sources, so hosts without them validate.
### Added
- File transfer Preferences with connect-permission visibility, read-only connection details,
optional public hostname, self-service Add key/Revoke actions and editable key labels/expiry.
- First-key USER account creation with configured defaults and unique usernames; transactional
ownership checks, friendly duplicate-key validation, core/HTTP tests and real SFTP acceptance.
- Phase (c) generated saved-view and saved-report downloads through the user-scoped UI API,
CSV/XLSX/JSON account formats, a bounded rolling read window and HTTP stream cleanup.
- Generated-file and UTC daily-download quotas, persisted byte charges under account locks,
per-file VIEW/REPORT transfer auditing and actionable SFTP API/limit errors.
- Real MINA download tests for filters/column order, POI XLSX validation, JSON, FieldSecurityLock
DENY parity, report rendering, hidden tables/storage, large streaming exports and quotas.
- Phase (b) bounded streaming uploads through the object-storage system API, SHA-256, concurrent
byte quotas, temporary-name promotion, dedupe and transfer auditing.
- Bounded upload workers with guarded claims, fresh user sessions, scanner/quarantine hooks,
@@ -29,12 +51,21 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
- Isolated consumer/outbox API contract and complete configuration for subsequent phases.
- Real SFTP/strict UI API acceptance tests, no-disk guard, schema round trips and adoption docs.
- Endpoints (aliases) with path validation and ENDPOINTS_ONLY chroot; BULK_LOAD, INBOX, VIEW,
REPORT and OUTBOX kinds.
- The outbox provider (`FileGatewayOutbox.put`), outbox files, pickup tracking and `onDownloaded`.
- Optional per-account PGP decryption of uploads and encryption of downloads.
- The typed app home, `file_gateway_server` heartbeats, Unlock/Disable/Revoke/End session record
actions, the daily retention process and `FileGatewayIntegrations`.
- Kubernetes example manifests in `docs/k8s/`.
### Compatibility
- Local invoker integration builds use `4.1.0-FG-SNAPSHOT`; CI retains the POM's
`4.1.0-SNAPSHOT` default. RDBMS remains a host-provided runtime dependency.
- Builds on QQQ `4.1.0-RC.7` (the first release with QRun-IO/qqq#1170). RDBMS remains a
host-provided runtime dependency.
- Alert type `filegateway.lockout` uses the dotted syntax required by core validation.
- Saved-view/report downloads, endpoint routing/chroot, outbox delivery and app home remain later
phases. Object-storage RC.2 controls S3 part size through its connection configuration.
- With the private core build, the file-mapping request resends its file to select multipart
encoding; URL-encoded process values are not accepted by the v1 route.
- Object-storage RC.2 controls S3 part size through its connection configuration.
- Core #1170 sends multipart process values without re-uploading the original file.
- FieldSecurityLock requires core permission enforcement enabled. Saved-report rendering still
uses a temporary file inside core; fully disk-free report generation needs a core change.
The report folder is hidden without the render process's configured storage table.
+29 -12
View File
@@ -1,4 +1,4 @@
# CLAUDE.md
# AGENTS.md
Guidance for coding agents working in this repository.
@@ -14,13 +14,12 @@ code and QInstance. Design: `docs/DESIGN.md`.
## Build commands
```bash
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT compile
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT test
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
./mvnw compile
./mvnw test
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
```
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd` and quote the `-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT` argument in PowerShell.
Keep the POM default at `4.1.0-SNAPSHOT` for CI; the private version prevents local artifact replacement.
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
## Architecture
@@ -51,12 +50,30 @@ Claims compare expected status under an account-row transaction; core's optimist
alone does not add a SQL WHERE guard. Fresh worker user sessions outlive SSH connections and are
revoked after delivery. Bulk process multipart field: `theFile`; mapping frontend step: `fileMapping`
(resumes `receiveFileMapping`). RC.2 object storage owns S3 part size at connection level. The
private invoker selects multipart only when a file part is present, so the mapping request
restreams the file with its values. See README adoption prerequisites and compatibility notes.
private invoker supports multipart process values without a file part, so the mapping request
sends only its values. See README adoption prerequisites and compatibility notes.
Saved-view/report downloads, endpoint routing/chroot, outbox delivery, Quick Search/ESB,
preferences, retention and the app home belong to later phases. The core-valid alert type is `filegateway.lockout`; the design's
underscore-only name is rejected by core validation.
Phase (c) adds `GeneratedDownloads` (user-scoped export/render/download API flow and durable
account-locked download quotas) and `GeneratedReadChannel` (fixed rolling window, no spool).
Saved-view JSON uses `queryFilter.orderBys` and `queryColumns.columns` visibility/order. Generated
stat size is zero; reads fill requests until real EOF to avoid MINA's short-read EOF inference.
Reports list only with a configured render-process storage table. FieldSecurityLock DENY tests
require `enforcePermissionsInCoreActions=true` and match direct same-user UI export bytes.
Core `RenderSavedReportExecuteStep` stages a rendered report in a JVM temp file and deletes it
in try-with-resources; that is the bounded temp area the owner rule allows (in-memory `/tmp`).
The core-valid alert type is `filegateway.lockout` (dotted names only).
Phase (e) adds endpoints and chroot (`AliasCustomizer` validates paths), `outbox/` (the
`FileGatewayOutboxProvider`, registered in every deployment), `pgp/` (BouncyCastle stream
decrypt/encrypt, keys in memory), `maintenance/` (daily retention process and record-action
effects), `FileGatewayHome` (typed STATISTICS and TABLE widgets only; guardrail G5),
`FileGatewayIntegrations` (ESB, Quick Search, API lists) and `file_gateway_server` heartbeats.
The `file-transfer` Preferences section requires only `file_gateway.connect`. Its provider uses
core's trusted scope with the original session, derives identity only from the request context,
and manages the caller's unrevoked USER-account keys through normal table actions. Add key
creates an account on first use; Revoke sets `revoked_at`; PATCH edits labels and expiry.
Optional `publicHostName` advertises the hostname; port/fingerprint use the latest heartbeat.
The k8s example is in `docs/k8s/`.
Tests run a strict UI API server, H2 from the adopter DDL and real MINA clients. NoDiskExtension
snapshots the working tree (excluding target) and isolated test home, and asserts the test temp
@@ -78,7 +95,7 @@ directory is empty. The host supplies RDBMS; `qqq-backend-module-rdbms` must rem
## Dependencies
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-SNAPSHOT` until 4.1.0-RC.6.
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.7` (first release with QRun-IO/qqq#1170); never a SNAPSHOT.
Only allow-listed qbit API artifacts (`qbit-object-storage` `api` package) may be depended on.
Notifications go through core `QAlerts.notify`, never qbit-alerts classes.
+133 -13
View File
@@ -4,11 +4,11 @@ An SSH-key-only SFTP server that presents a QQQ application's permitted apps and
virtual folder tree. Application metadata and saved objects are read through the spec-defined
`/qqq/v1` UI API as the signed-in user, using `UiApiInvoker` on channel `sftp`.
Phases (a) and (b) implement authentication, listing/stat, account administration, uploads into
saved bulk-load profiles, background consumer delivery, and readable sidecars/results/archive.
Saved-view/report downloads, endpoint routing/chroot, outbox delivery, preferences and the app
home follow in later phases. No native filesystem, host-key file, home directory or spool is
used by the gateway. See [the approved design](docs/DESIGN.md).
Phases (a) through (c) implement authentication, listing/stat, account administration, uploads into
saved bulk-load profiles, background consumer delivery, readable sidecars/results/archive and
saved-view/report downloads. Phase (e) adds endpoint routing/chroot, outbox delivery, PGP,
Preferences, retention and the app home. No native filesystem, host-key file, home directory
or spool is used by the gateway. See [the approved design](docs/DESIGN.md).
## Requirements
@@ -17,8 +17,8 @@ backend, qbit-object-storage 0.1.0-RC.2, and core saved bulk-load profile, saved
report metadata. The gateway's
RDBMS module dependency is test-scoped. Maven 3.9.11 is pinned by the wrapper.
Until the invoker core PR merges, local builds use the private `4.1.0-FG-SNAPSHOT` artifacts.
The POM default remains `4.1.0-SNAPSHOT` for CI, to be replaced by 4.1.0-RC.6 when released.
QQQ is pinned to `4.1.0-RC.7`, the first release with the UiApiInvoker in-process sessions,
streams and multipart (QRun-IO/qqq#1170).
## Install
@@ -36,6 +36,10 @@ Apply [docs/ddl/file_gateway.sql](docs/ddl/file_gateway.sql) to the host's datab
H2 and PostgreSQL and stores UTC timestamps without a zone, matching QQQ's RDBMS binding.
The gateway does not create or migrate tables at runtime.
Register core's saved views, saved reports and saved bulk-load profiles (if the app uses them)
before the gateway producer: endpoint pickers for a source the app hasn't registered become plain
id fields, so the instance still validates.
Register metadata while building the QInstance:
```java
@@ -68,6 +72,25 @@ least 3072 bits. Public-key validation runs on insert and update and computes it
fingerprint, type and size. The host authentication module must assign the user's permissions
when `createAutomatedSessionForUser` is called, including `file_gateway.connect`.
## Preferences
Users with `file_gateway.connect` manage their own SSH keys in **Preferences → File transfer**.
They need no File Gateway admin or table permissions. **Add key** accepts an OpenSSH public key,
a required label and optional expiry. The first successful key creates a USER account with
FULL_TREE access, the configured account defaults and a unique login derived from the user ID.
The section displays that login, plus the port and host key fingerprint from the latest gateway
heartbeat. Set `.withPublicHostName("sftp.example.com")` on the shared config to display the
public hostname; it defaults to null and is omitted when unset. The heartbeat port is the gateway
listener port; deployments publishing a different external port must communicate that mapping.
The keys list contains only the user's unrevoked keys. Users can edit labels and expiry dates,
or choose **Revoke**. Revoked keys cannot sign in; existing sessions close at the next revocation
poll (normally within 60 seconds). Fingerprints, key types and last-used times are read-only.
Ed25519, ECDSA and RSA of at least 3072 bits are supported. Duplicate keys are rejected with a
validation message. Normal table customizers and configured audits apply, attributed to the
signed-in user and client channel. Administrators manage service accounts and other users' keys
through the standard account and SSH-key tables.
## Tables and permissions
All seven tables are registered, with camelCase QQQ fields mapped to snake_case database columns.
@@ -109,12 +132,12 @@ The root lists visible apps; apps contain child apps and readable tables. Table
- `upload/<profile-slug>/` when the user has insert permission, a visible profile and a bulk-insert process;
- `views/<slug>.csv` and `.xlsx` (or account-enabled JSON);
- `reports/<slug>.<format>` when the saved-report table and render process are available;
- `reports/<slug>.<format>` when the saved-report table, render process and its configured storage table are available;
- account-scoped `results/<jobId>-<file>.results.csv` and `archive/<yyyy-mm-dd>/<jobId>-<file>`.
Hidden paths return no-such-file. Slug collisions append record IDs. Saved-object queries are
paged. Metadata and saved objects are cached per connection for 60 and 30 seconds respectively.
Saved-view/report files report size zero and remain unreadable until phase (c). Stored artifacts
Saved-view/report files report size zero (unknown); clients read until EOF. Stored artifacts
report their actual lengths and support ranged reads.
Only public-key auth is advertised. Password, keyboard-interactive, GSSAPI, host-based auth,
@@ -132,6 +155,46 @@ Lockouts raise core `QAlerts` signals. **Core compatibility:** the alert type is
`filegateway.lockout`, because the core validator rejects the design's underscore-only
`file_gateway_lockout`. Without an alert sink the core logging fallback is used.
## Generated downloads
Opening `views/<slug>.<format>` queries the saved view again as the account user and starts
`POST /table/{table}/export` through `UiApiInvoker.invokeForStream`. The saved view's
`queryFilter` (including `orderBys`) and visible `queryColumns.columns` are preserved in order;
the UI selection column and pagination are excluded. Legacy explicit `fieldNames` are also
supported. CSV includes a header; CSV, XLSX and JSON are offered only when enabled in the
account's `formats` (default `csv,xlsx`).
Opening `reports/<slug>.<format>` selects that saved report through the `renderSavedReport`
record action, submits `reportFormat`, polls asynchronous status and streams the returned
`/download/{file}` storage reference. The process's `storageTableName` input default must name
an existing storage-capable table, backed by object storage in production. Without that table,
`reports/` is hidden. Reports requiring additional interactive values fail with the API's error.
Each generated handle retains at most `downloadReadAheadMb` MiB (default 16). Sequential reads
and reordered reads within this rolling window work; a seek outside it fails with a message
to download sequentially. The HTTP stream closes with the SFTP handle. Read requests are filled
until EOF so MINA does not mistake a short HTTP packet for the end of an unknown-size file.
`maxGeneratedFileMb` (default 1024) and `max_download_mb_per_day` are enforced as generated
bytes enter the window. The daily total queries DOWNLOAD transfer rows for this account since
UTC midnight, including failed and in-progress transfers; updates hold the account row lock so
concurrent generated downloads share the cap. A rejected chunk is not exposed. Re-reading cached
bytes does not charge them twice. One DOWNLOAD row per handle records the view/report ID, format,
bytes, duration and outcome: OK, LIMIT, ERROR or DENIED for an HTTP 403. API error text reaches
the SFTP client.
**Field security:** configure `QInstance.enforcePermissionsInCoreActions=true` when using this
private core build's `FieldSecurityLock` enforcement. Acceptance tests use a DENY lock and compare
the generated CSV byte-for-byte with a direct UI export by the same user: default columns omit
the denied field. Explicitly selecting a denied field fails as it does through the UI API.
**Core filesystem limitation:** the current private core's `RenderSavedReportExecuteStep`
creates and deletes a temporary file before publishing to storage. The gateway does not spool,
and the existing no-disk snapshot/cleanup guard passes, but that guard cannot detect transient
files. Fully disk-free saved-report generation remains blocked on replacing core's temporary
file with storage streaming. CSV/XLSX/JSON table exports use core's streaming exporters; the
50,000-row CSV acceptance test consumes the result with a fixed buffer.
## Uploads and object storage
Configure an object-storage bucket alias matching `config.fileGatewayBucketAlias` (default
@@ -203,6 +266,7 @@ present without enabling those features.
| Options | Defaults |
|---|---|
| `backendName`, `sftpEnabled`, `sftpPort` | `rdbms`, `false`, `2222` |
| `publicHostName` | null (omit host from Preferences) |
| `hostKeyEnvVarName`, `allowGeneratedHostKey` | `FILE_GATEWAY_SSH_HOST_KEY`, `false` |
| `lockoutThreshold`, `lockoutWindow`, `lockoutDuration` | 5, 15 minutes, 30 minutes |
| `maxConnectionsPerIpPerMinute` | 30 |
@@ -223,19 +287,75 @@ The isolated `com.kingsrook.qbits.filegateway.api` package contains `FileGateway
`InboundFile` (including remote address and a fresh stream supplier), `FileAcceptance`,
`OutboxFileRef`, and the consumer metadata registry. `FileGatewayOutbox` delegates to a registered
`FileGatewayOutboxProvider`; without one it reports unavailable and throws
`FileGatewayUnavailableException`. The consumer pipeline is active in phase (b); endpoint binding
and the outbox provider remain phase (e).
`FileGatewayUnavailableException`. INBOX endpoints bind a consumer by `consumer_name`; the built-in
`bulk-load` consumer serves `upload/<profile>/` folders and BULK_LOAD endpoints. The outbox
provider is registered by the producer in every deployment (it needs only the database and object
storage), so other qbits can call `FileGatewayOutbox.put(...)` from web pods too.
## Endpoints and chroot
`file_gateway_alias` rows ("Endpoints") mount partner-friendly paths such as `/acme/orders-in`:
| Kind | Behaviour |
|---|---|
| `BULK_LOAD` | upload folder that bulk-loads with the endpoint's saved profile |
| `INBOX` | upload folder whose files go to the `consumer_name` consumer |
| `VIEW`, `REPORT` | one generated file (`file_name`, or `<slug>.<format>`) |
| `OUTBOX` | files put by `FileGatewayOutbox`; partners download and delete them |
Paths are absolute, `[a-z0-9._/-]`, unique, and never collide with app names. An endpoint is
listed only when it is active, bound to the account (or shared), and its target is reachable by
the user through `/qqq/v1`; endpoints grant nothing. `root_mode` FULL_TREE (default for users)
shows the app tree plus endpoints; ENDPOINTS_ONLY (default for service accounts) is the chroot.
## Outbox
`put` streams to `file-gateway/outbox/{aliasId}/{uuid}/{fileName}` with a SHA-256 and a
`file_gateway_outbox_file` row (READY). The first complete download marks it DOWNLOADED and calls
`onDownloaded` on the endpoint's consumer (at-least-once); `rm` marks it REMOVED; retention marks
old files EXPIRED and deletes the object.
## PGP (optional, per account)
`pgp_decrypt_uploads`: uploads named `*.pgp`/`*.gpg` are decrypted stream to stream into a new
object before the consumer runs, with the key and passphrase from the env vars named by
`pgpPrivateKeyEnvVarName`/`pgpPassphraseEnvVarName` (in memory only); a bad file is REJECTED.
`pgp_encrypt_downloads`: downloads are encrypted on the fly to the account's `pgp_public_key`
(validated on save) and listed with `.pgp` appended.
## App home, actions and retention
The `file_gateway` app home uses typed widgets only: a KPI row (transfers today, failed jobs in
24 hours, active sessions, MB in and out today, locked accounts), and full-width tables for recent
failures, top partners (7 days), active sessions and gateway servers (host key fingerprint, port,
pod, last seen, from `file_gateway_server` heartbeats). Record actions (standard Review/Result,
never in the nav): Unlock account, Disable account, Revoke key, End session, Retry upload. A daily
scheduled retention process applies the `*Retention` settings and aborts stale RECEIVING uploads.
## Integrations
`FileGatewayIntegrations` lists what a host wires in without this qbit depending on those modules:
`esbTableNames()` (accounts, endpoints, upload jobs), `quickSearchTables()` (accounts by username
and label, endpoints by name and path, upload jobs by file name) and `apiTableNames()` (all tables,
with the admin's rules).
## Kubernetes
`docs/k8s/gateway-deployment.yaml` and `docs/k8s/gateway-service.yaml`: the same image as the web
Deployment with `FILE_GATEWAY_SFTP_ENABLED=true`, the host key from a Secret, a read-only root
filesystem with an in-memory `/tmp`, 2 replicas and a PDB, and a LoadBalancer Service exposing
port 22 only (no passive range: v1 is SFTP only) with `externalTrafficPolicy: Local`.
## Build and acceptance tests
```bash
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify
./mvnw verify
```
PowerShell:
```powershell
./mvnw.cmd -o '-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT' verify
./mvnw.cmd verify
```
The tests use H2 from the adopter DDL, a strict UI API server, generated in-memory SSH keys and
+44 -18
View File
@@ -165,7 +165,7 @@ exists on any disk.
| table | the table is in the app and in `metaData.tables` (read permission) |
| `upload/` | the table has the bulk-insert process (`<table>.bulkInsert`) in metaData, and at least one saved bulk-load profile is visible (`POST /table/savedBulkLoadProfile/query` filtered by `tableName`) |
| `views/` | the user can query `savedView` |
| `reports/` | the user can query `savedReport` and has the `renderSavedReport` process |
| `reports/` | the user can query `savedReport`, has the `renderSavedReport` process, and that process's `storageTableName` input default names an existing storage-capable table |
| `results/`, `archive/` | always (they hold only this account's jobs) |
- **Caching.** Metadata responses are cached in memory per SFTP session for 60 seconds. Saved-object
@@ -250,8 +250,8 @@ in-process session token; there is never a service session.
(`ProcessSpecUtilsV1.addUploadedFiles`).
### Shipping
§3.1–3.3 go in one core PR to `QRun-IO/qqq` `develop`, for 4.1.0-RC.6. Until then the qbit builds
on `4.1.0-SNAPSHOT`, with the pom marked `STAND-IN(removeBy=4.1.0-RC.6)`.
§3.1–3.3 shipped in QRun-IO/qqq#1170 (merged 2026-10-14), first released in QQQ 4.1.0-RC.7, which
the qbit pins.
### 3.4 Permissions for automated sessions (qbit-authentik)
`AuthentikGroupPermissionMapper` builds permissions from token claims, and an automated session
@@ -265,13 +265,30 @@ has no token.
- **Who needs it.** It is a qbit-authentik PR. The gateway works without it under MOCK auth and in
tests, but kof22 needs it before going live.
### 3.5 Preferences (QRun-IO/qqq#1162)
The gateway registers a "File transfer" section with the provider `FileTransferPreferencesProvider`.
### 3.5 Preferences
The gateway registers `file-transfer` ("File transfer", order 60) with
`FileTransferPreferencesProvider`, using core's user-preferences API. Only users with
`file_gateway.connect` see it; no admin-app or backing-table permissions are needed.
- **Scalar fields:** the SFTP username (read-only), "Add a public key" (TEXT) and "Key label".
- **The `keys` list:** label, fingerprint (read-only), last used (read-only), expires and revoked.
- **Behaviour.** The first key creates the user's `USER` account. Revoking a key sets `revoked_at`.
- **Until #1162 is in the baseline,** admins add keys on the account record.
- **Read-only connection details:** the account username, plus port and host key fingerprint from
the most recent `file_gateway_server` heartbeat. The optional `publicHostName` config supplies the host; when
unset, the host field is omitted. Before the first key, username is blank and the keys list
explains that adding a key creates the account.
- **The `keys` list:** only the caller's unrevoked USER-account keys. Label and expiry are editable
through PATCH; fingerprint, key type and last-used time are read-only. Public-key text is never
returned in preference values.
- **Add key (`addKey`, LIST action):** requires OpenSSH public-key text and a label, with optional
expiry. The first successful addition creates a USER account with FULL_TREE and configured
account defaults. Its login is derived from the session user ID, normalized to lowercase
`[a-z0-9._-]`, at most 64 characters; short IDs receive a `user-` prefix. Collisions receive
`-2`, `-3`, etc. Subsequent keys reuse that account. The result names the computed fingerprint.
- **Revoke (`revokeKey`, ROW action):** confirms revocation, sets `revoked_at`, and removes the
key from the list. New logins fail immediately; existing sessions end on the next gateway poll.
- **Persistence:** ordinary table actions retain `SshKeyCustomizer` validation and configured
audits under the original user/client channel. Weak and duplicate keys return actionable 400
messages. Failed first-key inserts roll back account creation. Mutations lock and recheck
ownership; another user's key is not a setting here. Identity comes only from core's request
context, never from the request body. Service-account keys remain administrator-managed.
## 4. Upload
1. **Receive.** The SFTP write streams straight to object storage:
@@ -333,6 +350,11 @@ The gateway registers a "File transfer" section with the provider `FileTransferP
## 5. Download
Everything is streamed. Nothing is written to disk.
The gateway itself streams without disk. Core's `RenderSavedReportExecuteStep` stages the
rendered report in a JVM temp file and deletes it in try-with-resources before the process step
returns. That is the bounded temp area the owner rule allows, so the `emptyDir` temp mount must be
sized for the largest report. The no-disk test asserts the temp area is empty after each transfer.
- **Saved view** (`views/<slug>.<fmt>`):
- `POST /table/savedView/query` by table name, as the user;
- the filter, visible columns and order are parsed from `viewJson`;
@@ -436,9 +458,11 @@ Everything is streamed. Nothing is written to disk.
(§15) would add only a server and an authenticator.
## 7. Audit
- **Connections.** Every connection writes `file_gateway_connection_log`. The row is opened at TCP
accept and updated at auth and at disconnect. It records the outcome, IP, key fingerprint, SSH
client version, byte and operation counts, and channel `sftp`.
- **Connections.** Every sign-in attempt writes `file_gateway_connection_log`. The row is opened
at the first authentication attempt (or a pre-auth refusal such as the connection rate limit)
and updated at auth and at disconnect. It records the outcome, IP, key fingerprint, SSH client
version, byte and operation counts, and channel `sftp`. A TCP connection that never tries to
sign in, such as a kubelet health probe or a port scan, leaves no row.
- **Operations.** Every list, upload, download, rename and remove writes
`file_gateway_transfer_log`. It records the operation, path, object kind and id, bytes, sha256,
duration, outcome and message, with channel `sftp/<sessionId>`.
@@ -460,6 +484,7 @@ the table names.
| `file_gateway_outbox_file` (child of endpoint) | files waiting for pickup | `alias_id`, `file_name`, `size_bytes`, `sha256`, `storage_key`, `status` (READY, DOWNLOADED, REMOVED, EXPIRED), `first_downloaded_at`, `download_count`, `removed_at`, `expires_at`, `created_by_consumer` |
| `file_gateway_transfer_log` (main, "Transfers") | every operation | `account_id`, `connection_log_id`, `operation` (LIST, UPLOAD, DOWNLOAD, RENAME, REMOVE), `path`, `object_kind` (APP, TABLE, PROFILE, VIEW, REPORT, RESULT, ARCHIVE, ENDPOINT, OUTBOX), `object_id`, `format`, `bytes`, `sha256`, `duration_ms`, `outcome` (OK, DENIED, NOT_FOUND, LIMIT, ERROR), `message`, `client_channel`, `upload_job_id` |
| `file_gateway_connection_log` (main, "Connections") | every connection | `account_id`, `username`, `remote_ip`, `client_version`, `key_fingerprint`, `outcome` (CONNECTED, AUTH_FAILED, LOCKED_OUT, IP_DENIED, EXPIRED, DISABLED, LIMIT), `session_id`, `pod_name`, `connected_at`, `last_seen_at`, `disconnected_at`, `disconnect_reason`, `bytes_in`, `bytes_out`, `operation_count`, `client_channel` |
| `file_gateway_server` (system, not in nav) | gateway pod heartbeats for the app home | `pod_name` (unique), `host_key_fingerprint`, `port`, `started_at`, `last_seen_at`; rows unseen for a day are deleted by retention |
- **Child lists:**
- an account shows its keys, upload jobs and connections;
@@ -563,6 +588,7 @@ FileGatewayConsumersMetaData.ofOrWithNew(qInstance).withConsumer(new QCodeRefere
- `sftpEnabled` (default false; true only in the gateway deployment, from
`FILE_GATEWAY_SFTP_ENABLED`);
- `sftpPort` (2222);
- `publicHostName` (null; optional public hostname shown in Preferences);
- `hostKeyEnvVarName` (`FILE_GATEWAY_SSH_HOST_KEY`);
- `allowGeneratedHostKey` (false).
- **Storage and buffers:**
@@ -593,8 +619,8 @@ The gateway is a separate, stateless Deployment of the same image. An example is
- `FILE_GATEWAY_SSH_HOST_KEY` from a Secret; in kof22, through ESO from Infisical.
- 2 replicas, with a PodDisruptionBudget of `minAvailable: 1`.
- `readOnlyRootFilesystem: true`. The only writable mount is a small in-memory `emptyDir`
(`medium: Memory`, `sizeLimit: 64Mi`) for the JVM temp directory. Nothing is persisted.
- Requests of 250m CPU and 768Mi memory; a 1.5Gi memory limit. The in-memory buffers are bounded
(`medium: Memory`, `sizeLimit: 512Mi`, counted in the memory limit) for the JVM temp directory, sized for the largest saved report core stages there. Nothing is persisted.
- Requests of 250m CPU and 768Mi memory; a 2Gi memory limit. The in-memory buffers are bounded
by the config.
- **Ports and probes.**
- Container ports: 2222 (sftp) and the app's HTTP port, which serves loopback and probes only.
@@ -602,7 +628,7 @@ The gateway is a separate, stateless Deployment of the same image. An example is
- Liveness: HTTP health.
- Startup: allows 2 minutes for the QInstance.
- **Service.** `type: LoadBalancer`, with these settings:
- the annotation `lbipam.cilium.io/ips: 10.100.22.65` in kof22 (owner, 2026-10-13; .60, .61 and .64 are taken);
- the annotation `lbipam.cilium.io/ips` (or the older `io.cilium/lb-ipam-ips`, which k8s-galaxy uses): 10.100.22.65 in kof22 (owner, 2026-10-13; .60, .61 and .64 are taken);
- **port 22 → 2222 only (no passive range in v1)**;
- `externalTrafficPolicy: Local`, so the pod sees the real client IP for allowlists and
lockouts.
@@ -672,10 +698,10 @@ All tests run in the qbit's build.
`Class.forName` of another qbit. The gateway's own `api` package is the published contract.
- **G2.** Server tests run in strict mode.
- **G3.** No extension is registered: the gateway is a UI API client.
- **G4.** Stand-ins are marked `STAND-IN(core=..., removeBy=...)`: the 4.1.0-SNAPSHOT baseline
until RC.6, and the preferences section until #1162.
- **G4.** No stand-ins remain: the core baseline is a release, and the preferences section uses
core's API from QRun-IO/qqq#1162.
- **G5.** No HTML widgets.
- **G7.** Pin 4.1.0-RC.6 as soon as it is cut. Until then, develop builds on the SNAPSHOT.
- **G7.** Pinned to QQQ 4.1.0-RC.7; never a SNAPSHOT.
## 14. Phases
- **(a)** The SFTP server, the virtual filesystem, and auth:
+27
View File
@@ -93,6 +93,9 @@ CREATE TABLE file_gateway_upload_job (
result_summary TEXT,
results_storage_key VARCHAR(1024),
results_size_bytes BIGINT,
plaintext_storage_key VARCHAR(1024),
plaintext_sha256 VARCHAR(1024),
plaintext_size_bytes BIGINT,
error_message TEXT,
attempts INTEGER,
next_attempt_at TIMESTAMP,
@@ -116,6 +119,7 @@ CREATE TABLE file_gateway_outbox_file (
status VARCHAR(1024),
first_downloaded_at TIMESTAMP,
download_count INTEGER,
on_downloaded_pending BOOLEAN DEFAULT FALSE,
removed_at TIMESTAMP,
expires_at TIMESTAMP,
created_by_consumer VARCHAR(1024)
@@ -156,9 +160,32 @@ CREATE TABLE file_gateway_connection_log (
connected_at TIMESTAMP,
last_seen_at TIMESTAMP,
disconnected_at TIMESTAMP,
end_requested_at TIMESTAMP,
disconnect_reason TEXT,
bytes_in BIGINT,
bytes_out BIGINT,
operation_count BIGINT,
client_channel VARCHAR(1024)
);
CREATE TABLE file_gateway_server (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
create_date TIMESTAMP,
modify_date TIMESTAMP,
pod_name VARCHAR(1024) NOT NULL UNIQUE,
host_key_fingerprint VARCHAR(1024),
port INTEGER,
started_at TIMESTAMP,
last_seen_at TIMESTAMP
);
-- Indexes for the job queue, dedupe, live sessions, home queries and retention.
CREATE INDEX file_gateway_transfer_log_account_date ON file_gateway_transfer_log (account_id, create_date);
CREATE INDEX file_gateway_transfer_log_date ON file_gateway_transfer_log (create_date);
CREATE INDEX file_gateway_connection_log_account_live ON file_gateway_connection_log (account_id, disconnected_at);
CREATE INDEX file_gateway_connection_log_date ON file_gateway_connection_log (create_date);
CREATE INDEX file_gateway_upload_job_status_next ON file_gateway_upload_job (status, next_attempt_at);
CREATE INDEX file_gateway_upload_job_dedupe ON file_gateway_upload_job (account_id, sha256);
CREATE INDEX file_gateway_upload_job_date ON file_gateway_upload_job (create_date);
CREATE INDEX file_gateway_outbox_file_alias_status ON file_gateway_outbox_file (alias_id, status);
CREATE INDEX file_gateway_ssh_key_account ON file_gateway_ssh_key (account_id);
+91
View File
@@ -0,0 +1,91 @@
# qbit-file-gateway: the SFTP gateway Deployment (docs/DESIGN.md section 11).
# Same image and QInstance as the application's web Deployment; only FILE_GATEWAY_SFTP_ENABLED differs.
# Stateless: no volumes except an in-memory JVM temp area. State lives in the database and object storage.
# Replace every <PLACEHOLDER> and copy the application's usual environment (database, object storage, auth).
apiVersion: apps/v1
kind: Deployment
metadata:
name: <APP>-file-gateway
labels:
app.kubernetes.io/name: <APP>-file-gateway
app.kubernetes.io/component: sftp-gateway
spec:
replicas: 2
selector:
matchLabels:
app.kubernetes.io/name: <APP>-file-gateway
template:
metadata:
labels:
app.kubernetes.io/name: <APP>-file-gateway
app.kubernetes.io/component: sftp-gateway
spec:
terminationGracePeriodSeconds: 75 # SIGTERM: stop accepting, drain sessions up to 60 s, requeue claimed jobs
securityContext:
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
containers:
- name: app
image: <APP_IMAGE> # the same image as the web Deployment
env:
- name: FILE_GATEWAY_SFTP_ENABLED
value: "true"
- name: FILE_GATEWAY_SSH_HOST_KEY # OpenSSH private key text; parsed in memory, never written to disk
valueFrom:
secretKeyRef:
name: file-gateway-host-key
key: ssh_host_key
- name: JAVA_TOOL_OPTIONS
value: "-Djava.io.tmpdir=/tmp"
# ... the application's usual environment (database, object storage, authentication) ...
ports:
- name: sftp
containerPort: 2222
- name: http # loopback /qqq/v1 calls and kubelet probes only; not in the Service
containerPort: 8000
readinessProbe:
tcpSocket:
port: sftp
periodSeconds: 10
livenessProbe:
httpGet:
path: <APP_HEALTH_PATH>
port: http
periodSeconds: 20
failureThreshold: 3
startupProbe:
httpGet:
path: <APP_HEALTH_PATH>
port: http
periodSeconds: 5
failureThreshold: 24 # up to 2 minutes for the QInstance
resources:
requests:
cpu: 250m
memory: 768Mi
limits:
memory: 2Gi # includes the in-memory /tmp below and the bounded transfer buffers
securityContext:
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
volumeMounts:
- name: tmp
mountPath: /tmp
volumes:
- name: tmp # JVM temp only (core stages a rendered report here and deletes it)
emptyDir:
medium: Memory
sizeLimit: 512Mi
---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
name: <APP>-file-gateway
spec:
minAvailable: 1
selector:
matchLabels:
app.kubernetes.io/name: <APP>-file-gateway
+20
View File
@@ -0,0 +1,20 @@
# qbit-file-gateway: the SFTP LoadBalancer Service (docs/DESIGN.md section 11).
# One port only: v1 is SFTP only, so there is no FTPS passive port range (see DESIGN.md section 15).
apiVersion: v1
kind: Service
metadata:
name: <APP>-file-gateway
annotations:
# Cilium LB-IPAM: a fixed address (kof22 on k8s-galaxy: 10.100.22.65, owner decision 2026-10-13).
# Cilium 1.15+ reads lbipam.cilium.io/ips; older clusters (and k8s-galaxy manifests) use io.cilium/lb-ipam-ips.
lbipam.cilium.io/ips: "10.100.22.65"
spec:
type: LoadBalancer
externalTrafficPolicy: Local # keep the client IP: IP allowlists and lockouts need it
selector:
app.kubernetes.io/name: <APP>-file-gateway
ports:
- name: sftp
port: 22
targetPort: sftp
protocol: TCP
+11 -5
View File
@@ -22,6 +22,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
<revision>0.1.0-SNAPSHOT</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
@@ -33,7 +34,7 @@
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.0.0</version>
<version>2.2.1</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -73,10 +74,15 @@
</scm>
<dependencies>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcpg-jdk18on</artifactId>
<version>1.84</version>
</dependency>
<dependency>
<groupId>com.kingsrook.qbits</groupId>
<artifactId>qbit-object-storage</artifactId>
<version>0.1.0-RC.2</version>
<version>0.1.0-RC.10</version>
</dependency>
<!-- MINA's Ed25519 key decoder and signer. -->
<dependency>
@@ -112,7 +118,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-frontend-next</artifactId>
<version>1.0.0-RC.14</version>
<version>1.0.0-RC.38</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -233,8 +239,8 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<!-- STAND-IN(core=QRun-IO/qqq develop after v4.1.0-RC.5, removeBy=4.1.0-RC.6): UiApiInvoker client channels and QAlerts.notify are on develop only; pin the RC when it is cut -->
<qqq.snapshot.version>4.1.0-SNAPSHOT</qqq.snapshot.version>
<!-- QQQ 4.1.0-RC.7: the first released core with the UiApiInvoker in-process sessions, streams and multipart (QRun-IO/qqq#1170). Never a SNAPSHOT (guardrail G7). -->
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -0,0 +1,94 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
/*******************************************************************************
** What a host wires into its ESB, Quick Search and API, without this qbit
** depending on those modules (the same opt-in hooks as qbit-sql-workbench).
**
** <ul>
** <li>{@link #esbTableNames()}: publish these tables' inserts, updates and
** deletes to an ESB topic (the main tables).</li>
** <li>{@link #quickSearchTables()}: index these in Quick Search, with field
** weights.</li>
** <li>{@link #apiTableNames()}: expose these in the host's QQQ API. The API
** gets exactly the admin's rules: the same permissions, customizers and
** read-only tables.</li>
** </ul>
*******************************************************************************/
public final class FileGatewayIntegrations
{
/*******************************************************************************
** A table for Quick Search: field weights, the timestamp for incremental
** indexing, and the record label.
*******************************************************************************/
public record SearchableTable(String tableName, Map<String, Integer> fieldWeights, String basepullTimestamp, String recordLabelFormat, List<String> recordLabelFields)
{
}
/*******************************************************************************
** Utility class.
*******************************************************************************/
private FileGatewayIntegrations()
{
}
/*******************************************************************************
** Tables to publish on the ESB.
*******************************************************************************/
public static List<String> esbTableNames()
{
return (List.of(FileGatewayTables.ACCOUNT, FileGatewayTables.ALIAS, FileGatewayTables.UPLOAD_JOB));
}
/*******************************************************************************
** Tables for the host's QQQ API.
*******************************************************************************/
public static List<String> apiTableNames()
{
return (FileGatewayTables.ALL);
}
/*******************************************************************************
** Tables for Quick Search.
*******************************************************************************/
public static List<SearchableTable> quickSearchTables()
{
return (List.of(
new SearchableTable(FileGatewayTables.ACCOUNT, Map.of("username", 10, "label", 6), "modifyDate", "%s", List.of("username")),
new SearchableTable(FileGatewayTables.ALIAS, Map.of("name", 10, "path", 6), "modifyDate", "%s", List.of("name")),
new SearchableTable(FileGatewayTables.UPLOAD_JOB, Map.of("fileName", 10), "modifyDate", "%s", List.of("fileName"))));
}
}
@@ -34,9 +34,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
*******************************************************************************/
public class FileGatewayQBitConfig implements QBitConfig
{
private String schedulerName = null;
private Duration serverRetention = Duration.ofDays(1);
private String backendName = "rdbms";
private boolean sftpEnabled = false;
private int sftpPort = 2222;
private String publicHostName = null;
private String hostKeyEnvVarName = "FILE_GATEWAY_SSH_HOST_KEY";
private boolean allowGeneratedHostKey = false;
private String fileGatewayBucketAlias = "file-gateway";
@@ -121,6 +124,27 @@ public class FileGatewayQBitConfig implements QBitConfig
/*******************************************************************************
** Public SFTP hostname advertised in Preferences, when configured.
*******************************************************************************/
public String getPublicHostName()
{
return (publicHostName);
}
/*******************************************************************************
** Set the public SFTP hostname; null omits it from Preferences.
*******************************************************************************/
public FileGatewayQBitConfig withPublicHostName(String publicHostName)
{
this.publicHostName = publicHostName;
return (this);
}
/*******************************************************************************
** Fluent setter for sftpPort.
*******************************************************************************/
@@ -738,4 +762,49 @@ public class FileGatewayQBitConfig implements QBitConfig
this.tableMetaDataCustomizer = tableMetaDataCustomizer;
return (this);
}
/*******************************************************************************
** Getter for schedulerName.
*******************************************************************************/
public String getSchedulerName()
{
return (schedulerName);
}
/*******************************************************************************
** Fluent setter for schedulerName.
*******************************************************************************/
public FileGatewayQBitConfig withSchedulerName(String schedulerName)
{
this.schedulerName = schedulerName;
return (this);
}
/*******************************************************************************
** Getter for serverRetention.
*******************************************************************************/
public Duration getServerRetention()
{
return (serverRetention);
}
/*******************************************************************************
** Fluent setter for serverRetention.
*******************************************************************************/
public FileGatewayQBitConfig withServerRetention(Duration serverRetention)
{
this.serverRetention = serverRetention;
return (this);
}
}
@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
@@ -43,6 +44,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
*******************************************************************************/
public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGatewayQBitConfig>
{
/** Core possible-value sources a host may or may not register. */
static final java.util.Set<String> OPTIONAL_HOST_SOURCES = java.util.Set.of("savedView", "savedReport", "savedBulkLoadProfile");
/*******************************************************************************
** The sftp wall owns its list (core's qbit prefixes are transitional and the
** assistant may now read audits): partners never reach identity, audits,
@@ -149,6 +153,7 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
}
table.setSourceQBitName(qBit.getName());
dropMissingPickers(qInstance, table);
output.add(table);
if(FileGatewayTables.MAIN.contains(table.getName()))
{
@@ -161,12 +166,17 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
.withTables(List.of(FileGatewayTables.UPLOAD_JOB, FileGatewayTables.TRANSFER_LOG, FileGatewayTables.CONNECTION_LOG)));
FileGatewayTables.joins().forEach(output::add);
FileGatewayTables.childWidgets().forEach(output::add);
var home = com.kingsrook.qbits.filegateway.metadata.FileGatewayHome.widgets();
home.forEach(output::add);
app.withWidgets(home.stream().map(widget -> widget.getName()).toList());
output.add(app);
FileGatewayProcesses.processes().forEach(output::add);
output.add(com.kingsrook.qbits.filegateway.maintenance.GatewayMaintenance.metadata(qBitConfig));
output.add(new QAlertTypeMetaData().withName("filegateway.lockout").withLabel("File Gateway Lockout")
.withSourceQBitName(qBit.getName()).withSubjectTableName(FileGatewayTables.ACCOUNT)
.withVisibilityPermission(FileGatewayPermissions.VIEW));
output.addSelfToInstance(qInstance);
com.kingsrook.qbits.filegateway.preferences.FileTransferPreferencesProvider.register(qInstance, qBitConfig);
ClientChannelsMetaData channels = ClientChannelsMetaData.ofOrWithNew(qInstance)
.withChannel("sftp", "SFTP file gateway").withDeniedPattern("sftp", "file_gateway_*")
.withDeniedPattern("assistant", "file_gateway_*");
@@ -178,6 +188,29 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
{
channels.withDeniedPattern("sftp", pattern);
}
com.kingsrook.qbits.filegateway.api.FileGatewayOutbox.setProvider(new com.kingsrook.qbits.filegateway.outbox.OutboxService(qInstance,
new com.kingsrook.qbits.filegateway.storage.ObjectStorageGatewayStore(qInstance, qBitConfig.getFileGatewayBucketAlias()), qBitConfig, java.time.Clock.systemUTC()));
return (new MetaDataProducerMultiOutput());
}
/*******************************************************************************
** Saved views, saved reports and saved bulk-load profiles are optional in a
** host. A picker whose possible-value source the host has not registered (yet)
** stays a plain id field, so the instance validates; register those core
** metadata providers before this producer to get the pickers.
*******************************************************************************/
static void dropMissingPickers(QInstance qInstance, QTableMetaData table)
{
for(QFieldMetaData field : table.getFields().values())
{
String source = field.getPossibleValueSourceName();
if(source != null && OPTIONAL_HOST_SOURCES.contains(source) && qInstance.getPossibleValueSource(source) == null)
{
field.setPossibleValueSourceName(null);
field.setPossibleValueSourceFilter(null);
}
}
}
}
@@ -86,6 +86,7 @@ public final class FileGatewayRuntime implements AutoCloseable
return (null);
}
FileGatewayRuntime runtime = new FileGatewayRuntime(instance, server, config, hostKey(config, environment), Clock.systemUTC(), Duration.ofSeconds(60), environment.apply("HOSTNAME"), new ObjectStorageGatewayStore(instance, config.getFileGatewayBucketAlias()));
runtime.worker.withEnvironment(environment);
runtime.protocol.start();
runtime.worker.start(Duration.ofSeconds(60));
return (runtime);
@@ -0,0 +1,217 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.maintenance;
import java.io.IOException;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
import com.kingsrook.qbits.filegateway.FileGatewayQBitProducer;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.outbox.OutboxService;
import com.kingsrook.qbits.filegateway.sftp.AuditWriter;
import com.kingsrook.qbits.filegateway.storage.GatewayObjectStore;
import com.kingsrook.qbits.filegateway.storage.ObjectStorageGatewayStore;
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
/*******************************************************************************
** Idempotent daily cleanup and at-least-once outbox callback retries.
*******************************************************************************/
public class GatewayMaintenance implements BackendStep
{
public static final String NAME = "file_gateway_retention";
/*******************************************************************************
** Hidden host-scheduled process; no navigation or user action is registered.
*******************************************************************************/
public static QProcessMetaData metadata(FileGatewayQBitConfig config)
{
QProcessMetaData process = new QProcessMetaData().withName(NAME).withLabel("File gateway retention").withIsHidden(true)
.withPermissionRules(FileGatewayTables.permissionRules(NAME))
.withStep(new QBackendStepMetaData().withName("run").withCode(new QCodeReference(GatewayMaintenance.class)));
if(config.getSchedulerName() != null)
{
process.withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName()).withRepeatSeconds(86400).withInitialDelaySeconds(60));
}
return (process);
}
/*******************************************************************************
** Resolve registered host configuration without starting the protocol runtime.
*******************************************************************************/
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
QInstance instance = QContext.getQInstance();
FileGatewayQBitConfig config = (FileGatewayQBitConfig) instance.getQBits().get(FileGatewayQBitProducer.GROUP_ID + ":" + FileGatewayQBitProducer.ARTIFACT_ID).getConfig();
maintain(instance, config, new ObjectStorageGatewayStore(instance, config.getFileGatewayBucketAlias()), Clock.systemUTC());
}
/*******************************************************************************
** Cursor batches remain correct while rows are deleted, and bound memory use.
*******************************************************************************/
public static void maintain(QInstance instance, FileGatewayQBitConfig config, GatewayObjectStore store, Clock clock) throws QException
{
AuditWriter audit = new AuditWriter(instance);
OutboxService outbox = new OutboxService(instance, store, config, clock);
for(String table : List.of(FileGatewayTables.UPLOAD_JOB, FileGatewayTables.OUTBOX_FILE, FileGatewayTables.TRANSFER_LOG, FileGatewayTables.CONNECTION_LOG, FileGatewayTables.SERVER))
{
int last = 0;
while(true)
{
var rows = audit.query(table, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, last))
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(250));
if(rows.isEmpty())
{
break;
}
for(QRecord row : rows)
{
last = row.getValueInteger("id");
try
{
if(FileGatewayTables.UPLOAD_JOB.equals(table))
{
audit.withAccountLock("id", row.getValue("accountId"), () ->
{
try
{
cleanUpload(audit, store, audit.query(table, UploadJobs.equal("id", row.getValue("id"))).getFirst(), config, clock.instant());
}
catch(IOException e)
{
throw (new QException("Upload retention storage failure", e));
}
return (null);
});
}
else if(FileGatewayTables.OUTBOX_FILE.equals(table))
{
outbox.notifyDownloaded(row);
if("REMOVED".equals(row.getValueString("status")) || older(row, "createDate", config.getOutboxRetention(), clock.instant())
|| (row.getValueInstant("expiresAt") != null && !row.getValueInstant("expiresAt").isAfter(clock.instant())))
{
deleteObject(store, row, "storageKey");
audit.update(table, new QRecord().withValue("id", row.getValue("id")).withValue("status", "EXPIRED").withValue("storageKey", null));
}
}
else if(FileGatewayTables.SERVER.equals(table))
{
if(older(row, "lastSeenAt", config.getServerRetention(), clock.instant()))
{
audit.delete(table, row.getValue("id"));
}
}
else if(older(row, "createDate", config.getLogRetention(), clock.instant())
&& (!FileGatewayTables.CONNECTION_LOG.equals(table) || older(row, "lastSeenAt", Duration.ofMinutes(3), clock.instant())))
{
audit.delete(table, row.getValue("id"));
}
}
catch(IOException e)
{
throw (new QException("File gateway retention storage failure", e));
}
}
}
}
}
/*******************************************************************************
** Never delete an object owned by an active worker; retain rows for the audit trail.
*******************************************************************************/
private static void cleanUpload(AuditWriter audit, GatewayObjectStore store, QRecord row, FileGatewayQBitConfig config, Instant now) throws IOException, QException
{
String status = row.getValueString("status");
QRecord changes = new QRecord().withValue("id", row.getValue("id"));
if("RECEIVING".equals(status) && older(row, "createDate", config.getReceivingStaleAfter(), now))
{
deleteObject(store, row, "storageKey");
changes.withValue("status", "ABORTED").withValue("storageKey", null).withValue("finishedAt", now).withValue("errorMessage", "Incomplete upload expired");
}
else if(!List.of("RECEIVING", "RUNNING", "QUEUED").contains(status))
{
if(older(row, "createDate", config.getArchiveRetention(), now))
{
deleteObject(store, row, "storageKey");
deleteObject(store, row, "plaintextStorageKey");
changes.withValue("storageKey", null).withValue("plaintextStorageKey", null);
}
if(older(row, "createDate", config.getResultsRetention(), now))
{
deleteObject(store, row, "resultsStorageKey");
changes.withValue("resultsStorageKey", null).withValue("resultsSizeBytes", null);
}
}
if(changes.getValues().size() > 1)
{
audit.update(FileGatewayTables.UPLOAD_JOB, changes);
}
}
/*******************************************************************************
** Storage deletes precede database clearing so transient errors can be retried.
*******************************************************************************/
private static void deleteObject(GatewayObjectStore store, QRecord row, String field) throws IOException
{
if(row.getValueString(field) != null)
{
store.delete(row.getValueString(field));
}
}
/*******************************************************************************
** Null timestamps are not evidence that a record is old enough to remove.
*******************************************************************************/
private static boolean older(QRecord row, String field, Duration age, Instant now)
{
return (row.getValueInstant(field) != null && !row.getValueInstant(field).plus(age).isAfter(now));
}
}
@@ -44,6 +44,27 @@ public class AccountCustomizer implements TableCustomizerInterface
{
record.addError(new BadInputStatusMessage("Username must be 3 to 64 lowercase letters, digits, dots, underscores or hyphens"));
}
if(oldRecords.isEmpty() && record.getValue("rootMode") == null)
{
record.setValue("rootMode", "SERVICE".equals(record.getValueString("accountType")) ? "ENDPOINTS_ONLY" : "FULL_TREE");
}
String pgpKey = record.getValueString("pgpPublicKey");
if(pgpKey != null && !pgpKey.isBlank())
{
try
{
com.kingsrook.qbits.filegateway.pgp.PgpStreams.encryptionKey(pgpKey);
}
catch(Exception e)
{
record.addError(new BadInputStatusMessage("PGP public key must parse and contain a valid encryption subkey"));
}
}
if(Boolean.TRUE.equals(record.getValueBoolean("pgpEncryptDownloads")) && (pgpKey == null || pgpKey.isBlank())
&& oldRecords.orElse(List.of()).stream().noneMatch(old -> java.util.Objects.equals(old.getValue("id"), record.getValue("id")) && old.getValueString("pgpPublicKey") != null))
{
record.addError(new BadInputStatusMessage("PGP download encryption requires a public key"));
}
for(String limit : List.of("maxConcurrentSessions", "maxOperationsPerMinute", "maxFileSizeMb", "maxUploadMbPerDay", "maxDownloadMbPerDay"))
{
if(record.getValueInteger(limit) != null && record.getValueInteger(limit) < 1)
@@ -0,0 +1,104 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.metadata;
import java.util.List;
import java.util.Optional;
import com.kingsrook.qbits.filegateway.sftp.AuditWriter;
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
/*******************************************************************************
** Endpoints are canonical mounts, never a way to shadow application paths.
*******************************************************************************/
public class AliasCustomizer implements TableCustomizerInterface
{
/*******************************************************************************
** Validate the merged row so sparse updates cannot bypass mount validation.
*******************************************************************************/
@Override
public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean preview, Optional<List<QRecord>> oldRecords) throws QException
{
AuditWriter audit = new AuditWriter(QContext.getQInstance());
for(QRecord record : records)
{
QRecord merged = new QRecord();
oldRecords.orElse(List.of()).stream().filter(old -> java.util.Objects.equals(old.getValue("id"), record.getValue("id")))
.findFirst().ifPresent(old -> merged.getValues().putAll(old.getValues()));
merged.getValues().putAll(record.getValues());
String path = merged.getValueString("path");
String error = null;
if(path == null || !path.matches("/[a-z0-9._/-]+") || path.contains("..") || path.contains("//") || path.endsWith("/") || List.of(path.split("/")).contains("."))
{
error = "Endpoint path must be absolute, use lowercase letters, digits, dots, underscores, hyphens and slashes, and contain no dot segments or empty segments";
}
else
{
boolean fullTree = merged.getValue("accountId") == null;
if(!fullTree)
{
var accounts = audit.query(FileGatewayTables.ACCOUNT, UploadJobs.equal("id", merged.getValue("accountId")));
fullTree = accounts.isEmpty() || !"ENDPOINTS_ONLY".equals(accounts.getFirst().getValueString("rootMode"));
}
if(fullTree && QContext.getQInstance().getApps().containsKey(path.substring(1).split("/")[0]))
{
error = "Endpoint path collides with an application name for a FULL_TREE account";
}
for(QRecord other : audit.query(FileGatewayTables.ALIAS, new QQueryFilter()))
{
String mount = other.getValueString("path");
if(!java.util.Objects.equals(other.getValue("id"), merged.getValue("id")) && mount != null
&& (path.equals(mount) || path.startsWith(mount + "/") || mount.startsWith(path + "/")))
{
error = "Endpoint paths must be unique and must not overlap another endpoint mount";
}
}
}
String filename = merged.getValueString("fileName");
if(filename != null && !validFileName(filename))
{
error = "Endpoint file name must be a single safe filename";
}
if(error != null)
{
record.addError(new BadInputStatusMessage(error));
}
}
return (records);
}
/*******************************************************************************
** Object names cannot escape a mount or inject control characters.
*******************************************************************************/
public static boolean validFileName(String name)
{
return (name != null && !name.isBlank() && !name.equals(".") && !name.equals("..") && !name.contains("/")
&& !name.contains("\\") && !name.contains(":") && name.chars().noneMatch(Character::isISOControl));
}
}
@@ -73,7 +73,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -112,7 +112,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -151,7 +151,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -190,7 +190,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -229,7 +229,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -268,7 +268,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -307,7 +307,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -346,7 +346,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -385,7 +385,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -424,7 +424,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -463,7 +463,7 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
@@ -502,7 +502,18 @@ public final class FileGatewayEnums
@Override
public String getPossibleValueLabel()
{
return (value);
return (label(value));
}
}
/*******************************************************************************
** Plain label for a persisted value: ENDPOINTS_ONLY becomes "Endpoints only".
*******************************************************************************/
static String label(String value)
{
String words = value.replace('_', ' ').toLowerCase();
return (Character.toUpperCase(words.charAt(0)) + words.substring(1));
}
}
@@ -0,0 +1,232 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.metadata;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBlockData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextValues;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
/*******************************************************************************
** Typed home widgets over real gateway data, queried as the viewing user.
*******************************************************************************/
public class FileGatewayHome extends AbstractWidgetRenderer
{
/** Readable UTC timestamps in home tables (the widget title already names the table). */
private static final java.time.format.DateTimeFormatter WHEN = java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
public static final List<String> KEYS = List.of("transfers", "failures", "sessions", "in", "out", "locked", "recent", "partners", "active", "servers");
private static final List<String> LABELS = List.of("Transfers today", "Failed jobs (24 hours)", "Active sessions", "MB in today", "MB out today", "Locked accounts",
"Recent failures", "Top partners (7 days)", "Active sessions", "Server");
/*******************************************************************************
** Six small statistics followed by full-width responsive tables, without HTML.
*******************************************************************************/
public static List<QWidgetMetaData> widgets()
{
List<QWidgetMetaData> widgets = new ArrayList<>();
for(int i = 0; i < KEYS.size(); i++)
{
widgets.add(new QWidgetMetaData().withName("file_gateway_home_" + KEYS.get(i)).withLabel(LABELS.get(i))
.withType((i < 6 ? WidgetType.STATISTICS : WidgetType.TABLE).getType()).withGridColumns(i < 6 ? 2 : 12)
.withIsCard(true).withShowReloadButton(true).withCodeReference(new QCodeReference(FileGatewayHome.class))
.withPermissionRules(FileGatewayTables.permissionRules("file_gateway")));
}
return (widgets);
}
/*******************************************************************************
** The same permission is required for direct renderer calls and widget routes.
*******************************************************************************/
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
return (new RenderWidgetOutput(data(input.getWidgetMetaData().getName().replace("file_gateway_home_", ""), Instant.now())));
}
/*******************************************************************************
** Accept an explicit time for deterministic seeded-data assertions.
*******************************************************************************/
public QWidgetData data(String key, Instant now) throws QException
{
if(QContext.getQSession() == null || !(QContext.getQSession().hasPermission(FileGatewayPermissions.VIEW) || QContext.getQSession().hasPermission(FileGatewayPermissions.ADMIN)))
{
throw (new QPermissionDeniedException("Missing file_gateway.view"));
}
Instant today = now.atZone(ZoneOffset.UTC).toLocalDate().atStartOfDay(ZoneOffset.UTC).toInstant();
if(List.of("transfers", "in", "out").contains(key))
{
var rows = query(FileGatewayTables.TRANSFER_LOG, since("createDate", today));
long value = "transfers".equals(key) ? rows.size() : rows.stream()
.filter(row -> ("in".equals(key) ? "UPLOAD" : "DOWNLOAD").equals(row.getValueString("operation")))
.mapToLong(row -> number(row, "bytes")).sum();
return (new StatisticsData("transfers".equals(key) ? value : Math.round(value / 10485.76) / 100.0, null, null).withIncreaseIsGood(true));
}
if("locked".equals(key))
{
long count = query(FileGatewayTables.ACCOUNT, new QQueryFilter()).stream().filter(row -> "LOCKED".equals(row.getValueString("status"))
|| (row.getValueInstant("lockedUntil") != null && row.getValueInstant("lockedUntil").isAfter(now))).count();
return (new StatisticsData(count, null, null).withIncreaseIsGood(false));
}
if(List.of("failures", "recent").contains(key))
{
var rows = query(FileGatewayTables.UPLOAD_JOB, since("finishedAt", now.minusSeconds("failures".equals(key) ? 86400 : 7 * 86400))
.withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, "FAILED", "PARTIAL", "REJECTED")));
if("failures".equals(key))
{
return (new StatisticsData(rows.size(), null, null).withIncreaseIsGood(false));
}
rows.sort(java.util.Comparator.comparing((QRecord row) -> row.getValueInstant("finishedAt")).reversed());
return (table("Recent failures", FileGatewayTables.UPLOAD_JOB, rows.stream().limit(20).toList(),
List.of("fileName", "status", "errorMessage", "finishedAt"), List.of("File", "Status", "Message", "Finished"), "No failed transfers in the last 7 days."));
}
if(List.of("sessions", "active").contains(key))
{
var rows = query(FileGatewayTables.CONNECTION_LOG, since("lastSeenAt", now.minusSeconds(180))
.withCriteria(new QFilterCriteria("disconnectedAt", QCriteriaOperator.IS_BLANK))
.withCriteria(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "CONNECTED")));
return ("sessions".equals(key) ? new StatisticsData(rows.size(), null, null).withIncreaseIsGood(true) : table("Active sessions", FileGatewayTables.CONNECTION_LOG, rows,
List.of("username", "remoteIp", "connectedAt", "operationCount"), List.of("Username", "IP", "Connected", "Operations"),
"No one is connected right now (no session has been seen in the last 3 minutes)."));
}
if("servers".equals(key))
{
return (table("Server", FileGatewayTables.SERVER, query(FileGatewayTables.SERVER, since("lastSeenAt", now.minusSeconds(180))),
List.of("podName", "hostKeyFingerprint", "port", "lastSeenAt"), List.of("Pod", "Host key fingerprint", "Port", "Last seen"),
"No gateway server has reported in during the last 3 minutes."));
}
Map<Integer, QRecord> partners = new LinkedHashMap<>();
for(QRecord account : query(FileGatewayTables.ACCOUNT, new QQueryFilter()))
{
partners.put(account.getValueInteger("id"), account.withValue("transfers", 0L).withValue("mb", 0.0));
}
for(QRecord transfer : query(FileGatewayTables.TRANSFER_LOG, since("createDate", now.minusSeconds(7 * 86400))))
{
QRecord partner = partners.get(transfer.getValueInteger("accountId"));
if(partner != null)
{
partner.withValue("transfers", number(partner, "transfers") + 1)
.withValue("mb", ((Number) partner.getValue("mb")).doubleValue() + number(transfer, "bytes") / 1048576.0);
}
}
List<QRecord> rows = partners.values().stream().filter(row -> number(row, "transfers") > 0)
.sorted(java.util.Comparator.comparingLong((QRecord row) -> number(row, "transfers")).reversed()).limit(10).toList();
return (table("Top partners (7 days)", FileGatewayTables.ACCOUNT, rows, List.of("username", "transfers", "mb"), List.of("Partner", "Transfers", "MB"),
"No partner has transferred a file in the last 7 days."));
}
/*******************************************************************************
** Typed text blocks carry record links; cells never contain markup.
*******************************************************************************/
private TableData table(String label, String table, List<QRecord> records, List<String> fields, List<String> labels, String noRowsText)
{
List<TableData.Column> columns = new ArrayList<>();
for(int i = 0; i < fields.size(); i++)
{
columns.add(new TableData.Column(i == 0 ? "block" : "default", labels.get(i), fields.get(i), null, null));
}
List<Map<String, Object>> rows = new ArrayList<>();
for(QRecord record : records)
{
Map<String, Object> row = new LinkedHashMap<>();
for(String field : fields)
{
Object value = record.getValue(field);
row.put(field, value == null ? "" : value instanceof Instant instant ? WHEN.format(instant) : value);
}
row.put(fields.getFirst(), new TextBlockData().withValues(new TextValues(record.getValueString(fields.getFirst())))
.withLink("/record/" + table + "/" + record.getValue("id")));
rows.add(row);
}
return (new TableData(null, columns, rows).withRowsPerPage(10).withHidePaginationDropdown(true).withNoRowsFoundHTML(noRowsText));
}
/*******************************************************************************
** Queries preserve viewer security and use bounded database pages.
*******************************************************************************/
private List<QRecord> query(String table, QQueryFilter filter) throws QException
{
List<QRecord> rows = new ArrayList<>();
for(int offset = 0; ; offset += 250)
{
var page = new QueryAction().execute(new QueryInput(table).withFilter(filter.withSkip(offset).withLimit(250).withOrderBy(new QFilterOrderBy("id", true)))).getRecords();
rows.addAll(page);
if(page.size() < 250)
{
return (rows);
}
}
}
/*******************************************************************************
** UTC time windows agree with quota and transfer bookkeeping.
*******************************************************************************/
private QQueryFilter since(String field, Instant from)
{
return (new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.GREATER_THAN_OR_EQUALS, from)));
}
/*******************************************************************************
** Older audit rows may have no counters.
*******************************************************************************/
private long number(QRecord row, String field)
{
return (row.getValueLong(field) == null ? 0 : row.getValueLong(field));
}
}
@@ -229,7 +229,7 @@ public final class FileGatewayProcesses
case END_SESSION ->
{
table = FileGatewayTables.CONNECTION_LOG;
update.withValue("disconnectedAt", Instant.now()).withValue("disconnectReason", "Ended by administrator");
update.withValue("endRequestedAt", Instant.now()).withValue("disconnectReason", "Ended by administrator");
}
default -> throw (new QException("Unsupported gateway action"));
}
@@ -62,9 +62,10 @@ public final class FileGatewayTables
public static final String OUTBOX_FILE = "file_gateway_outbox_file";
public static final String TRANSFER_LOG = "file_gateway_transfer_log";
public static final String CONNECTION_LOG = "file_gateway_connection_log";
public static final List<String> ALL = List.of(ACCOUNT, SSH_KEY, ALIAS, UPLOAD_JOB, OUTBOX_FILE, TRANSFER_LOG, CONNECTION_LOG);
public static final String SERVER = "file_gateway_server";
public static final List<String> ALL = List.of(ACCOUNT, SSH_KEY, ALIAS, UPLOAD_JOB, OUTBOX_FILE, TRANSFER_LOG, CONNECTION_LOG, SERVER);
public static final List<String> MAIN = List.of(ACCOUNT, ALIAS, UPLOAD_JOB, TRANSFER_LOG, CONNECTION_LOG);
public static final List<String> SYSTEM_WRITTEN = List.of(UPLOAD_JOB, OUTBOX_FILE, TRANSFER_LOG, CONNECTION_LOG);
public static final List<String> SYSTEM_WRITTEN = List.of(UPLOAD_JOB, OUTBOX_FILE, TRANSFER_LOG, CONNECTION_LOG, SERVER);
@@ -112,12 +113,45 @@ public final class FileGatewayTables
/** Plain labels for child lists and their record-screen tabs. */
static final java.util.Map<String, String> CHILD_LABELS = java.util.Map.of(
SSH_KEY, "SSH keys", UPLOAD_JOB, "Upload jobs", CONNECTION_LOG, "Connections",
TRANSFER_LOG, "Transfers", OUTBOX_FILE, "Outbox files");
/** Plain field labels where the name-derived label reads badly. */
static final java.util.Map<String, String> FIELD_LABELS = java.util.Map.ofEntries(
java.util.Map.entry("allowedCidrs", "Allowed CIDRs"), java.util.Map.entry("userId", "User"),
java.util.Map.entry("ownerUserId", "Owner"), java.util.Map.entry("maxFileSizeMb", "Max file size (MB)"),
java.util.Map.entry("maxUploadMbPerDay", "Max upload per day (MB)"), java.util.Map.entry("maxDownloadMbPerDay", "Max download per day (MB)"),
java.util.Map.entry("lastLoginIp", "Last login IP"), java.util.Map.entry("lastUsedIp", "Last used IP"),
java.util.Map.entry("pgpPublicKey", "PGP public key"), java.util.Map.entry("pgpDecryptUploads", "Decrypt PGP uploads"),
java.util.Map.entry("pgpEncryptDownloads", "Encrypt downloads with PGP"), java.util.Map.entry("remoteIp", "Remote IP"),
java.util.Map.entry("remoteAddress", "Remote IP"), java.util.Map.entry("sha256", "SHA-256"),
java.util.Map.entry("processUuid", "Process UUID"), java.util.Map.entry("savedBulkLoadProfileId", "Bulk load profile"),
java.util.Map.entry("savedViewId", "Saved view"), java.util.Map.entry("savedReportId", "Saved report"),
java.util.Map.entry("sizeBytes", "Size (bytes)"), java.util.Map.entry("accountId", "Account"),
java.util.Map.entry("aliasId", "Endpoint"), java.util.Map.entry("connectionLogId", "Connection"),
java.util.Map.entry("hostKeyFingerprint", "Host key fingerprint"), java.util.Map.entry("keyFingerprint", "Key fingerprint"));
/*******************************************************************************
** Build all seven tables.
*******************************************************************************/
public static List<QTableMetaData> tables(FileGatewayQBitConfig config)
{
return (List.of(account(config), ssh_key(config), alias(config), upload_job(config), outbox_file(config), transfer_log(config), connection_log(config)));
List<QTableMetaData> tables = List.of(account(config), ssh_key(config), alias(config), upload_job(config), outbox_file(config), transfer_log(config), connection_log(config), server(config));
for(QTableMetaData table : tables)
{
for(QFieldMetaData field : table.getFields().values())
{
if(FIELD_LABELS.containsKey(field.getName()))
{
field.setLabel(FIELD_LABELS.get(field.getName()));
}
}
}
return (tables);
}
@@ -160,7 +194,7 @@ public final class FileGatewayTables
.withField(field("userId", QFieldType.STRING).withIsRequired(true))
.withField(field("ownerUserId", QFieldType.STRING))
.withField(field("status", QFieldType.STRING).withPossibleValueSourceName("file_gateway_account_status").withDefaultValue("ACTIVE").withIsRequired(true))
.withField(field("rootMode", QFieldType.STRING).withPossibleValueSourceName("file_gateway_root_mode").withDefaultValue("FULL_TREE").withIsRequired(true))
.withField(field("rootMode", QFieldType.STRING).withPossibleValueSourceName("file_gateway_root_mode"))
.withField(field("allowedCidrs", QFieldType.TEXT))
.withField(field("expiresAt", QFieldType.DATE_TIME))
.withField(field("formats", QFieldType.STRING).withDefaultValue("csv,xlsx"))
@@ -187,9 +221,9 @@ public final class FileGatewayTables
table.withSection(SectionFactory.defaultT1("id", "username", "label", "accountType", "userId", "ownerUserId"));
table.withSection(SectionFactory.defaultT2("status", "rootMode", "allowedCidrs", "expiresAt", "formats", "maxConcurrentSessions", "maxOperationsPerMinute", "maxFileSizeMb", "maxUploadMbPerDay", "maxDownloadMbPerDay", "dedupeUploads", "notifyOnSuccess", "pgpPublicKey", "pgpDecryptUploads", "pgpEncryptDownloads", "operationWindowAt", "operationWindowCount", "failedLoginCount", "failedLoginWindowAt", "lockedUntil", "lastLoginAt", "lastLoginIp"));
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
table.withSection(SectionFactory.customT2(SSH_KEY, new QIcon("list")).withWidgetName(ACCOUNT + "_children_" + SSH_KEY));
table.withSection(SectionFactory.customT2(UPLOAD_JOB, new QIcon("list")).withWidgetName(ACCOUNT + "_children_" + UPLOAD_JOB));
table.withSection(SectionFactory.customT2(CONNECTION_LOG, new QIcon("list")).withWidgetName(ACCOUNT + "_children_" + CONNECTION_LOG));
table.withSection(SectionFactory.customT2(SSH_KEY, new QIcon("list")).withLabel(CHILD_LABELS.get(SSH_KEY)).withWidgetName(ACCOUNT + "_children_" + SSH_KEY));
table.withSection(SectionFactory.customT2(UPLOAD_JOB, new QIcon("list")).withLabel(CHILD_LABELS.get(UPLOAD_JOB)).withWidgetName(ACCOUNT + "_children_" + UPLOAD_JOB));
table.withSection(SectionFactory.customT2(CONNECTION_LOG, new QIcon("list")).withLabel(CHILD_LABELS.get(CONNECTION_LOG)).withWidgetName(ACCOUNT + "_children_" + CONNECTION_LOG));
return (table);
}
@@ -246,11 +280,13 @@ public final class FileGatewayTables
.withField(field("description", QFieldType.TEXT));
table.withUniqueKey(new UniqueKey("name"));
table.withUniqueKey(new UniqueKey("path"));
table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(AliasCustomizer.class));
table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(AliasCustomizer.class));
table.withSection(SectionFactory.defaultT1("id", "name", "path", "kind", "accountId", "tableName"));
table.withSection(SectionFactory.defaultT2("savedBulkLoadProfileId", "savedViewId", "savedReportId", "format", "fileName", "consumerName", "isActive", "description"));
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
table.withSection(SectionFactory.customT2(UPLOAD_JOB, new QIcon("list")).withWidgetName(ALIAS + "_children_" + UPLOAD_JOB));
table.withSection(SectionFactory.customT2(OUTBOX_FILE, new QIcon("list")).withWidgetName(ALIAS + "_children_" + OUTBOX_FILE));
table.withSection(SectionFactory.customT2(UPLOAD_JOB, new QIcon("list")).withLabel(CHILD_LABELS.get(UPLOAD_JOB)).withWidgetName(ALIAS + "_children_" + UPLOAD_JOB));
table.withSection(SectionFactory.customT2(OUTBOX_FILE, new QIcon("list")).withLabel(CHILD_LABELS.get(OUTBOX_FILE)).withWidgetName(ALIAS + "_children_" + OUTBOX_FILE));
return (table);
}
@@ -261,7 +297,7 @@ public final class FileGatewayTables
*******************************************************************************/
private static QTableMetaData upload_job(FileGatewayQBitConfig config)
{
QTableMetaData table = base(config, UPLOAD_JOB, "Upload Jobs").withRecordVersionField("status")
QTableMetaData table = base(config, UPLOAD_JOB, "Upload Jobs")
.withRecordLabelFormatAndFields("%s", "id")
.withField(field("accountId", QFieldType.INTEGER).withPossibleValueSourceName(ACCOUNT))
.withField(field("aliasId", QFieldType.INTEGER).withPossibleValueSourceName(ALIAS))
@@ -284,6 +320,9 @@ public final class FileGatewayTables
.withField(field("resultSummary", QFieldType.TEXT))
.withField(field("resultsStorageKey", QFieldType.STRING))
.withField(field("resultsSizeBytes", QFieldType.LONG))
.withField(field("plaintextStorageKey", QFieldType.STRING))
.withField(field("plaintextSha256", QFieldType.STRING))
.withField(field("plaintextSizeBytes", QFieldType.LONG))
.withField(field("errorMessage", QFieldType.TEXT))
.withField(field("attempts", QFieldType.INTEGER))
.withField(field("nextAttemptAt", QFieldType.DATE_TIME))
@@ -294,9 +333,9 @@ public final class FileGatewayTables
.withField(field("finishedAt", QFieldType.DATE_TIME))
.withField(field("acknowledgedAt", QFieldType.DATE_TIME));
table.withSection(SectionFactory.defaultT1("id", "accountId", "aliasId", "connectionLogId", "remoteAddress", "tableName"));
table.withSection(SectionFactory.defaultT2("savedBulkLoadProfileId", "consumerName", "consumerReference", "path", "fileName", "sizeBytes", "sha256", "storageKey", "status", "duplicateOfJobId", "processUuid", "insertedCount", "errorCount", "resultSummary", "resultsStorageKey", "resultsSizeBytes", "errorMessage", "attempts", "nextAttemptAt", "claimedBy", "claimedAt", "queuedAt", "startedAt", "finishedAt", "acknowledgedAt"));
table.withSection(SectionFactory.defaultT2("savedBulkLoadProfileId", "consumerName", "consumerReference", "path", "fileName", "sizeBytes", "sha256", "storageKey", "status", "duplicateOfJobId", "processUuid", "insertedCount", "errorCount", "resultSummary", "resultsStorageKey", "resultsSizeBytes", "plaintextStorageKey", "plaintextSha256", "plaintextSizeBytes", "errorMessage", "attempts", "nextAttemptAt", "claimedBy", "claimedAt", "queuedAt", "startedAt", "finishedAt", "acknowledgedAt"));
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
table.withSection(SectionFactory.customT2(TRANSFER_LOG, new QIcon("list")).withWidgetName(UPLOAD_JOB + "_children_" + TRANSFER_LOG));
table.withSection(SectionFactory.customT2(TRANSFER_LOG, new QIcon("list")).withLabel(CHILD_LABELS.get(TRANSFER_LOG)).withWidgetName(UPLOAD_JOB + "_children_" + TRANSFER_LOG));
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
return (table);
}
@@ -318,11 +357,12 @@ public final class FileGatewayTables
.withField(field("status", QFieldType.STRING).withPossibleValueSourceName("file_gateway_outbox_status"))
.withField(field("firstDownloadedAt", QFieldType.DATE_TIME))
.withField(field("downloadCount", QFieldType.INTEGER))
.withField(field("onDownloadedPending", QFieldType.BOOLEAN).withDefaultValue(false))
.withField(field("removedAt", QFieldType.DATE_TIME))
.withField(field("expiresAt", QFieldType.DATE_TIME))
.withField(field("createdByConsumer", QFieldType.STRING));
table.withSection(SectionFactory.defaultT1("id", "aliasId", "fileName", "sizeBytes", "sha256", "storageKey"));
table.withSection(SectionFactory.defaultT2("status", "firstDownloadedAt", "downloadCount", "removedAt", "expiresAt", "createdByConsumer"));
table.withSection(SectionFactory.defaultT2("status", "firstDownloadedAt", "downloadCount", "onDownloadedPending", "removedAt", "expiresAt", "createdByConsumer"));
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
return (table);
@@ -378,13 +418,14 @@ public final class FileGatewayTables
.withField(field("connectedAt", QFieldType.DATE_TIME))
.withField(field("lastSeenAt", QFieldType.DATE_TIME))
.withField(field("disconnectedAt", QFieldType.DATE_TIME))
.withField(field("endRequestedAt", QFieldType.DATE_TIME))
.withField(field("disconnectReason", QFieldType.TEXT))
.withField(field("bytesIn", QFieldType.LONG))
.withField(field("bytesOut", QFieldType.LONG))
.withField(field("operationCount", QFieldType.LONG))
.withField(field("clientChannel", QFieldType.STRING));
table.withSection(SectionFactory.defaultT1("id", "accountId", "username", "remoteIp", "clientVersion", "keyFingerprint"));
table.withSection(SectionFactory.defaultT2("outcome", "sessionId", "podName", "connectedAt", "lastSeenAt", "disconnectedAt", "disconnectReason", "bytesIn", "bytesOut", "operationCount", "clientChannel"));
table.withSection(SectionFactory.defaultT2("outcome", "sessionId", "podName", "connectedAt", "lastSeenAt", "disconnectedAt", "endRequestedAt", "disconnectReason", "bytesIn", "bytesOut", "operationCount", "clientChannel"));
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
return (table);
@@ -392,6 +433,25 @@ public final class FileGatewayTables
/*******************************************************************************
** Gateway heartbeat shared with the main web deployment, never a navigation table.
*******************************************************************************/
private static QTableMetaData server(FileGatewayQBitConfig config)
{
return (base(config, SERVER, "Servers").withRecordLabelFormatAndFields("%s", "podName")
.withField(field("podName", QFieldType.STRING).withIsRequired(true))
.withField(field("hostKeyFingerprint", QFieldType.STRING))
.withField(field("port", QFieldType.INTEGER))
.withField(field("startedAt", QFieldType.DATE_TIME))
.withField(field("lastSeenAt", QFieldType.DATE_TIME))
.withUniqueKey(new UniqueKey("podName"))
.withSection(SectionFactory.defaultT1("id", "podName", "hostKeyFingerprint", "port", "startedAt", "lastSeenAt"))
.withSection(SectionFactory.defaultT3("createDate", "modifyDate"))
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE));
}
/*******************************************************************************
** Parent and child relationships.
*******************************************************************************/
@@ -423,7 +483,7 @@ public final class FileGatewayTables
for(QJoinMetaData join : joins())
{
widgets.add(ChildRecordListRenderer.widgetMetaDataBuilder(join)
.withName(join.getLeftTable() + "_children_" + join.getRightTable()).withLabel(join.getRightTable().replace("file_gateway_", "").replace("_", " "))
.withName(join.getLeftTable() + "_children_" + join.getRightTable()).withLabel(CHILD_LABELS.getOrDefault(join.getRightTable(), join.getRightTable()))
.withCanAddChildRecord(SSH_KEY.equals(join.getRightTable())).withMaxRows(100)
.getWidgetMetaData().withGridColumns(12).withPermissionRules(permissionRules(join.getRightTable())));
}
@@ -22,12 +22,21 @@ package com.kingsrook.qbits.filegateway.metadata;
import java.security.PublicKey;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import org.apache.sshd.common.config.keys.KeyUtils;
@@ -47,8 +56,9 @@ public class SshKeyCustomizer implements TableCustomizerInterface
** Parse and validate inserts and partial updates without logging key material.
*******************************************************************************/
@Override
public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecords)
public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecords) throws QException
{
Set<String> fingerprints = new HashSet<>();
for(QRecord record : records)
{
QRecord old = oldRecords.orElse(List.of()).stream().filter(candidate -> Objects.equals(candidate.getValue("id"), record.getValue("id"))).findFirst().orElse(null);
@@ -69,6 +79,19 @@ public class SshKeyCustomizer implements TableCustomizerInterface
catch(Exception e)
{
record.addError(new BadInputStatusMessage("Use an OpenSSH Ed25519, ECDSA, or RSA key of at least 3072 bits"));
continue;
}
// Core's unique-key check precedes this computed fingerprint.
QQueryFilter filter = new QQueryFilter().withCriteria("fingerprint", QCriteriaOperator.EQUALS, record.getValue("fingerprint"));
if(old != null)
{
filter.withCriteria("id", QCriteriaOperator.NOT_EQUALS, old.getValue("id"));
}
var transaction = input instanceof InsertInput insert ? insert.getTransaction() : ((UpdateInput) input).getTransaction();
if(!fingerprints.add(record.getValueString("fingerprint")) || !new QueryAction().execute(new QueryInput(FileGatewayTables.SSH_KEY)
.withFilter(filter.withLimit(1)).withTransaction(transaction)).getRecords().isEmpty())
{
record.addError(new BadInputStatusMessage("Fingerprint: this SSH key is already registered. Use a different key."));
}
}
return (records);
@@ -94,6 +94,10 @@ public class FileGatewayConnectionLog extends QRecordEntity
@QField
private Instant endRequestedAt;
/*******************************************************************************
** Create an empty entity.
*******************************************************************************/
@@ -699,4 +703,35 @@ public class FileGatewayConnectionLog extends QRecordEntity
{
this.clientChannel = clientChannel;
}
/*******************************************************************************
** Getter for endRequestedAt.
*******************************************************************************/
public Instant getEndRequestedAt()
{
return (endRequestedAt);
}
/*******************************************************************************
** Setter used by QRecordEntity population.
*******************************************************************************/
public void setEndRequestedAt(Instant endRequestedAt)
{
this.endRequestedAt = endRequestedAt;
}
/*******************************************************************************
** Fluent setter for endRequestedAt.
*******************************************************************************/
public FileGatewayConnectionLog withEndRequestedAt(Instant endRequestedAt)
{
this.endRequestedAt = endRequestedAt;
return (this);
}
}
@@ -79,6 +79,10 @@ public class FileGatewayOutboxFile extends QRecordEntity
@QField
private Boolean onDownloadedPending;
/*******************************************************************************
** Create an empty entity.
*******************************************************************************/
@@ -529,4 +533,35 @@ public class FileGatewayOutboxFile extends QRecordEntity
{
this.createdByConsumer = createdByConsumer;
}
/*******************************************************************************
** Getter for onDownloadedPending.
*******************************************************************************/
public Boolean getOnDownloadedPending()
{
return (onDownloadedPending);
}
/*******************************************************************************
** Setter used by QRecordEntity population.
*******************************************************************************/
public void setOnDownloadedPending(Boolean onDownloadedPending)
{
this.onDownloadedPending = onDownloadedPending;
}
/*******************************************************************************
** Fluent setter for onDownloadedPending.
*******************************************************************************/
public FileGatewayOutboxFile withOnDownloadedPending(Boolean onDownloadedPending)
{
this.onDownloadedPending = onDownloadedPending;
return (this);
}
}
@@ -0,0 +1,324 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.model;
import java.time.Instant;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QField;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
/*******************************************************************************
** Persisted gateway heartbeat, visible across deployments.
*******************************************************************************/
public class FileGatewayServer extends QRecordEntity
{
public static final String TABLE_NAME = "file_gateway_server";
@QField
private Integer id;
@QField
private Instant createDate;
@QField
private Instant modifyDate;
@QField
private String podName;
@QField
private String hostKeyFingerprint;
@QField
private Integer port;
@QField
private Instant startedAt;
@QField
private Instant lastSeenAt;
/*******************************************************************************
** Empty record.
*******************************************************************************/
public FileGatewayServer()
{
}
/*******************************************************************************
** Populate from gateway persistence.
*******************************************************************************/
public FileGatewayServer(QRecord record) throws QException
{
populateFromQRecord(record);
}
/*******************************************************************************
** Access id.
*******************************************************************************/
public Integer getId()
{
return (id);
}
/*******************************************************************************
** Access id.
*******************************************************************************/
public void setId(Integer id)
{
this.id = id;
}
/*******************************************************************************
** Access id.
*******************************************************************************/
public FileGatewayServer withId(Integer id)
{
this.id = id;
return (this);
}
/*******************************************************************************
** Access createDate.
*******************************************************************************/
public Instant getCreateDate()
{
return (createDate);
}
/*******************************************************************************
** Access createDate.
*******************************************************************************/
public void setCreateDate(Instant createDate)
{
this.createDate = createDate;
}
/*******************************************************************************
** Access createDate.
*******************************************************************************/
public FileGatewayServer withCreateDate(Instant createDate)
{
this.createDate = createDate;
return (this);
}
/*******************************************************************************
** Access modifyDate.
*******************************************************************************/
public Instant getModifyDate()
{
return (modifyDate);
}
/*******************************************************************************
** Access modifyDate.
*******************************************************************************/
public void setModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
}
/*******************************************************************************
** Access modifyDate.
*******************************************************************************/
public FileGatewayServer withModifyDate(Instant modifyDate)
{
this.modifyDate = modifyDate;
return (this);
}
/*******************************************************************************
** Access podName.
*******************************************************************************/
public String getPodName()
{
return (podName);
}
/*******************************************************************************
** Access podName.
*******************************************************************************/
public void setPodName(String podName)
{
this.podName = podName;
}
/*******************************************************************************
** Access podName.
*******************************************************************************/
public FileGatewayServer withPodName(String podName)
{
this.podName = podName;
return (this);
}
/*******************************************************************************
** Access hostKeyFingerprint.
*******************************************************************************/
public String getHostKeyFingerprint()
{
return (hostKeyFingerprint);
}
/*******************************************************************************
** Access hostKeyFingerprint.
*******************************************************************************/
public void setHostKeyFingerprint(String hostKeyFingerprint)
{
this.hostKeyFingerprint = hostKeyFingerprint;
}
/*******************************************************************************
** Access hostKeyFingerprint.
*******************************************************************************/
public FileGatewayServer withHostKeyFingerprint(String hostKeyFingerprint)
{
this.hostKeyFingerprint = hostKeyFingerprint;
return (this);
}
/*******************************************************************************
** Access port.
*******************************************************************************/
public Integer getPort()
{
return (port);
}
/*******************************************************************************
** Access port.
*******************************************************************************/
public void setPort(Integer port)
{
this.port = port;
}
/*******************************************************************************
** Access port.
*******************************************************************************/
public FileGatewayServer withPort(Integer port)
{
this.port = port;
return (this);
}
/*******************************************************************************
** Access startedAt.
*******************************************************************************/
public Instant getStartedAt()
{
return (startedAt);
}
/*******************************************************************************
** Access startedAt.
*******************************************************************************/
public void setStartedAt(Instant startedAt)
{
this.startedAt = startedAt;
}
/*******************************************************************************
** Access startedAt.
*******************************************************************************/
public FileGatewayServer withStartedAt(Instant startedAt)
{
this.startedAt = startedAt;
return (this);
}
/*******************************************************************************
** Access lastSeenAt.
*******************************************************************************/
public Instant getLastSeenAt()
{
return (lastSeenAt);
}
/*******************************************************************************
** Access lastSeenAt.
*******************************************************************************/
public void setLastSeenAt(Instant lastSeenAt)
{
this.lastSeenAt = lastSeenAt;
}
/*******************************************************************************
** Access lastSeenAt.
*******************************************************************************/
public FileGatewayServer withLastSeenAt(Instant lastSeenAt)
{
this.lastSeenAt = lastSeenAt;
return (this);
}
}
@@ -136,6 +136,16 @@ public class FileGatewayUploadJob extends QRecordEntity
@QField
private String plaintextStorageKey;
@QField
private String plaintextSha256;
@QField
private Long plaintextSizeBytes;
/*******************************************************************************
** Create an empty entity.
*******************************************************************************/
@@ -1170,4 +1180,97 @@ public class FileGatewayUploadJob extends QRecordEntity
return (this);
}
/*******************************************************************************
** Getter for plaintextStorageKey.
*******************************************************************************/
public String getPlaintextStorageKey()
{
return (plaintextStorageKey);
}
/*******************************************************************************
** Setter used by QRecordEntity population.
*******************************************************************************/
public void setPlaintextStorageKey(String plaintextStorageKey)
{
this.plaintextStorageKey = plaintextStorageKey;
}
/*******************************************************************************
** Fluent setter for plaintextStorageKey.
*******************************************************************************/
public FileGatewayUploadJob withPlaintextStorageKey(String plaintextStorageKey)
{
this.plaintextStorageKey = plaintextStorageKey;
return (this);
}
/*******************************************************************************
** Getter for plaintextSha256.
*******************************************************************************/
public String getPlaintextSha256()
{
return (plaintextSha256);
}
/*******************************************************************************
** Setter used by QRecordEntity population.
*******************************************************************************/
public void setPlaintextSha256(String plaintextSha256)
{
this.plaintextSha256 = plaintextSha256;
}
/*******************************************************************************
** Fluent setter for plaintextSha256.
*******************************************************************************/
public FileGatewayUploadJob withPlaintextSha256(String plaintextSha256)
{
this.plaintextSha256 = plaintextSha256;
return (this);
}
/*******************************************************************************
** Getter for plaintextSizeBytes.
*******************************************************************************/
public Long getPlaintextSizeBytes()
{
return (plaintextSizeBytes);
}
/*******************************************************************************
** Setter used by QRecordEntity population.
*******************************************************************************/
public void setPlaintextSizeBytes(Long plaintextSizeBytes)
{
this.plaintextSizeBytes = plaintextSizeBytes;
}
/*******************************************************************************
** Fluent setter for plaintextSizeBytes.
*******************************************************************************/
public FileGatewayUploadJob withPlaintextSizeBytes(Long plaintextSizeBytes)
{
this.plaintextSizeBytes = plaintextSizeBytes;
return (this);
}
}
@@ -0,0 +1,244 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.outbox;
import java.io.IOException;
import java.io.InputStream;
import java.time.Clock;
import java.util.List;
import java.util.UUID;
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
import com.kingsrook.qbits.filegateway.api.FileGatewayConsumer;
import com.kingsrook.qbits.filegateway.api.FileGatewayConsumersMetaData;
import com.kingsrook.qbits.filegateway.api.FileGatewayOutboxProvider;
import com.kingsrook.qbits.filegateway.api.OutboxFileRef;
import com.kingsrook.qbits.filegateway.metadata.AliasCustomizer;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.sftp.AuditWriter;
import com.kingsrook.qbits.filegateway.storage.GatewayObjectStore;
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
/*******************************************************************************
** Database and object-storage outbox usable in web pods without an SFTP listener.
*******************************************************************************/
public class OutboxService implements FileGatewayOutboxProvider
{
private static final QLogger LOG = QLogger.getLogger(OutboxService.class);
private final QInstance instance;
private final AuditWriter audit;
private final GatewayObjectStore store;
private final FileGatewayQBitConfig config;
private final Clock clock;
/*******************************************************************************
** No runtime, threads or protocol credentials are required.
*******************************************************************************/
public OutboxService(QInstance instance, GatewayObjectStore store, FileGatewayQBitConfig config, Clock clock)
{
this.instance = instance;
this.audit = new AuditWriter(instance);
this.store = store;
this.config = config;
this.clock = clock;
}
/*******************************************************************************
** Publish only after the object is complete and its caller-supplied size matches.
*******************************************************************************/
@Override
public OutboxFileRef put(String endpointName, String fileName, InputStream content, long sizeBytes) throws QException
{
if(!AliasCustomizer.validFileName(fileName) || sizeBytes < 0)
{
throw (new QException("Invalid outbox filename or size"));
}
QRecord alias = endpoint(endpointName);
String key = "file-gateway/outbox/" + alias.getValue("id") + "/" + UUID.randomUUID() + "/" + fileName;
try
{
GatewayObjectStore.Receipt receipt = store.put(key, content);
if(receipt.size() != sizeBytes)
{
throw (new QException("Outbox stream size does not match sizeBytes"));
}
return (audit.withRowLock(FileGatewayTables.ALIAS, "id", alias.getValue("id"), () ->
{
QRecord current = endpoint(endpointName);
for(QRecord existing : list(current.getValueInteger("id")))
{
if(fileName.equals(existing.getValueString("fileName")))
{
throw (new QException("An active outbox file already has this filename"));
}
}
QRecord row = audit.insert(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("aliasId", current.getValue("id"))
.withValue("fileName", fileName).withValue("storageKey", key).withValue("sizeBytes", receipt.size()).withValue("sha256", receipt.sha256())
.withValue("status", "READY").withValue("downloadCount", 0).withValue("onDownloadedPending", false)
.withValue("createdByConsumer", current.getValue("consumerName")).withValue("createDate", clock.instant())
.withValue("expiresAt", clock.instant().plus(config.getOutboxRetention())));
return (reference(row, current));
}));
}
catch(Exception e)
{
try
{
store.delete(key);
}
catch(IOException cleanup)
{
e.addSuppressed(cleanup);
}
throw (new QException("Unable to publish outbox file: " + (e instanceof QException ? e.getMessage() : "storage failed"), e));
}
}
/*******************************************************************************
** A missing, inactive or incorrectly typed endpoint cannot accept output.
*******************************************************************************/
private QRecord endpoint(String name) throws QException
{
var rows = audit.query(FileGatewayTables.ALIAS, UploadJobs.equal("name", name));
if(rows.isEmpty() || !Boolean.TRUE.equals(rows.getFirst().getValueBoolean("isActive")) || !"OUTBOX".equals(rows.getFirst().getValueString("kind")))
{
throw (new QException("Outbox endpoint is missing, inactive or not OUTBOX"));
}
return (rows.getFirst());
}
/*******************************************************************************
** Removed and expired files disappear immediately from the virtual folder.
*******************************************************************************/
public List<QRecord> list(Integer aliasId) throws QException
{
return (audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("aliasId", aliasId)).stream()
.filter(row -> List.of("READY", "DOWNLOADED").contains(row.getValueString("status"))).toList());
}
/*******************************************************************************
** Only a complete read schedules the durable, at-least-once notification.
*******************************************************************************/
public void downloaded(QRecord row) throws QException
{
audit.withRowLock(FileGatewayTables.OUTBOX_FILE, "id", row.getValue("id"), () ->
{
QRecord current = audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", row.getValue("id"))).getFirst();
if(List.of("READY", "DOWNLOADED").contains(current.getValueString("status")))
{
QRecord changes = new QRecord().withValue("id", current.getValue("id")).withValue("status", "DOWNLOADED")
.withValue("downloadCount", current.getValueInteger("downloadCount") + 1);
if(current.getValue("firstDownloadedAt") == null)
{
changes.withValue("firstDownloadedAt", clock.instant()).withValue("onDownloadedPending", true);
}
audit.update(FileGatewayTables.OUTBOX_FILE, changes);
}
return (null);
});
notifyDownloaded(row);
}
/*******************************************************************************
** Holding the row lock prevents duplicate concurrent delivery; crash retries remain possible.
*******************************************************************************/
public void notifyDownloaded(QRecord row) throws QException
{
audit.withRowLock(FileGatewayTables.OUTBOX_FILE, "id", row.getValue("id"), () ->
{
QRecord current = audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", row.getValue("id"))).getFirst();
if(!Boolean.TRUE.equals(current.getValueBoolean("onDownloadedPending")))
{
return (null);
}
try
{
QRecord alias = audit.query(FileGatewayTables.ALIAS, UploadJobs.equal("id", current.getValue("aliasId"))).getFirst();
String name = alias.getValueString("consumerName");
if(name != null && !name.isBlank())
{
FileGatewayConsumer selected = null;
for(var code : FileGatewayConsumersMetaData.ofOrWithNew(instance).getConsumers())
{
FileGatewayConsumer consumer = QCodeLoader.getAdHoc(FileGatewayConsumer.class, code);
if(name.equals(consumer.getName()))
{
selected = consumer;
}
}
if(selected == null)
{
throw (new QException("Outbox consumer is not registered"));
}
selected.onDownloaded(reference(current, alias));
}
audit.update(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("id", row.getValue("id")).withValue("onDownloadedPending", false));
}
catch(Exception e)
{
LOG.warn("Outbox download notification failed; maintenance will retry");
}
return (null);
});
}
/*******************************************************************************
** Acknowledgment retains audit state until object retention runs.
*******************************************************************************/
public void remove(QRecord row) throws QException
{
audit.withRowLock(FileGatewayTables.OUTBOX_FILE, "id", row.getValue("id"), () ->
{
audit.update(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("id", row.getValue("id"))
.withValue("status", "REMOVED").withValue("removedAt", clock.instant()));
return (null);
});
}
/*******************************************************************************
** Consumer contract contains no implementation types or storage credentials.
*******************************************************************************/
private OutboxFileRef reference(QRecord row, QRecord alias)
{
return (new OutboxFileRef(row.getValueInteger("id"), alias.getValueString("name"), alias.getValueString("path") + "/" + row.getValueString("fileName"),
row.getValueString("fileName"), row.getValueLong("sizeBytes"), row.getValueString("sha256"), row.getValueInstant("createDate")));
}
}
@@ -22,14 +22,19 @@ package com.kingsrook.qbits.filegateway.permissions;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qqq.backend.core.actions.permissions.AvailablePermission;
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionSubType;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
@@ -60,4 +65,42 @@ public class FileGatewayPermissionChecker implements CustomPermissionChecker
throw (new QPermissionDeniedException("Missing file gateway permission"));
}
}
/*******************************************************************************
** This checker gates on the three gateway permission names, not on per-object
** names, so it tells the catalog (getAllAvailablePermissions) those names.
*******************************************************************************/
@Override
public boolean handlesBuildAvailablePermission()
{
return (true);
}
/*******************************************************************************
** Collapse every gateway object onto file_gateway.view, .admin or .connect.
** The object name and type are fixed so the catalog (a set) lists each name
** once: reads and the app need view, writes and processes need admin, and the
** app also stands for SFTP sign-in (connect), which the gateway grants to
** sessions rather than checking on an object.
*******************************************************************************/
@Override
public AvailablePermission buildAvailablePermission(QPermissionRules rules, PermissionSubType permissionSubType, String baseName, MetaDataWithName metaDataWithName, String objectType)
{
AvailablePermission base = buildBaseAvailablePermission(rules, permissionSubType, metaDataWithName, objectType);
if(base == null)
{
return (null);
}
boolean write = "WRITE".equals(base.getPermissionType()) || "Process".equals(objectType);
String name = write ? FileGatewayPermissions.ADMIN : FileGatewayPermissions.VIEW;
if(metaDataWithName instanceof QAppMetaData)
{
name = FileGatewayPermissions.CONNECT;
}
return (base.withName(name).withObjectName("File Gateway").withObjectType("App").withPermissionType(name.substring(name.indexOf('.') + 1)));
}
}
@@ -0,0 +1,284 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.pgp;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.util.Date;
import org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags;
import org.bouncycastle.openpgp.PGPCompressedData;
import org.bouncycastle.openpgp.PGPEncryptedDataGenerator;
import org.bouncycastle.openpgp.PGPEncryptedDataList;
import org.bouncycastle.openpgp.PGPException;
import org.bouncycastle.openpgp.PGPLiteralData;
import org.bouncycastle.openpgp.PGPLiteralDataGenerator;
import org.bouncycastle.openpgp.PGPObjectFactory;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.bouncycastle.openpgp.PGPPublicKeyEncryptedData;
import org.bouncycastle.openpgp.PGPPublicKeyRingCollection;
import org.bouncycastle.openpgp.PGPSecretKeyRingCollection;
import org.bouncycastle.openpgp.PGPUtil;
import org.bouncycastle.openpgp.operator.bc.BcKeyFingerprintCalculator;
import org.bouncycastle.openpgp.operator.bc.BcPBESecretKeyDecryptorBuilder;
import org.bouncycastle.openpgp.operator.bc.BcPGPDataEncryptorBuilder;
import org.bouncycastle.openpgp.operator.bc.BcPGPDigestCalculatorProvider;
import org.bouncycastle.openpgp.operator.bc.BcPublicKeyDataDecryptorFactory;
import org.bouncycastle.openpgp.operator.bc.BcPublicKeyKeyEncryptionMethodGenerator;
/*******************************************************************************
** Bounded, in-memory OpenPGP streams; no key files, providers or background threads.
*******************************************************************************/
public final class PgpStreams
{
/*******************************************************************************
** Static stream factories only.
*******************************************************************************/
private PgpStreams()
{
}
/*******************************************************************************
** Require a usable encryption subkey, rather than accepting signing-only keys.
*******************************************************************************/
public static PGPPublicKey encryptionKey(String text) throws IOException, PGPException
{
if(text == null || text.isBlank())
{
throw (new PGPException("PGP public key is required"));
}
var rings = new PGPPublicKeyRingCollection(PGPUtil.getDecoderStream(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8))), new BcKeyFingerprintCalculator());
for(var ring : rings)
{
for(var key : ring)
{
if(!key.isMasterKey() && key.isEncryptionKey() && !key.hasRevocation()
&& (key.getValidSeconds() == 0 || key.getCreationTime().toInstant().plusSeconds(key.getValidSeconds()).isAfter(java.time.Instant.now())))
{
return (key);
}
}
}
throw (new PGPException("PGP public key must contain an unrevoked, unexpired encryption subkey"));
}
/*******************************************************************************
** AES-256 binary packets with integrity protection and bounded partial packets.
*******************************************************************************/
public static InputStream encrypt(InputStream source, String keyText, String filename) throws IOException
{
try
{
ByteArrayOutputStream pending = new ByteArrayOutputStream(131072);
var generator = new PGPEncryptedDataGenerator(new BcPGPDataEncryptorBuilder(SymmetricKeyAlgorithmTags.AES_256)
.setWithIntegrityPacket(true).setSecureRandom(new SecureRandom()));
generator.addMethod(new BcPublicKeyKeyEncryptionMethodGenerator(encryptionKey(keyText)));
OutputStream encrypted = generator.open(pending, new byte[32768]);
var literalGenerator = new PGPLiteralDataGenerator();
OutputStream literal = literalGenerator.open(encrypted, PGPLiteralData.BINARY, filename, new Date(0), new byte[32768]);
return (new InputStream()
{
private InputStream available = InputStream.nullInputStream();
private boolean finished;
private boolean closed;
@Override
public int read() throws IOException
{
byte[] one = new byte[1];
return (read(one, 0, 1) < 0 ? -1 : one[0] & 255);
}
@Override
public int read(byte[] bytes, int offset, int length) throws IOException
{
java.util.Objects.checkFromIndexSize(offset, length, bytes.length);
if(closed)
{
throw (new IOException("PGP stream closed"));
}
if(length == 0)
{
return (0);
}
while(true)
{
int count = available.read(bytes, offset, length);
if(count >= 0)
{
return (count);
}
if(pending.size() > 0)
{
available = new ByteArrayInputStream(pending.toByteArray());
pending.reset();
}
else if(finished)
{
return (-1);
}
else
{
byte[] chunk = source.readNBytes(32768);
if(chunk.length == 0)
{
literal.close();
encrypted.close();
finished = true;
}
else
{
literal.write(chunk);
}
}
}
}
@Override
public void close() throws IOException
{
closed = true;
source.close();
}
});
}
catch(PGPException e)
{
throw (new IOException("Unable to encrypt PGP download", e));
}
}
/*******************************************************************************
** Verify integrity at EOF before object storage can complete its plaintext upload.
*******************************************************************************/
public static InputStream decrypt(InputStream source, String secret, char[] passphrase, long maximum) throws IOException
{
try
{
if(secret == null)
{
throw (new PGPException("PGP private key environment variable is missing"));
}
var keys = new PGPSecretKeyRingCollection(PGPUtil.getDecoderStream(new ByteArrayInputStream(secret.getBytes(StandardCharsets.UTF_8))), new BcKeyFingerprintCalculator());
var factory = new PGPObjectFactory(PGPUtil.getDecoderStream(source), new BcKeyFingerprintCalculator());
Object packet = factory.nextObject();
if(!(packet instanceof PGPEncryptedDataList))
{
packet = factory.nextObject();
}
if(!(packet instanceof PGPEncryptedDataList encrypted))
{
throw (new PGPException("Expected PGP encrypted data"));
}
for(var item : encrypted)
{
if(!(item instanceof PGPPublicKeyEncryptedData data))
{
continue;
}
var key = keys.getSecretKey(data.getKeyID());
if(key == null)
{
continue;
}
var privateKey = key.extractPrivateKey(new BcPBESecretKeyDecryptorBuilder(new BcPGPDigestCalculatorProvider()).build(passphrase));
InputStream clear = data.getDataStream(new BcPublicKeyDataDecryptorFactory(privateKey));
Object message = new PGPObjectFactory(clear, new BcKeyFingerprintCalculator()).nextObject();
for(int depth = 0; message instanceof PGPCompressedData compressed && depth < 4; depth++)
{
message = new PGPObjectFactory(compressed.getDataStream(), new BcKeyFingerprintCalculator()).nextObject();
}
if(!(message instanceof PGPLiteralData literal) || !data.isIntegrityProtected())
{
throw (new PGPException("Expected integrity-protected literal PGP content"));
}
return (new FilterInputStream(literal.getInputStream())
{
private long consumed;
private boolean verified;
@Override
public int read() throws IOException
{
byte[] one = new byte[1];
return (read(one, 0, 1) < 0 ? -1 : one[0] & 255);
}
@Override
public int read(byte[] bytes, int offset, int length) throws IOException
{
int count = in.read(bytes, offset, length);
if(count > 0 && (consumed += count) > maximum)
{
throw (new IOException("Decrypted upload exceeds file size limit"));
}
if(count < 0 && !verified)
{
try
{
clear.transferTo(OutputStream.nullOutputStream());
if(!data.verify())
{
throw (new IOException("PGP integrity check failed"));
}
verified = true;
}
catch(PGPException e)
{
throw (new IOException("PGP integrity check failed", e));
}
}
return (count);
}
@Override
public void close() throws IOException
{
try
{
super.close();
}
finally
{
source.close();
}
}
});
}
throw (new PGPException("No matching PGP private key"));
}
catch(PGPException | RuntimeException e)
{
throw (new IOException("Unable to decrypt PGP upload", e));
}
}
}
@@ -0,0 +1,411 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.preferences;
import java.io.Serializable;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionRequest;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionResult;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionScope;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceChange;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceListMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceProviderInterface;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRequestContext;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRow;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRowChange;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceSectionMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceValues;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferencesMetaData;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
/*******************************************************************************
** Self-service SSH keys. Core supplies the trusted scope and retains the user
** and client channel; this provider never switches to a system session.
*******************************************************************************/
public class FileTransferPreferencesProvider implements UserPreferenceProviderInterface
{
public static final String SECTION_NAME = "file-transfer";
private static final List<String> KEY_FIELDS = List.of("label", "fingerprint", "keyType", "lastUsedAt", "expiresAt");
/*******************************************************************************
** Register without an app scope: connect permission is sufficient.
*******************************************************************************/
public static void register(QInstance instance, FileGatewayQBitConfig config)
{
UserPreferenceSectionMetaData section = new UserPreferenceSectionMetaData()
.withName(SECTION_NAME).withLabel("File transfer").withIcon(new QIcon("swap_vert")).withOrder(60)
.withDescription("Sign in to the SFTP file gateway with your SSH keys.")
.withPermissionName(FileGatewayPermissions.CONNECT).withProvider(new QCodeReference(FileTransferPreferencesProvider.class))
.withField(new QFieldMetaData("username", QFieldType.STRING).withLabel("Username").withIsEditable(false)
.withHelpContent(new QHelpContent("Adding your first key creates your SFTP account and username.")))
.withField(new QFieldMetaData("port", QFieldType.INTEGER).withLabel("Port").withIsEditable(false))
.withField(new QFieldMetaData("hostKeyFingerprint", QFieldType.STRING).withLabel("Host key fingerprint").withIsEditable(false))
.withList(new UserPreferenceListMetaData().withName("keys").withLabel("SSH keys").withRowLabel("Key")
.withDescription("Adding your first key creates your SFTP account.")
.withField(new QFieldMetaData("label", QFieldType.STRING).withLabel("Label").withIsRequired(true).withMaxLength(1024))
.withField(new QFieldMetaData("fingerprint", QFieldType.STRING).withLabel("Fingerprint").withIsEditable(false))
.withField(new QFieldMetaData("keyType", QFieldType.STRING).withLabel("Key type").withIsEditable(false))
.withField(new QFieldMetaData("lastUsedAt", QFieldType.DATE_TIME).withLabel("Last used").withIsEditable(false))
.withField(new QFieldMetaData("expiresAt", QFieldType.DATE_TIME).withLabel("Expires at")))
.withAction(new UserPreferenceActionMetaData().withName("addKey").withLabel("Add key")
.withScope(UserPreferenceActionScope.LIST).withListName("keys").withOrder(10)
.withInputField(new QFieldMetaData("publicKey", QFieldType.TEXT).withLabel("Public key").withIsRequired(true))
.withInputField(new QFieldMetaData("label", QFieldType.STRING).withLabel("Label").withIsRequired(true).withMaxLength(1024))
.withInputField(new QFieldMetaData("expiresAt", QFieldType.DATE_TIME).withLabel("Expires at")))
.withAction(new UserPreferenceActionMetaData().withName("revokeKey").withLabel("Revoke")
.withScope(UserPreferenceActionScope.ROW).withListName("keys").withOrder(20).withIsDestructive(true)
.withConfirmationText("Revoke this SSH key? It will no longer be able to sign in."));
if(config.getPublicHostName() != null && !config.getPublicHostName().isBlank())
{
section.withField(new QFieldMetaData("host", QFieldType.STRING).withLabel("Host").withIsEditable(false).withDefaultValue(config.getPublicHostName()));
}
UserPreferencesMetaData.ofOrWithNew(instance).withSection(section);
}
/*******************************************************************************
** Publish only the user's active key metadata, never the public-key text.
*******************************************************************************/
@Override
public UserPreferenceValues load(UserPreferenceRequestContext context) throws QException
{
QRecord account = account(context, null);
UserPreferenceValues result = new UserPreferenceValues();
result.getValues().put("username", account == null ? "" : account.getValue("username"));
List<QRecord> servers = query(FileGatewayTables.SERVER, new QQueryFilter()
.withOrderBy(new QFilterOrderBy("lastSeenAt", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(1), null);
if(!servers.isEmpty())
{
result.getValues().put("port", servers.getFirst().getValue("port"));
result.getValues().put("hostKeyFingerprint", servers.getFirst().getValue("hostKeyFingerprint"));
}
List<UserPreferenceRow> rows = new ArrayList<>();
if(account != null)
{
for(QRecord key : query(FileGatewayTables.SSH_KEY, keysFilter(account), null))
{
Map<String, Serializable> values = new LinkedHashMap<>();
KEY_FIELDS.forEach(field -> values.put(field, key.getValue(field)));
rows.add(new UserPreferenceRow().withKey(key.getValueString("id")).withLabel(key.getValueString("label")).withValues(values));
}
}
result.getLists().put("keys", rows);
return (result);
}
/*******************************************************************************
** Core validates types; reject blank labels as well as missing ones.
*******************************************************************************/
@Override
public List<String> validate(UserPreferenceRequestContext context, UserPreferenceChange change)
{
List<String> errors = new ArrayList<>();
for(UserPreferenceRowChange row : change.getLists().getOrDefault("keys", List.of()))
{
if(row.getValues().containsKey("label") && (row.getValues().get("label") == null || row.getValues().get("label").toString().isBlank()))
{
errors.add("Label: is required");
}
}
return (errors);
}
/*******************************************************************************
** Apply partial changes atomically, checking ownership again under row locks.
*******************************************************************************/
@Override
public void store(UserPreferenceRequestContext context, UserPreferenceChange change) throws QException
{
try(var tx = QBackendTransaction.openFor(new QueryInput(FileGatewayTables.ACCOUNT)))
{
try
{
QRecord account = lockedAccount(context, tx);
for(UserPreferenceRowChange row : change.getLists().getOrDefault("keys", List.of()))
{
QRecord key = lockedKey(account, row.getKey(), tx);
QRecord update = new QRecord().withValue("id", key.getValue("id"));
for(String field : List.of("label", "expiresAt"))
{
if(row.getValues().containsKey(field))
{
update.setValue(field, row.getValues().get(field));
}
}
update(FileGatewayTables.SSH_KEY, update, tx);
}
tx.commit();
}
catch(QException | RuntimeException e)
{
tx.rollback();
throw e;
}
}
}
/*******************************************************************************
** Add through the normal key customizer, or revoke an owned, unrevoked row.
*******************************************************************************/
@Override
public UserPreferenceActionResult runAction(UserPreferenceRequestContext context, UserPreferenceActionRequest request) throws QException
{
if(!List.of("addKey", "revokeKey").contains(request.getActionName()))
{
throw (new QBadRequestException("This action isn't available."));
}
if("addKey".equals(request.getActionName()))
{
List<String> errors = new ArrayList<>();
for(String field : List.of("publicKey", "label"))
{
if(request.getValues().get(field) == null || request.getValues().get(field).toString().isBlank())
{
errors.add(("publicKey".equals(field) ? "Public key" : "Label") + ": is required");
}
}
if(!errors.isEmpty())
{
throw (new QBadRequestException(String.join("\n", errors)));
}
}
try(var tx = QBackendTransaction.openFor(new QueryInput(FileGatewayTables.ACCOUNT)))
{
try
{
QRecord account = lockedAccount(context, tx);
UserPreferenceActionResult result;
if("addKey".equals(request.getActionName()))
{
if(account == null)
{
account = insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", availableUsername(context.getUserId(), tx))
.withValue("userId", context.getUserId()).withValue("accountType", "USER").withValue("rootMode", "FULL_TREE"), tx);
}
QRecord key = insert(FileGatewayTables.SSH_KEY, new QRecord().withValue("accountId", account.getValue("id"))
.withValue("publicKey", request.getValues().get("publicKey")).withValue("label", request.getValues().get("label"))
.withValue("expiresAt", request.getValues().get("expiresAt")), tx);
result = new UserPreferenceActionResult().withMessage("Added key " + key.getValueString("fingerprint") + ".").withFocusRowKey(key.getValueString("id"));
}
else
{
QRecord key = lockedKey(account, request.getRowKey(), tx);
update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", key.getValue("id")).withValue("revokedAt", Instant.now()), tx);
result = new UserPreferenceActionResult().withMessage("Key revoked.");
}
tx.commit();
return (result);
}
catch(QException | RuntimeException e)
{
tx.rollback();
throw e;
}
}
}
/*******************************************************************************
** Use only the session identity; service accounts stay in the admin tables.
*******************************************************************************/
private QRecord account(UserPreferenceRequestContext context, QBackendTransaction tx) throws QException
{
List<QRecord> rows = query(FileGatewayTables.ACCOUNT, new QQueryFilter()
.withCriteria("userId", QCriteriaOperator.EQUALS, context.getUserId())
.withCriteria("accountType", QCriteriaOperator.EQUALS, "USER").withOrderBy(new QFilterOrderBy("id", true)).withLimit(1), tx);
return (rows.isEmpty() ? null : rows.getFirst());
}
/*******************************************************************************
** A transactional timestamp update acquires the same lock used by gateway auth.
** Re-read after the lock in case an administrator changed ownership meanwhile.
*******************************************************************************/
private QRecord lockedAccount(UserPreferenceRequestContext context, QBackendTransaction tx) throws QException
{
QRecord account = account(context, tx);
if(account != null)
{
update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("modifyDate", Instant.now()), tx);
QRecord current = account(context, tx);
if(current == null || !Objects.equals(current.getValue("id"), account.getValue("id")))
{
throw (new QBadRequestException("Account: isn't a setting here"));
}
return (current);
}
return (null);
}
/*******************************************************************************
** Scope every key lookup to the caller's account and unrevoked rows.
*******************************************************************************/
private QQueryFilter keysFilter(QRecord account)
{
return (new QQueryFilter().withCriteria("accountId", QCriteriaOperator.EQUALS, account.getValue("id"))
.withCriteria("revokedAt", QCriteriaOperator.IS_BLANK).withOrderBy(new QFilterOrderBy("id", true)));
}
/*******************************************************************************
** Check ownership both before and after locking the key against reassignment.
*******************************************************************************/
private QRecord lockedKey(QRecord account, String rowKey, QBackendTransaction tx) throws QException
{
QRecord key = ownKey(account, rowKey, tx);
update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", key.getValue("id")).withValue("modifyDate", Instant.now()), tx);
return (ownKey(account, rowKey, tx));
}
/*******************************************************************************
** Do not interpret an arbitrary caller-supplied key as an unrestricted ID.
*******************************************************************************/
private QRecord ownKey(QRecord account, String rowKey, QBackendTransaction tx) throws QException
{
if(account != null)
{
for(QRecord key : query(FileGatewayTables.SSH_KEY, keysFilter(account), tx))
{
if(key.getValueString("id").equals(rowKey))
{
return (key);
}
}
}
throw (new QBadRequestException("Key: isn't a setting here"));
}
/*******************************************************************************
** Normalize opaque user IDs to the account validator's 3-64 character syntax.
** Truncate before suffixing, so even a 64-character collision stays valid.
*******************************************************************************/
private String availableUsername(String userId, QBackendTransaction tx) throws QException
{
String base = userId.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]", "-");
if(base.length() < 3)
{
base = "user-" + base;
}
base = base.substring(0, Math.min(64, base.length()));
String username = base;
int suffix = 2;
while(!query(FileGatewayTables.ACCOUNT, new QQueryFilter().withCriteria("username", QCriteriaOperator.EQUALS, username).withLimit(1), tx).isEmpty())
{
String ending = "-" + suffix++;
username = base.substring(0, Math.min(base.length(), 64 - ending.length())) + ending;
}
return (username);
}
/*******************************************************************************
** Standard actions retain customizers, configured audits and the caller's session.
*******************************************************************************/
private List<QRecord> query(String table, QQueryFilter filter, QBackendTransaction tx) throws QException
{
return (new QueryAction().execute(new QueryInput(table).withFilter(filter).withTransaction(tx)).getRecords());
}
/*******************************************************************************
** Preserve domain validation messages while withholding internal database errors.
*******************************************************************************/
private QRecord checked(QRecord record) throws QException
{
if(record.getErrors() != null && !record.getErrors().isEmpty())
{
List<String> messages = record.getErrors().stream().map(error -> error instanceof BadInputStatusMessage ? error.getMessage() : "Could not save this setting.").toList();
throw (new QBadRequestException(String.join("\n", messages)));
}
return (record);
}
/*******************************************************************************
** Insert via ordinary table validation, including fingerprint uniqueness.
*******************************************************************************/
private QRecord insert(String table, QRecord record, QBackendTransaction tx) throws QException
{
return (checked(new InsertAction().execute(new InsertInput(table).withRecord(record).withTransaction(tx)).getRecords().getFirst()));
}
/*******************************************************************************
** Persist only supplied fields, with the original user's audit attribution.
*******************************************************************************/
private void update(String table, QRecord record, QBackendTransaction tx) throws QException
{
checked(new UpdateAction().execute(new UpdateInput(table).withRecords(List.of(record)).withTransaction(tx)).getRecords().getFirst());
}
}
@@ -24,7 +24,7 @@ package com.kingsrook.qbits.filegateway.sftp;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer;
@@ -115,17 +115,28 @@ public class AuditWriter
*******************************************************************************/
public <T> T withAccountLock(String field, Serializable value, Work<T> work) throws QException
{
return (withRowLock(FileGatewayTables.ACCOUNT, field, value, work));
}
/*******************************************************************************
** Serialize transitions on one gateway-owned row across gateway and web pods.
*******************************************************************************/
public <T> T withRowLock(String table, String field, Serializable value, Work<T> work) throws QException
{
checkTable(table);
return (system(() ->
{
try(var tx = QBackendTransaction.openFor(new QueryInput(FileGatewayTables.ACCOUNT)))
try(var tx = QBackendTransaction.openFor(new QueryInput(table)))
{
transaction.set(tx);
try
{
var rows = query(FileGatewayTables.ACCOUNT, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)));
var rows = query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)));
if(!rows.isEmpty())
{
update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", rows.getFirst().getValue("id")).withValue("modifyDate", Instant.now()));
update(table, new QRecord().withValue("id", rows.getFirst().getValue("id")).withValue("modifyDate", Instant.now()));
}
T result = work.run();
tx.commit();
@@ -221,18 +232,35 @@ public class AuditWriter
/*******************************************************************************
** Optimistic status transition, called under withAccountLock because core checks
** expected versions before SQL execution rather than in the SQL WHERE clause.
** Retention deletes only gateway-owned rows under the bookkeeping session.
*******************************************************************************/
public void delete(String table, Serializable id) throws QException
{
checkTable(table);
system(() -> new com.kingsrook.qqq.backend.core.actions.tables.DeleteAction().execute(
new com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput(table).withPrimaryKey(id).withTransaction(transaction.get())));
}
/*******************************************************************************
** Guarded status transition. Callers hold withAccountLock, so the status re-read in
** that transaction is serialized across pods; status is compared here rather than
** through a core record-version field, because core advances version fields on
** every update (a STRING version becomes a random UUID).
*******************************************************************************/
public boolean transitionUpload(QRecord changes, String expectedStatus) throws QException
{
QRecord result = system(() -> new UpdateAction().execute(new UpdateInput(FileGatewayTables.UPLOAD_JOB)
.withRecords(List.of(changes)).withTransaction(transaction.get())
.withExpectedVersions(Map.of(changes.getValue("id"), expectedStatus))).getRecords().getFirst());
if(result.getErrors() != null && !result.getErrors().isEmpty())
if(transaction.get() == null)
{
throw (new QException("Upload transitions require the account lock transaction"));
}
var rows = query(FileGatewayTables.UPLOAD_JOB, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, changes.getValue("id"))));
if(rows.isEmpty() || !Objects.equals(expectedStatus, rows.getFirst().getValueString("status")))
{
return (false);
}
update(FileGatewayTables.UPLOAD_JOB, changes);
return (true);
}
@@ -48,6 +48,7 @@ public class GatewaySession
public final ServerSession ssh;
public final String remoteIp;
public Integer connectionId;
public java.time.Instant connectedAt;
public QRecord account;
public QRecord key;
public long operations;
@@ -37,6 +37,7 @@ import com.kingsrook.qbits.filegateway.storage.ObjectStorageGatewayStore;
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
import com.kingsrook.qbits.filegateway.vfs.GatewayFileSystem;
import com.kingsrook.qbits.filegateway.vfs.GatewayUiApiClient;
import com.kingsrook.qbits.filegateway.vfs.GeneratedDownloads;
import com.kingsrook.qbits.filegateway.vfs.VirtualTree;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
@@ -69,6 +70,11 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
private final SshKeyAuthenticator authenticator;
private final Duration pollInterval;
private ScheduledExecutorService poller;
private final AuditWriter heartbeatAudit;
private final Clock heartbeatClock;
private final String heartbeatPod;
private final String hostFingerprint;
private java.time.Instant startedAt;
@@ -88,7 +94,13 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
public SftpProtocolServer(QInstance instance, FileGatewayQBitConfig config, UiApiInvoker invoker, KeyPair hostKey, Clock clock, Duration pollInterval, String podName, GatewayObjectStore store)
{
this.pollInterval = pollInterval;
heartbeatAudit = new AuditWriter(instance);
heartbeatClock = clock;
heartbeatPod = podName == null ? java.util.UUID.randomUUID().toString() : podName;
hostFingerprint = org.apache.sshd.common.config.keys.KeyUtils.getFingerPrint(org.apache.sshd.common.digest.BuiltinDigests.sha256, hostKey.getPublic());
AuditWriter audit = new AuditWriter(instance);
var outbox = new com.kingsrook.qbits.filegateway.outbox.OutboxService(instance, store, config, clock);
com.kingsrook.qbits.filegateway.api.FileGatewayOutbox.setProvider(outbox);
authenticator = new SshKeyAuthenticator(instance, config, audit, clock, podName);
server = SshServer.setUpDefaultServer();
server.setPort(config.getSftpPort());
@@ -108,7 +120,18 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
server.setCommandFactory(null);
server.setForwardingFilter(RejectAllForwardingFilter.INSTANCE);
server.setAgentFactory(null);
server.setSubsystemFactories(List.of(new SftpSubsystemFactory.Builder().withFileSystemAccessor(new org.apache.sshd.sftp.server.SftpFileSystemAccessor()
server.setSubsystemFactories(List.of(new SftpSubsystemFactory.Builder().withSftpErrorStatusDataHandler(new org.apache.sshd.sftp.server.SftpErrorStatusDataHandler()
{
/*******************************************************************************
** Preserve actionable gateway and UI API errors in SFTP failure responses.
*******************************************************************************/
@Override
public String resolveErrorMessage(org.apache.sshd.sftp.server.SftpSubsystemEnvironment subsystem, int id, Throwable error, int status, int command, Object... args)
{
return (error instanceof IOException && error.getMessage() != null ? error.getMessage()
: org.apache.sshd.sftp.common.SftpHelper.resolveStatusMessage(status));
}
}).withFileSystemAccessor(new org.apache.sshd.sftp.server.SftpFileSystemAccessor()
{
/*******************************************************************************
** MINA must use our seekable channels, not the native FileChannel default.
@@ -140,7 +163,9 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
public FileSystem createFileSystem(SessionContext session)
{
GatewaySession state = session.getAttribute(GatewaySession.ATTRIBUTE);
return (new GatewayFileSystem(new VirtualTree(new GatewayUiApiClient(invoker, state), state, audit, clock).withUploads(new UploadJobs(audit, store, config, clock), store)));
GatewayUiApiClient client = new GatewayUiApiClient(invoker, state);
return (new GatewayFileSystem(new VirtualTree(client, state, audit, clock).withUploads(new UploadJobs(audit, store, config, clock), store)
.withDownloads(new GeneratedDownloads(client, state, audit, config, clock, instance)).withOutbox(outbox)));
}
});
server.addSessionListener(new SessionListener()
@@ -211,6 +236,16 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
public void start() throws IOException
{
server.start();
startedAt = heartbeatClock.instant();
try
{
heartbeat();
}
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
{
server.stop(true);
throw (new IOException("Unable to publish gateway heartbeat", e));
}
poller = Executors.newSingleThreadScheduledExecutor(runnable ->
{
Thread thread = new Thread(runnable, "file-gateway-revocation");
@@ -222,6 +257,7 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
try
{
authenticator.poll();
heartbeat();
}
catch(Exception e)
{
@@ -233,6 +269,28 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
/*******************************************************************************
** Publish host identity to the database for app homes in other deployments.
*******************************************************************************/
public void heartbeat() throws com.kingsrook.qqq.backend.core.exceptions.QException
{
var rows = heartbeatAudit.query(com.kingsrook.qbits.filegateway.metadata.FileGatewayTables.SERVER,
UploadJobs.equal("podName", heartbeatPod));
var values = new com.kingsrook.qqq.backend.core.model.data.QRecord().withValue("podName", heartbeatPod)
.withValue("hostKeyFingerprint", hostFingerprint).withValue("port", server.getPort())
.withValue("startedAt", startedAt).withValue("lastSeenAt", heartbeatClock.instant());
if(rows.isEmpty())
{
heartbeatAudit.insert(com.kingsrook.qbits.filegateway.metadata.FileGatewayTables.SERVER, values);
}
else
{
heartbeatAudit.update(com.kingsrook.qbits.filegateway.metadata.FileGatewayTables.SERVER, values.withValue("id", rows.getFirst().getValue("id")));
}
}
/*******************************************************************************
** Stop polling, revoke sessions and stop the listener.
*******************************************************************************/
@@ -82,16 +82,15 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
/*******************************************************************************
** Audit TCP accept and enforce the pre-authentication connection rate.
** Track TCP accept and enforce the pre-authentication connection rate. The
** connection-log row is written at the first sign-in attempt (or a refusal),
** so TCP health probes and port scans that never authenticate leave no row.
*******************************************************************************/
public synchronized void accepted(ServerSession ssh) throws QException
{
GatewaySession state = new GatewaySession(instance, ssh, clock);
ssh.setAttribute(GatewaySession.ATTRIBUTE, state);
state.connectionId = audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord()
.withValue("sessionId", state.id).withValue("remoteIp", state.remoteIp).withValue("podName", podName)
.withValue("connectedAt", clock.instant()).withValue("lastSeenAt", clock.instant())
.withValue("clientChannel", "sftp").withValue("outcome", "AUTH_FAILED")).getValueInteger("id");
state.connectedAt = clock.instant();
sessions.put(state.id, state);
prune(connections, clock.instant().minusSeconds(60));
Deque<Instant> window = connections.computeIfAbsent(state.remoteIp, ignored -> new ArrayDeque<>());
@@ -144,6 +143,7 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
}
QRecord account = first(FileGatewayTables.ACCOUNT, "username", username);
String fingerprint = KeyUtils.getFingerPrint(BuiltinDigests.sha256, publicKey);
ensureConnectionRow(state);
audit.update(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("id", state.connectionId)
.withValue("username", username).withValue("keyFingerprint", fingerprint).withValue("clientVersion", ssh.getClientVersion())
.withValue("accountId", account == null ? null : account.getValue("id")));
@@ -292,11 +292,30 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
/*******************************************************************************
** Write the connection-log row on first need (an auth attempt or a refusal).
** Its outcome stays AUTH_FAILED until the sign-in succeeds.
*******************************************************************************/
private void ensureConnectionRow(GatewaySession state) throws QException
{
if(state.connectionId == null)
{
Instant at = state.connectedAt == null ? clock.instant() : state.connectedAt;
state.connectionId = audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord()
.withValue("sessionId", state.id).withValue("remoteIp", state.remoteIp).withValue("podName", podName)
.withValue("connectedAt", at).withValue("lastSeenAt", clock.instant())
.withValue("clientChannel", "sftp").withValue("outcome", "AUTH_FAILED")).getValueInteger("id");
}
}
/*******************************************************************************
** Record the authentication outcome without credentials.
*******************************************************************************/
private boolean outcome(GatewaySession state, String outcome, String reason) throws QException
{
ensureConnectionRow(state);
audit.update(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("id", state.connectionId)
.withValue("outcome", outcome).withValue("disconnectReason", reason).withValue("lastSeenAt", clock.instant()));
if(reason != null)
@@ -364,7 +383,7 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
QRecord connection = first(FileGatewayTables.CONNECTION_LOG, "id", state.connectionId);
if(account == null || key == null || !"ACTIVE".equals(account.getValueString("status"))
|| expired(account, "expiresAt") || expired(key, "expiresAt") || key.getValue("revokedAt") != null
|| connection == null || connection.getValue("disconnectedAt") != null)
|| connection == null || connection.getValue("disconnectedAt") != null || connection.getValue("endRequestedAt") != null)
{
state.disconnectReason = connection != null && connection.getValueString("disconnectReason") != null
? connection.getValueString("disconnectReason") : "Account or key revoked";
@@ -0,0 +1,268 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.storage;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.NonWritableChannelException;
import java.nio.channels.SeekableByteChannel;
/*******************************************************************************
** A generated stream with a fixed-size rolling window; never regenerates or spools.
*******************************************************************************/
public final class GeneratedReadChannel implements SeekableByteChannel
{
/*******************************************************************************
** Charge newly consumed bytes and finalize the transfer when its handle closes.
*******************************************************************************/
public interface Audit
{
/*******************************************************************************
** Enforce limits before exposing the newly generated bytes to the client.
*******************************************************************************/
void charge(int bytes) throws IOException;
/*******************************************************************************
** Record the final outcome, including stream and seek errors.
*******************************************************************************/
void finish(IOException failure) throws IOException;
}
private final InputStream input;
private final byte[] window;
private final Audit audit;
private long end;
private long position;
private boolean eof;
private boolean open = true;
private IOException failure;
/*******************************************************************************
** The supplied stream is owned by this channel from construction through close.
*******************************************************************************/
public GeneratedReadChannel(InputStream input, int windowBytes, Audit audit)
{
if(windowBytes < 1)
{
throw (new IllegalArgumentException("Read-ahead window must be positive"));
}
this.input = input;
this.window = new byte[windowBytes];
this.audit = audit;
}
/*******************************************************************************
** Retain the most recent window, including skipped bytes for pipelined reads.
*******************************************************************************/
@Override
public synchronized int read(ByteBuffer destination) throws IOException
{
checkOpen();
int total = 0;
while(destination.hasRemaining())
{
int count = readChunk(destination);
if(count < 0)
{
return (total == 0 ? -1 : total);
}
total += count;
}
return (total);
}
/*******************************************************************************
** Fill one ring segment; the public read fills its request to avoid false MINA EOF.
*******************************************************************************/
private int readChunk(ByteBuffer destination) throws IOException
{
checkOpen();
if(failure != null)
{
throw (failure);
}
if(!destination.hasRemaining())
{
return (0);
}
try
{
if(eof && position >= end)
{
return (-1);
}
if(position < Math.max(0, end - window.length) || position - end >= window.length)
{
throw (new IOException("Read outside the generated-file window at " + position + " (stream offset " + end + "); download sequentially"));
}
while(position >= end && !eof)
{
int offset = (int) (end % window.length);
int length = (int) Math.min(Math.min(65536, window.length - offset), position - end + Math.min(destination.remaining(), window.length));
int count = input.readNBytes(window, offset, length);
if(count == 0)
{
eof = true;
}
else if(count > 0)
{
audit.charge(count);
end += count;
}
}
if(position >= end)
{
return (-1);
}
int offset = (int) (position % window.length);
int count = (int) Math.min(Math.min(destination.remaining(), end - position), window.length - offset);
destination.put(window, offset, count);
position += count;
return (count);
}
catch(IOException e)
{
failure = e;
try
{
input.close();
}
catch(IOException closeError)
{
e.addSuppressed(closeError);
}
throw (e);
}
}
/*******************************************************************************
** Generated content is read-only.
*******************************************************************************/
@Override
public int write(ByteBuffer source)
{
throw (new NonWritableChannelException());
}
/*******************************************************************************
** Current logical read offset.
*******************************************************************************/
@Override
public synchronized long position() throws IOException
{
checkOpen();
return (position);
}
/*******************************************************************************
** Validate window membership on read, when an actual request is made.
*******************************************************************************/
@Override
public synchronized SeekableByteChannel position(long value) throws IOException
{
checkOpen();
if(value < 0)
{
throw (new IllegalArgumentException("Negative read offset"));
}
position = value;
return (this);
}
/*******************************************************************************
** Generated size is unknown; consumers must read until EOF.
*******************************************************************************/
@Override
public long size() throws IOException
{
checkOpen();
return (0);
}
/*******************************************************************************
** Generated content cannot be truncated.
*******************************************************************************/
@Override
public SeekableByteChannel truncate(long size)
{
throw (new NonWritableChannelException());
}
/*******************************************************************************
** Handle state.
*******************************************************************************/
@Override
public boolean isOpen()
{
return (open);
}
/*******************************************************************************
** Always close the HTTP stream and finalize exactly one transfer row.
*******************************************************************************/
@Override
public synchronized void close() throws IOException
{
if(open)
{
open = false;
try
{
input.close();
}
catch(IOException e)
{
failure = e;
throw (e);
}
finally
{
audit.finish(failure);
}
}
}
/*******************************************************************************
** Reject operations after close.
*******************************************************************************/
private void checkOpen() throws ClosedChannelException
{
if(!open)
{
throw (new ClosedChannelException());
}
}
}
@@ -60,6 +60,9 @@ public final class ObjectReadChannel implements SeekableByteChannel
private final Ranges ranges;
private final Completion completion;
private long bytesRead;
private Completion fullDownload;
private long contiguousEnd;
private final java.util.TreeMap<Long, Long> readRanges = new java.util.TreeMap<>();
private boolean failed;
private long position;
private boolean open = true;
@@ -109,6 +112,7 @@ public final class ObjectReadChannel implements SeekableByteChannel
throw (new IOException("Stored object ended before its recorded size"));
}
destination.put(bytes);
recordRange(position, position + bytes.length);
position += bytes.length;
bytesRead += bytes.length;
return (bytes.length);
@@ -191,9 +195,45 @@ public final class ObjectReadChannel implements SeekableByteChannel
{
open = false;
completion.done(bytesRead, failed);
if(!failed && contiguousEnd >= size && fullDownload != null)
{
fullDownload.done(bytesRead, false);
}
}
}
/*******************************************************************************
** Optional acknowledgment, only after all bytes have been read on this handle.
*******************************************************************************/
public ObjectReadChannel withFullDownload(Completion callback)
{
fullDownload = callback;
return (this);
}
/*******************************************************************************
** Bounded interval tracking avoids counting repeated ranged reads as a complete file.
*******************************************************************************/
private void recordRange(long start, long end)
{
if(start <= contiguousEnd)
{
contiguousEnd = Math.max(contiguousEnd, end);
}
else if(readRanges.size() < 4096)
{
readRanges.merge(start, end, Math::max);
}
while(!readRanges.isEmpty() && readRanges.firstKey() <= contiguousEnd)
{
contiguousEnd = Math.max(contiguousEnd, readRanges.pollFirstEntry().getValue());
}
}
/*******************************************************************************
** Reject use after close.
*******************************************************************************/
@@ -63,6 +63,7 @@ public class UploadJobWorker implements AutoCloseable
private final AuditWriter audit;
private final Clock clock;
private final String pod;
private java.util.function.Function<String, String> environment = System::getenv;
private volatile boolean stopped;
private ScheduledExecutorService pool;
@@ -80,6 +81,17 @@ public class UploadJobWorker implements AutoCloseable
audit = new AuditWriter(instance);
}
/*******************************************************************************
** Share the host-key environment seam; secret values are never persisted or logged.
*******************************************************************************/
public UploadJobWorker withEnvironment(java.util.function.Function<String, String> environment)
{
this.environment = environment;
return (this);
}
/*******************************************************************************
** Each scheduled lane handles one job at a time, without an unbounded work queue.
*******************************************************************************/
@@ -222,6 +234,19 @@ public class UploadJobWorker implements AutoCloseable
return;
}
}
if(Boolean.TRUE.equals(account.getValueBoolean("pgpDecryptUploads")) && job.getValueString("fileName").toLowerCase(java.util.Locale.ROOT).matches(".*[.](pgp|gpg)$"))
{
try
{
decrypt(job, account);
file = inbound(job, account);
}
catch(Exception e)
{
finish(job, new QRecord().withValue("status", "REJECTED").withValue("errorMessage", "PGP decryption failed: invalid ciphertext, key, integrity or size limit"), account);
return;
}
}
FileGatewayConsumer consumer = consumer(job, user);
if(consumer == null)
{
@@ -251,6 +276,44 @@ public class UploadJobWorker implements AutoCloseable
}
}
/*******************************************************************************
** Keep the original for the archive and persist authenticated plaintext for retries.
*******************************************************************************/
private void decrypt(QRecord job, QRecord account) throws IOException, QException
{
if(job.getValue("plaintextStorageKey") != null)
{
return;
}
String key = "file-gateway/plaintext/" + job.getValue("id") + "/" + java.util.UUID.randomUUID();
String pass = config.getPgpPassphraseEnvVarName() == null ? null : environment.apply(config.getPgpPassphraseEnvVarName());
char[] password = pass == null ? new char[0] : pass.toCharArray();
try(var source = store.open(job.getValueString("storageKey"));
var plaintext = com.kingsrook.qbits.filegateway.pgp.PgpStreams.decrypt(source,
config.getPgpPrivateKeyEnvVarName() == null ? null : environment.apply(config.getPgpPrivateKeyEnvVarName()), password,
(account.getValueInteger("maxFileSizeMb") == null ? config.getDefaultMaxFileSizeMb() : account.getValueInteger("maxFileSizeMb")) * UploadJobs.MIB))
{
var receipt = store.put(key, plaintext);
QRecord changes = new QRecord().withValue("plaintextStorageKey", key).withValue("plaintextSha256", receipt.sha256()).withValue("plaintextSizeBytes", receipt.size());
if(!persistOwned(job, changes))
{
throw (new IOException("Upload claim expired during PGP decryption"));
}
job.getValues().putAll(changes.getValues());
}
catch(Exception e)
{
store.delete(key);
throw (new IOException("PGP decryption failed", e));
}
finally
{
java.util.Arrays.fill(password, '\0');
}
}
/*******************************************************************************
** Registry lookup is by declared name, with the built-in bulk-load consumer reserved.
*******************************************************************************/
@@ -275,15 +338,24 @@ public class UploadJobWorker implements AutoCloseable
/*******************************************************************************
** Immutable delivery envelope; each openStream gets an independent object stream.
*******************************************************************************/
private InboundFile inbound(QRecord job, QRecord account)
private InboundFile inbound(QRecord job, QRecord account) throws QException
{
return (new InboundFile(job.getValueInteger("id"), null, account.getValueString("username"), account.getValueString("userId"),
job.getValueString("remoteAddress"), job.getValueString("path"), job.getValueString("fileName"), job.getValueLong("sizeBytes"),
job.getValueString("sha256"), job.getValueInstant("createDate"), config.getFileGatewayBucketAlias(), job.getValueString("storageKey"), () ->
String endpointName = job.getValue("aliasId") == null ? null
: audit.query(FileGatewayTables.ALIAS, UploadJobs.equal("id", job.getValue("aliasId"))).getFirst().getValueString("name");
boolean plaintext = job.getValue("plaintextStorageKey") != null;
String storageKey = job.getValueString(plaintext ? "plaintextStorageKey" : "storageKey");
String filename = job.getValueString("fileName");
if(plaintext)
{
filename = filename.substring(0, filename.length() - 4);
}
return (new InboundFile(job.getValueInteger("id"), endpointName, account.getValueString("username"), account.getValueString("userId"),
job.getValueString("remoteAddress"), job.getValueString("path"), filename, job.getValueLong(plaintext ? "plaintextSizeBytes" : "sizeBytes"),
job.getValueString(plaintext ? "plaintextSha256" : "sha256"), job.getValueInstant("createDate"), config.getFileGatewayBucketAlias(), storageKey, () ->
{
try
{
return (store.open(job.getValueString("storageKey")));
return (store.open(storageKey));
}
catch(IOException e)
{
@@ -65,6 +65,16 @@ public class UploadJobs
** The target was resolved through the user's metadata; endpoint routing can reuse this seam.
*******************************************************************************/
public SeekableByteChannel receive(GatewaySession session, String path, String table, Integer profile) throws IOException
{
return (receive(session, path, table, profile, null));
}
/*******************************************************************************
** Persist the resolved endpoint with the upload before any bytes are accepted.
*******************************************************************************/
public SeekableByteChannel receive(GatewaySession session, String path, String table, Integer profile, QRecord alias) throws IOException
{
String name = path.substring(path.lastIndexOf('/') + 1);
if(name.isBlank() || name.contains(":") || name.contains("\\") || name.chars().anyMatch(Character::isISOControl))
@@ -82,7 +92,8 @@ public class UploadJobs
QRecord job = audit.insert(FileGatewayTables.UPLOAD_JOB, new QRecord()
.withValue("accountId", session.account.getValue("id")).withValue("connectionLogId", session.connectionId)
.withValue("remoteAddress", session.remoteIp).withValue("tableName", table).withValue("savedBulkLoadProfileId", profile)
.withValue("consumerName", "bulk-load").withValue("path", path).withValue("fileName", name).withValue("storageKey", key)
.withValue("aliasId", alias == null ? null : alias.getValue("id"))
.withValue("consumerName", alias != null && "INBOX".equals(alias.getValueString("kind")) ? alias.getValueString("consumerName") : "bulk-load").withValue("path", path).withValue("fileName", name).withValue("storageKey", key)
.withValue("status", "RECEIVING").withValue("sizeBytes", 0L).withValue("attempts", 0).withValue("createDate", clock.instant()));
long started = System.nanoTime();
return (new UploadChannel(store, key, Math.toIntExact(config.getUploadPartSizeMb() * MIB), config.getUploadReorderWindowMb() * MIB, new UploadChannel.Lifecycle()
@@ -22,6 +22,7 @@ package com.kingsrook.qbits.filegateway.vfs;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.AccessDeniedException;
import java.nio.file.NoSuchFileException;
import com.fasterxml.jackson.databind.JsonNode;
@@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInvoker;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiStreamResponse;
import com.kingsrook.qqq.openapi.model.HttpMethod;
@@ -94,7 +96,7 @@ public class GatewayUiApiClient
/*******************************************************************************
** Apply the fixed channel identity, credentials and contract check on every call.
*******************************************************************************/
protected JsonNode invoke(UiApiRequest request) throws IOException
public JsonNode invoke(UiApiRequest request) throws IOException
{
if(invoker.findOperation(request.getMethod(), request.getPath()).isEmpty())
{
@@ -110,11 +112,11 @@ public class GatewayUiApiClient
}
if(response.getStatus() == 401 || response.getStatus() == 403)
{
throw (new AccessDeniedException(request.getPath()));
throw (new AccessDeniedException(request.getPath(), null, response.getBody()));
}
if(!response.isSuccess())
{
throw (new IOException("UI API request failed with status " + response.getStatus()));
throw (new IOException("UI API request failed with status " + response.getStatus() + ": " + response.getBody()));
}
return (response.getJson());
}
@@ -123,4 +125,39 @@ public class GatewayUiApiClient
throw (new IOException("User session unavailable", e));
}
}
/*******************************************************************************
** Open a binary response as this user, retaining only a bounded error body.
*******************************************************************************/
public UiApiStreamResponse stream(UiApiRequest request) throws IOException
{
if(invoker.findOperation(request.getMethod(), request.getPath()).isEmpty())
{
throw (new IOException("Gateway requires a spec-defined UI API operation"));
}
try
{
UiApiStreamResponse response = invoker.invokeForStream(request.withCredentials(UiApiCredentials.ofInProcessSession(session.token()))
.withInvokedBy("file-gateway").withClientDetail(session.id));
if(response.isSuccess())
{
return (response);
}
try(response)
{
String error = new String(response.getInputStream().readNBytes(65536), StandardCharsets.UTF_8);
if(response.getStatus() == 401 || response.getStatus() == 403)
{
throw (new AccessDeniedException(request.getPath(), null, error));
}
throw (new IOException("UI API request failed with status " + response.getStatus() + ": " + error));
}
}
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
{
throw (new IOException("User session unavailable", e));
}
}
}
@@ -0,0 +1,342 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway.vfs;
import java.io.IOException;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.AccessDeniedException;
import java.nio.file.NoSuchFileException;
import java.time.Clock;
import java.time.ZoneOffset;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.sftp.AuditWriter;
import com.kingsrook.qbits.filegateway.sftp.GatewaySession;
import com.kingsrook.qbits.filegateway.storage.GeneratedReadChannel;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiStreamResponse;
import com.kingsrook.qqq.openapi.model.HttpMethod;
/*******************************************************************************
** Generate user-authorized exports and reports through the UI API, with durable quotas.
*******************************************************************************/
public final class GeneratedDownloads
{
private final GatewayUiApiClient client;
private final GatewaySession session;
private final AuditWriter audit;
private final FileGatewayQBitConfig config;
private final Clock clock;
private final boolean reportsAvailable;
private final ObjectMapper mapper = new ObjectMapper();
/*******************************************************************************
** Inspect only host configuration for report storage; visibility still uses user metadata.
*******************************************************************************/
public GeneratedDownloads(GatewayUiApiClient client, GatewaySession session, AuditWriter audit, FileGatewayQBitConfig config, Clock clock, QInstance instance)
{
this.client = client;
this.session = session;
this.audit = audit;
this.config = config;
this.clock = clock;
var process = instance.getProcess("renderSavedReport");
reportsAvailable = process != null && process.getInputFields().stream().anyMatch(field -> "storageTableName".equals(field.getName())
&& field.getDefaultValue() != null && instance.getTable(field.getDefaultValue().toString()) != null);
}
/*******************************************************************************
** A render process without its configured storage table cannot provide downloads.
*******************************************************************************/
public boolean reportsAvailable()
{
return (reportsAvailable);
}
/*******************************************************************************
** Start generation at open, owning the HTTP stream until the SFTP handle closes.
*******************************************************************************/
public SeekableByteChannel open(VirtualTree.Node node) throws IOException
{
Transfer transfer = new Transfer(node);
try
{
if(!session.operationAllowed())
{
transfer.limit = true;
throw (new IOException("Operation limit exceeded"));
}
int window = Math.toIntExact(config.getDownloadReadAheadMb() * 1024L * 1024L);
if(window < 1)
{
throw (new IOException("Read-ahead window must be positive"));
}
String prefix = node.path().substring(0, node.path().lastIndexOf('/'));
prefix = prefix.substring(0, prefix.lastIndexOf('/'));
String table = node.job() == null ? prefix.substring(prefix.lastIndexOf('/') + 1) : node.job().getValueString("tableName");
JsonNode record = null;
for(JsonNode row : client.savedObjects("VIEW".equals(node.kind()) ? "savedView" : "savedReport", table))
{
if(node.objectId().equals(row.path("values").path("id").asText()))
{
record = row.path("values");
}
}
if(record == null)
{
throw (new NoSuchFileException(node.path()));
}
UiApiStreamResponse response = "VIEW".equals(node.kind()) ? export(table, record, transfer.format) : report(node.objectId(), transfer.format);
return (new GeneratedReadChannel(response.getInputStream(), window, transfer));
}
catch(IOException | RuntimeException e)
{
IOException failure = e instanceof IOException io ? io : new IOException("Unable to generate download", e);
transfer.finish(failure);
throw (failure);
}
}
/*******************************************************************************
** Use the Next/Material saved-view schema, retaining filter ordering and visible columns.
*******************************************************************************/
private UiApiStreamResponse export(String table, JsonNode record, String format) throws IOException
{
JsonNode view = mapper.readTree(record.path("viewJson").asText());
if(view == null || !view.isObject())
{
throw (new IOException("Saved view has invalid viewJson"));
}
ObjectNode body = mapper.createObjectNode().put("format", format).put("includeHeaderRow", true);
JsonNode filter = view.has("queryFilter") ? view.path("queryFilter") : view.path("filter");
if(filter.isObject())
{
ObjectNode query = filter.deepCopy();
query.remove(List.of("skip", "limit"));
body.set("filter", query);
}
JsonNode columns = view.path("queryColumns").path("columns");
if(columns.isArray())
{
var names = body.putArray("fieldNames");
for(JsonNode column : columns)
{
String name = column.path("name").asText();
if(column.path("isVisible").asBoolean(true) && !name.isEmpty() && !name.equals("__check__"))
{
names.add(name);
}
}
if(names.isEmpty())
{
throw (new IOException("Saved view has no visible export columns"));
}
}
else if(view.path("fieldNames").isArray())
{
body.set("fieldNames", view.path("fieldNames"));
}
return (client.stream(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/table/{tableName}/export")
.withPathParam("tableName", table).withJsonBody(body)));
}
/*******************************************************************************
** Select one record, submit the format input, then follow asynchronous render status.
*******************************************************************************/
private UiApiStreamResponse report(String id, String format) throws IOException
{
String values = mapper.writeValueAsString(Map.of("reportFormat", format));
JsonNode response = client.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/processes/{processName}/init")
.withPathParam("processName", "renderSavedReport").withFormParam("recordsParam", "recordIds").withFormParam("recordIds", id)
.withFormParam("values", values));
String uuid = response.path("processUUID").asText();
String job = response.path("jobUUID").asText();
for(int poll = 0; poll < 3600; poll++)
{
String type = response.path("type").asText();
if("ERROR".equals(type))
{
throw (new IOException(response.path("userFacingError").asText("Report generation failed")));
}
if("JOB_STARTED".equals(type) || "RUNNING".equals(type))
{
if(response.hasNonNull("jobUUID"))
{
job = response.path("jobUUID").asText();
}
try
{
Thread.sleep(100);
}
catch(InterruptedException e)
{
Thread.currentThread().interrupt();
throw (new IOException("Report generation interrupted", e));
}
response = client.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath("/processes/{processName}/{processUUID}/status/{jobUUID}")
.withPathParam("processName", "renderSavedReport").withPathParam("processUUID", uuid).withPathParam("jobUUID", job));
continue;
}
JsonNode output = response.path("values");
if(output.hasNonNull("storageReference") && output.hasNonNull("downloadFileName") && output.hasNonNull("storageTableName"))
{
return (client.stream(new UiApiRequest().withMethod(HttpMethod.GET).withPath("/download/{file}")
.withPathParam("file", output.path("downloadFileName").asText())
.withQueryParam("storageTableName", output.path("storageTableName").asText())
.withQueryParam("storageReference", output.path("storageReference").asText())));
}
if(!"input".equals(response.path("nextStep").asText()))
{
throw (new IOException("Unexpected report step: " + response.path("nextStep").asText()));
}
response = client.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/processes/{processName}/{processUUID}/step/{stepName}")
.withPathParam("processName", "renderSavedReport").withPathParam("processUUID", uuid).withPathParam("stepName", "input")
.withFormParam("values", values));
}
throw (new IOException("Report generation timed out"));
}
/*******************************************************************************
** One durable row per handle, updated under the account lock as bytes flow.
*******************************************************************************/
private final class Transfer implements GeneratedReadChannel.Audit
{
private final long began = System.nanoTime();
private final String format;
private final QRecord row;
private long bytes;
private boolean limit;
/*******************************************************************************
** Default unfinished transfers to ERROR, including a process terminated mid-read.
*******************************************************************************/
private Transfer(VirtualTree.Node node) throws IOException
{
format = node.job() == null ? node.path().substring(node.path().lastIndexOf('.') + 1) : node.job().getValueString("format");
try
{
row = audit.insert(FileGatewayTables.TRANSFER_LOG, new QRecord().withValue("accountId", session.account.getValue("id"))
.withValue("connectionLogId", session.connectionId).withValue("operation", "DOWNLOAD").withValue("path", node.path())
.withValue("objectKind", node.kind()).withValue("objectId", node.objectId()).withValue("format", format)
.withValue("bytes", 0L).withValue("outcome", "ERROR").withValue("clientChannel", "sftp/" + session.id));
}
catch(QException e)
{
throw (new IOException("Unable to persist download audit", e));
}
}
/*******************************************************************************
** Filter by account, operation and UTC day; include in-progress and failed transfers.
*******************************************************************************/
@Override
public void charge(int count) throws IOException
{
try
{
boolean allowed = audit.withAccountLock("id", session.account.getValue("id"), () ->
{
long maximum = config.getMaxGeneratedFileMb() * 1024L * 1024L;
Long dailyMb = session.account.getValueLong("maxDownloadMbPerDay");
long total = 0;
if(dailyMb != null)
{
var start = clock.instant().atZone(ZoneOffset.UTC).toLocalDate().atStartOfDay(ZoneOffset.UTC).toInstant();
for(int skip = 0; ; skip += 250)
{
var rows = audit.query(FileGatewayTables.TRANSFER_LOG, new QQueryFilter()
.withCriteria(new QFilterCriteria("accountId", QCriteriaOperator.EQUALS, session.account.getValue("id")))
.withCriteria(new QFilterCriteria("operation", QCriteriaOperator.EQUALS, "DOWNLOAD"))
.withCriteria(new QFilterCriteria("createDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, start))
.withOrderBy(new QFilterOrderBy("id", true)).withSkip(skip).withLimit(250));
for(QRecord record : rows)
{
Long value = record.getValueLong("bytes");
total += value == null ? 0 : value;
}
if(rows.size() < 250)
{
break;
}
}
}
if(bytes + count > maximum || (dailyMb != null && total + count > dailyMb * 1024L * 1024L))
{
return (false);
}
audit.update(FileGatewayTables.TRANSFER_LOG, new QRecord().withValue("id", row.getValue("id")).withValue("bytes", bytes + count));
return (true);
});
if(!allowed)
{
limit = true;
throw (new IOException("Generated file or daily download limit exceeded"));
}
bytes += count;
}
catch(QException e)
{
throw (new IOException("Unable to enforce download limits", e));
}
}
/*******************************************************************************
** Preserve API denial and limit outcomes instead of collapsing them to generic errors.
*******************************************************************************/
@Override
public void finish(IOException failure) throws IOException
{
try
{
audit.update(FileGatewayTables.TRANSFER_LOG, new QRecord().withValue("id", row.getValue("id"))
.withValue("durationMs", (System.nanoTime() - began) / 1_000_000)
.withValue("outcome", limit ? "LIMIT" : failure instanceof AccessDeniedException ? "DENIED" : failure == null ? "OK" : "ERROR")
.withValue("message", failure == null ? null : failure.getMessage()));
}
catch(QException e)
{
throw (new IOException("Unable to finalize download audit", e));
}
}
}
}
@@ -70,6 +70,8 @@ public class VirtualTree
private final Clock clock;
private UploadJobs uploads;
private GatewayObjectStore store;
private GeneratedDownloads downloads;
private com.kingsrook.qbits.filegateway.outbox.OutboxService outbox;
private JsonNode metadata;
private Instant metadataAt;
private final Map<String, JsonNode> saved = new HashMap<>();
@@ -101,6 +103,27 @@ public class VirtualTree
}
/*******************************************************************************
** Attach generated downloads and the host's report-storage availability.
*******************************************************************************/
public VirtualTree withDownloads(GeneratedDownloads downloads)
{
this.downloads = downloads;
return (this);
}
/*******************************************************************************
** Use the same provider in the web deployment and the virtual outbox folders.
*******************************************************************************/
public VirtualTree withOutbox(com.kingsrook.qbits.filegateway.outbox.OutboxService outbox)
{
this.outbox = outbox;
return (this);
}
/*******************************************************************************
** Cache user metadata for sixty seconds.
@@ -204,9 +227,42 @@ public class VirtualTree
** Resolve permitted app and table folders from UI API metadata.
*******************************************************************************/
private List<Node> children(String path) throws IOException
{
List<Node> nodes = rawChildren(path);
if(Boolean.TRUE.equals(session.account.getValueBoolean("pgpEncryptDownloads")))
{
return (nodes.stream().map(node -> node.directory() || "TEMPORARY".equals(node.kind()) ? node
: new Node(node.path() + ".pgp", false, node.kind(), node.objectId(), 0, node.job())).toList());
}
return (nodes);
}
/*******************************************************************************
** Resolve canonical plaintext paths before applying download encryption names.
*******************************************************************************/
private List<Node> rawChildren(String path) throws IOException
{
JsonNode meta = metadata();
List<Node> result = new ArrayList<>();
List<Node> endpoints = endpointChildren(path, meta);
if(!endpoints.isEmpty() || endpointMount(path, meta) != null)
{
result.addAll(endpoints);
if(!"/".equals(path))
{
return (result);
}
}
if("ENDPOINTS_ONLY".equals(session.account.getValueString("rootMode")))
{
if("/".equals(path))
{
return (result);
}
throw (new NoSuchFileException(path));
}
if("/".equals(path))
{
addAppChildren(result, "", meta.path("appTree"), meta);
@@ -259,7 +315,7 @@ public class VirtualTree
{
folders.add("views");
}
if(meta.path("tables").has("savedReport") && meta.path("processes").has("renderSavedReport"))
if(downloads != null && downloads.reportsAvailable() && meta.path("tables").has("savedReport") && meta.path("processes").has("renderSavedReport"))
{
folders.add("reports");
}
@@ -325,6 +381,130 @@ public class VirtualTree
/*******************************************************************************
** Endpoint targets are resolved from the same user-scoped saved-object queries.
*******************************************************************************/
private List<QRecord> visibleEndpoints(JsonNode meta) throws IOException
{
try
{
List<QRecord> visible = new ArrayList<>();
for(QRecord alias : audit.query(FileGatewayTables.ALIAS, UploadJobs.equal("isActive", true)))
{
if(alias.getValue("accountId") != null && !java.util.Objects.equals(alias.getValue("accountId"), session.account.getValue("id")))
{
continue;
}
String kind = alias.getValueString("kind");
if(List.of("INBOX", "OUTBOX").contains(kind))
{
visible.add(alias);
continue;
}
String table = alias.getValueString("tableName");
if(table == null || !meta.path("tables").has(table))
{
continue;
}
String source = "BULK_LOAD".equals(kind) ? "savedBulkLoadProfile" : "VIEW".equals(kind) ? "savedView" : "savedReport";
if(!meta.path("tables").has(source)
|| ("BULK_LOAD".equals(kind) && (!meta.path("tables").path(table).path("insertPermission").asBoolean(false) || !meta.path("processes").has(table + ".bulkInsert")))
|| ("REPORT".equals(kind) && (downloads == null || !downloads.reportsAvailable() || !meta.path("processes").has("renderSavedReport"))))
{
continue;
}
for(JsonNode row : saved(source, table))
{
if(row.path("values").path("id").asText().equals(alias.getValueString(source + "Id")))
{
alias.withValue("_label", row.path("values").path("label").asText());
visible.add(alias);
break;
}
}
}
return (visible);
}
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
{
throw (new IOException("Unable to resolve endpoints", e));
}
}
/*******************************************************************************
** An exact visible mount, including an empty upload or outbox folder.
*******************************************************************************/
private QRecord endpointMount(String path, JsonNode meta) throws IOException
{
return (visibleEndpoints(meta).stream().filter(alias -> path.equals(alias.getValueString("path"))).findFirst().orElse(null));
}
/*******************************************************************************
** Mount ancestors reveal only the next segment, preserving endpoint chroots.
*******************************************************************************/
private List<Node> endpointChildren(String path, JsonNode meta) throws IOException
{
List<Node> nodes = new ArrayList<>();
String prefix = "/".equals(path) ? "/" : path + "/";
for(QRecord alias : visibleEndpoints(meta))
{
String mount = alias.getValueString("path");
if(mount.startsWith(prefix))
{
String suffix = mount.substring(prefix.length()).split("/")[0];
String child = prefix + suffix;
if(nodes.stream().noneMatch(node -> child.equals(node.path())))
{
nodes.add(new Node(child, true, "ENDPOINT", alias.getValueString("id"), 0, mount.equals(child) ? alias : null));
}
}
if(!mount.equals(path))
{
continue;
}
String kind = alias.getValueString("kind");
if(List.of("BULK_LOAD", "INBOX").contains(kind))
{
nodes.addAll(artifacts("", alias.getValueString("tableName"), path, "upload"));
}
else if("OUTBOX".equals(kind) && outbox != null)
{
try
{
for(QRecord row : outbox.list(alias.getValueInteger("id")))
{
nodes.add(new Node(prefix + row.getValueString("fileName"), false, "OUTBOX", row.getValueString("id"), row.getValueLong("sizeBytes"), row));
}
}
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
{
throw (new IOException("Unable to list outbox", e));
}
}
else if(List.of("VIEW", "REPORT").contains(kind))
{
String format = alias.getValueString("format");
String formats = session.account.getValueString("formats");
if(format != null && List.of((formats == null ? "csv,xlsx" : formats).split("[,\\s]+")).contains(format))
{
String name = alias.getValueString("fileName");
if(name == null || name.isBlank())
{
name = slug(alias.getValueString("_label")) + "." + format;
}
nodes.add(new Node(prefix + name, false, kind, alias.getValueString("VIEW".equals(kind) ? "savedViewId" : "savedReportId"), 0, alias));
}
}
}
return (nodes);
}
/*******************************************************************************
** Sidecars and results expire after seven days; archive objects remain until retention.
*******************************************************************************/
@@ -364,7 +544,7 @@ public class VirtualTree
job.getValueLong("resultsSizeBytes") == null ? 0 : job.getValueLong("resultsSizeBytes"), job));
}
if("archive".equals(folder) && !active && job.getValueString("sha256") != null
&& !job.getValueString("storageKey").startsWith("file-gateway/quarantine/"))
&& job.getValueString("storageKey") != null && !job.getValueString("storageKey").startsWith("file-gateway/quarantine/"))
{
String date = job.getValueInstant("createDate").atZone(ZoneOffset.UTC).toLocalDate().toString();
String directory = prefix + "/archive/" + date;
@@ -412,6 +592,12 @@ public class VirtualTree
{
String parent = path.substring(0, path.lastIndexOf('/'));
Node target = stat(parent);
if(uploads != null && target.job() != null && "ENDPOINT".equals(target.kind())
&& List.of("BULK_LOAD", "INBOX").contains(target.job().getValueString("kind")))
{
QRecord alias = target.job();
return (uploads.receive(session, path, alias.getValueString("tableName"), alias.getValueInteger("savedBulkLoadProfileId"), alias));
}
if(uploads == null || !target.directory() || !"PROFILE".equals(target.kind()) || target.objectId() == null)
{
deny(path, "UPLOAD", "Uploads require a bulk load profile folder");
@@ -424,11 +610,91 @@ public class VirtualTree
/*******************************************************************************
** Read only stored artifacts and generated sidecars; view/report export is phase c.
** Open generated views/reports or stored artifacts without using the filesystem.
*******************************************************************************/
public SeekableByteChannel read(String path) throws IOException
{
Node node = stat(path);
if(Boolean.TRUE.equals(session.account.getValueBoolean("pgpEncryptDownloads")) && !node.directory() && path.endsWith(".pgp"))
{
String plainPath = path.substring(0, path.length() - 4);
Node plain = rawChildren(plainPath.substring(0, plainPath.lastIndexOf('/'))).stream()
.filter(candidate -> candidate.path().equals(plainPath)).findFirst().orElseThrow(() -> new NoSuchFileException(path));
return (encrypted(plain));
}
return (readNode(node));
}
/*******************************************************************************
** Encrypt lazily into the existing fixed rolling window, without a producer thread.
*******************************************************************************/
private SeekableByteChannel encrypted(Node node) throws IOException
{
SeekableByteChannel source = readNode(node);
try
{
var encrypted = com.kingsrook.qbits.filegateway.pgp.PgpStreams.encrypt(java.nio.channels.Channels.newInputStream(source),
session.account.getValueString("pgpPublicKey"), node.path().substring(node.path().lastIndexOf('/') + 1));
boolean[] complete = {false};
var tracked = new java.io.FilterInputStream(encrypted)
{
@Override
public int read(byte[] bytes, int offset, int length) throws IOException
{
int count = in.read(bytes, offset, length);
if(count < 0)
{
complete[0] = true;
}
return (count);
}
};
return (new com.kingsrook.qbits.filegateway.storage.GeneratedReadChannel(tracked, 65536, new com.kingsrook.qbits.filegateway.storage.GeneratedReadChannel.Audit()
{
@Override
public void charge(int bytes)
{
// The underlying transfer accounts for plaintext bytes and applies its limits.
}
@Override
public void finish(IOException failure) throws IOException
{
if(complete[0] && failure == null && "OUTBOX".equals(node.kind()))
{
try
{
outbox.downloaded(node.job());
}
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
{
throw (new IOException("Unable to acknowledge encrypted outbox", e));
}
}
}
}));
}
catch(IOException | RuntimeException e)
{
source.close();
throw (e);
}
}
/*******************************************************************************
** Open a resolved plaintext node through its original authorized delivery path.
*******************************************************************************/
private SeekableByteChannel readNode(Node node) throws IOException
{
String path = node.path();
if(downloads != null && !node.directory() && List.of("VIEW", "REPORT").contains(node.kind()))
{
return (downloads.open(node));
}
if(node.job() == null || node.directory() || "TEMPORARY".equals(node.kind()))
{
deny(path, "DOWNLOAD", "This file is not downloadable");
@@ -445,7 +711,22 @@ public class VirtualTree
return (new ObjectReadChannel(content.length, (start, end) -> new ByteArrayInputStream(content, (int) start, (int) (end - start + 1)), completion));
}
String key = node.job().getValueString("RESULT".equals(node.kind()) ? "resultsStorageKey" : "storageKey");
return (new ObjectReadChannel(node.size(), (start, end) -> store.openRange(key, start, end), completion));
ObjectReadChannel channel = new ObjectReadChannel(node.size(), (start, end) -> store.openRange(key, start, end), completion);
if("OUTBOX".equals(node.kind()) && !Boolean.TRUE.equals(session.account.getValueBoolean("pgpEncryptDownloads")))
{
channel.withFullDownload((bytes, failed) ->
{
try
{
outbox.downloaded(node.job());
}
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
{
throw (new IOException("Unable to acknowledge outbox download", e));
}
});
}
return (channel);
}
@@ -460,7 +741,7 @@ public class VirtualTree
audit.insert(FileGatewayTables.TRANSFER_LOG, new QRecord().withValue("accountId", session.account.getValue("id"))
.withValue("connectionLogId", session.connectionId).withValue("operation", "DOWNLOAD").withValue("path", node.path())
.withValue("objectKind", "SIDECAR".equals(node.kind()) ? "RESULT" : node.kind()).withValue("objectId", node.objectId())
.withValue("uploadJobId", node.job().getValue("id")).withValue("bytes", bytes)
.withValue("uploadJobId", "OUTBOX".equals(node.kind()) ? null : node.job().getValue("id")).withValue("bytes", bytes)
.withValue("sha256", "ARCHIVE".equals(node.kind()) ? node.job().getValue("sha256") : null)
.withValue("durationMs", (System.nanoTime() - began) / 1_000_000).withValue("outcome", failed ? "ERROR" : "OK")
.withValue("clientChannel", "sftp/" + session.id));
@@ -479,6 +760,19 @@ public class VirtualTree
public void remove(String path) throws IOException
{
Node node = stat(path);
if("OUTBOX".equals(node.kind()) && outbox != null)
{
try
{
outbox.remove(node.job());
log(path, "REMOVE", "OK", null, System.nanoTime());
return;
}
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
{
throw (new IOException("Unable to remove outbox file", e));
}
}
if(node.job() == null || !List.of("SIDECAR", "RESULT").contains(node.kind()))
{
deny(path, "REMOVE", "Only sidecars and results can be acknowledged");
@@ -0,0 +1,360 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInProcessSessions;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
import com.kingsrook.qqq.openapi.model.HttpMethod;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.apache.sshd.common.config.keys.KeyUtils;
import org.apache.sshd.sftp.client.SftpClient;
import org.apache.sshd.sftp.client.SftpClientFactory;
import org.apache.sshd.sftp.common.SftpConstants;
import org.apache.sshd.sftp.common.SftpException;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Real SFTP downloads against strict user-authorized UI exports and report processes.
*******************************************************************************/
@ExtendWith(NoDiskExtension.class)
class DownloadIntegrationTest
{
private static final String VIEW = """
{"queryFilter":{"criteria":[{"fieldName":"quantity","operator":"GREATER_THAN","values":[1]}],
"orderBys":[{"fieldName":"quantity","isAscending":false}],"skip":30,"limit":1},
"queryColumns":{"columns":[{"name":"__check__","isVisible":true},{"name":"quantity","isVisible":true},
{"name":"name","isVisible":true},{"name":"id","isVisible":false}]}}
""";
/*******************************************************************************
** Store the same JSON used by Next UI, including selection and hidden columns.
*******************************************************************************/
private QRecord view(FileGatewayTestServer h, String json) throws Exception
{
return (h.audit.system(() -> new InsertAction().execute(new InsertInput("savedView").withRecord(new QRecord()
.withValue("tableName", "alpha").withValue("label", "Selected rows").withValue("userId", "A").withValue("viewJson", json))).getRecords().getFirst()));
}
/*******************************************************************************
** Read until EOF regardless of the generated file's zero stat size.
*******************************************************************************/
private byte[] read(SftpClient client, String path) throws Exception
{
try(var input = client.read(path))
{
return (input.readAllBytes());
}
}
/*******************************************************************************
** Filter, column order, ordering, three formats, hidden tables and transfer identity.
*******************************************************************************/
@Test
void viewFormatsPermissionsAndAudit() throws Exception
{
try(var h = new FileGatewayTestServer())
{
h.seedRows(3);
QRecord saved = view(h, VIEW);
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
var account = h.account("alice", "A");
h.audit.update("file_gateway_account", new QRecord().withValue("id", account.getValue("id")).withValue("formats", "csv,xlsx,json"));
h.key(account, key);
var bob = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(h.account("bob", "B"), bob);
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session);
var deniedSession = h.login("bob", bob); var denied = SftpClientFactory.instance().createSftpClient(deniedSession))
{
String path = "/testapp/alpha/views/selected-rows.";
assertThat(client.stat(path + "csv").getSize()).isZero();
String csv = new String(read(client, path + "csv"), StandardCharsets.UTF_8);
assertThat(csv.replace("\r\n", "\n")).isEqualTo("\"Quantity\",\"Name\"\n\"3\",\"Row 3 " + "x".repeat(50) + "\"\n\"2\",\"Row 2 " + "x".repeat(50) + "\"\n");
try(var workbook = new XSSFWorkbook(new java.io.ByteArrayInputStream(read(client, path + "xlsx"))))
{
assertThat(workbook.getSheetAt(0).getRow(0).getCell(0).getStringCellValue()).isEqualTo("Quantity");
assertThat(workbook.getSheetAt(0).getRow(1).getCell(0).getNumericCellValue()).isEqualTo(3);
}
var json = new ObjectMapper().readTree(read(client, path + "json"));
assertThat(json.toString()).contains("Row 3", "Row 2").doesNotContain("Row 1");
assertThatThrownBy(() -> denied.stat(path + "csv")).isInstanceOfSatisfying(SftpException.class,
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_NO_SUCH_FILE));
assertThatThrownBy(() -> read(denied, path + "csv")).isInstanceOfSatisfying(SftpException.class,
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_NO_SUCH_FILE));
}
var downloads = h.audit.query("file_gateway_transfer_log", new QQueryFilter()).stream()
.filter(r -> "DOWNLOAD".equals(r.getValueString("operation"))).toList();
assertThat(downloads).hasSize(3).allSatisfy(row ->
{
assertThat(row.getValueString("outcome")).isEqualTo("OK");
assertThat(row.getValueString("objectKind")).isEqualTo("VIEW");
assertThat(row.getValueString("objectId")).isEqualTo(saved.getValueString("id"));
assertThat(row.getValueLong("bytes")).isPositive();
assertThat(row.getValueLong("durationMs")).isNotNegative();
assertThat(row.getValueString("format")).isIn("csv", "xlsx", "json");
});
assertThat(h.calls).allSatisfy(call ->
{
assertThat(call.channel()).startsWith("sftp/");
assertThat(call.invokedBy()).isEqualTo("file-gateway");
});
}
}
/*******************************************************************************
** FieldSecurityLock DENY removes the protected value exactly as a direct user export.
*******************************************************************************/
@Test
void fieldSecurityLockMatchesDirectUserExport() throws Exception
{
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance ->
{
instance.setEnforcePermissionsInCoreActions(true);
instance.addSecurityKeyType(new QSecurityKeyType().withName("privateField").withAllAccessKeyName("privateFieldAll"));
instance.getTable("alpha").getField("name").setFieldSecurityLock(new FieldSecurityLock()
.withSecurityKeyType("privateField").withDefaultBehavior(FieldSecurityLock.Behavior.DENY).withOverrideValues(List.of("reader")));
}))
{
h.seedRows(3);
view(h, "{\"queryFilter\":{}}");
h.audit.system(() -> new InsertAction().execute(new InsertInput("savedView").withRecord(new QRecord()
.withValue("tableName", "alpha").withValue("label", "Denied field").withValue("userId", "A").withValue("viewJson", VIEW))));
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(h.account("alice", "A"), key);
var user = h.audit.system(() -> new QAuthenticationModuleDispatcher().getQModule(h.instance.getAuthentication()).createAutomatedSessionForUser(h.instance, "A"));
String token = UiApiInProcessSessions.register(user, Duration.ofMinutes(5));
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
{
var direct = h.runtime.getInvoker().invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/table/{tableName}/export")
.withPathParam("tableName", "alpha").withCredentials(UiApiCredentials.ofInProcessSession(token)).withClientDetail("direct-comparison")
.withJsonBody(Map.of("format", "csv", "filter", Map.of(), "includeHeaderRow", true)));
assertThat(direct.isSuccess()).as(direct.getBody()).isTrue();
byte[] actual = read(client, "/testapp/alpha/views/selected-rows.csv");
assertThat(new String(actual, StandardCharsets.UTF_8)).doesNotContain("Row ");
assertThat(actual).isEqualTo(direct.getBody().getBytes(StandardCharsets.UTF_8));
assertThatThrownBy(() -> read(client, "/testapp/alpha/views/denied-field.csv")).isInstanceOf(SftpException.class);
}
finally
{
UiApiInProcessSessions.revoke(token);
}
}
}
/*******************************************************************************
** The daily quota excludes prior UTC days, other accounts and non-download operations.
*******************************************************************************/
@Test
void dailyQuotaUsesFilteredUtcHistory() throws Exception
{
try(var h = new FileGatewayTestServer())
{
h.seedRows(3);
view(h, "{}");
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
var account = h.account("alice", "A");
h.audit.update("file_gateway_account", new QRecord().withValue("id", account.getValue("id")).withValue("maxDownloadMbPerDay", 1));
h.seedTransfer(account, "DOWNLOAD", 5_000_000L, Instant.now().minusSeconds(86400));
h.seedTransfer(account, "UPLOAD", 5_000_000L, Instant.now());
h.seedTransfer(h.account("other", "A"), "DOWNLOAD", 5_000_000L, Instant.now());
h.key(account, key);
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
{
assertThat(read(client, "/testapp/alpha/views/selected-rows.csv")).isNotEmpty();
assertThatThrownBy(() -> read(client, "/testapp/alpha/views/selected-rows.json")).isInstanceOfSatisfying(SftpException.class,
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_NO_SUCH_FILE));
}
}
}
/*******************************************************************************
** Non-success stream responses preserve their text and distinguish denied transfers.
*******************************************************************************/
@ParameterizedTest
@ValueSource(ints = {403, 500})
void apiErrorsAreAudited(int status) throws Exception
{
try(var h = new FileGatewayTestServer())
{
view(h, "{}");
h.exportFailureStatus = status;
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(h.account("alice", "A"), key);
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
{
assertThatThrownBy(() -> read(client, "/testapp/alpha/views/selected-rows.csv"))
.isInstanceOf(SftpException.class).hasMessageContaining("Export failure test");
}
assertThat(h.audit.query("file_gateway_transfer_log", new QQueryFilter())).anySatisfy(row ->
{
assertThat(row.getValueString("operation")).isEqualTo("DOWNLOAD");
assertThat(row.getValueString("outcome")).isEqualTo(status == 403 ? "DENIED" : "ERROR");
assertThat(row.getValueString("message")).contains("Export failure test");
assertThat(row.getValueLong("bytes")).isZero();
});
}
}
/*******************************************************************************
** Real renderSavedReport input/execute/output and storage download as the SSH user.
*******************************************************************************/
@Test
void savedReportDownload() throws Exception
{
try(var h = new FileGatewayTestServer())
{
h.seedRows(3);
QRecord report = h.audit.system(() -> new InsertAction().execute(new InsertInput("savedReport").withRecord(new QRecord()
.withValue("tableName", "alpha").withValue("label", "Daily").withValue("userId", "A")
.withValue("columnsJson", "{\"columns\":[{\"name\":\"quantity\"}]}")
.withValue("queryFilterJson", "{\"criteria\":[{\"fieldName\":\"quantity\",\"operator\":\"EQUALS\",\"values\":[2]}]}"))).getRecords().getFirst());
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(h.account("alice", "A"), key);
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
{
String csv = new String(read(client, "/testapp/alpha/reports/daily.csv"), StandardCharsets.UTF_8);
assertThat(csv.replace("\r\n", "\n")).isEqualTo("\"Quantity\"\n\"2\"\n");
}
assertThat(h.calls).anySatisfy(call -> assertThat(call.path()).contains("/processes/renderSavedReport/init"))
.anySatisfy(call -> assertThat(call.path()).contains("/download/"))
.allSatisfy(call -> assertThat(call.channel()).startsWith("sftp/"));
assertThat(h.audit.query("file_gateway_transfer_log", new QQueryFilter())).anySatisfy(row ->
{
assertThat(row.getValueString("objectKind")).isEqualTo("REPORT");
assertThat(row.getValueString("objectId")).isEqualTo(report.getValueString("id"));
assertThat(row.getValueString("outcome")).isEqualTo("OK");
});
}
}
/*******************************************************************************
** Missing configured storage hides the report folder, including guessed paths.
*******************************************************************************/
@Test
void missingReportStorageHidesFolder() throws Exception
{
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance -> instance.getTables().remove("reportStorage")))
{
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(h.account("alice", "A"), key);
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
{
List<String> names = new ArrayList<>();
client.readDir("/testapp/alpha").forEach(entry -> names.add(entry.getFilename()));
assertThat(names).doesNotContain("reports");
assertThatThrownBy(() -> client.stat("/testapp/alpha/reports")).isInstanceOfSatisfying(SftpException.class,
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_NO_SUCH_FILE));
}
}
}
/*******************************************************************************
** Large exports are consumed with fixed buffers; caps fail during read and audit LIMIT.
*******************************************************************************/
@ParameterizedTest
@ValueSource(strings = {"large", "generated", "daily"})
void streamingAndLimits(String scenario) throws Exception
{
try(var h = new FileGatewayTestServer())
{
h.seedRows(50000);
view(h, "{\"fieldNames\":[\"name\"],\"queryFilter\":{\"orderBys\":[{\"fieldName\":\"id\",\"isAscending\":true}]}}");
h.config.withDownloadReadAheadMb(1).withMaxGeneratedFileMb(scenario.equals("generated") ? 1 : 20);
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
var account = h.account("alice", "A");
if(scenario.equals("daily"))
{
h.audit.update("file_gateway_account", new QRecord().withValue("id", account.getValue("id")).withValue("maxDownloadMbPerDay", 1));
h.audit.insert("file_gateway_transfer_log", new QRecord().withValue("accountId", account.getValue("id"))
.withValue("operation", "DOWNLOAD").withValue("bytes", 900000L).withValue("outcome", "OK")
.withValue("path", "/earlier.csv").withValue("clientChannel", "sftp/earlier"));
}
h.key(account, key);
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
{
if(scenario.equals("large"))
{
try(var input = client.read("/testapp/alpha/views/selected-rows.csv"))
{
byte[] buffer = new byte[32768];
long bytes = 0;
int lines = 0;
for(int count; (count = input.read(buffer)) >= 0; )
{
bytes += count;
for(int i = 0; i < count; i++)
{
if(buffer[i] == '\n')
{
lines++;
}
}
}
assertThat(bytes).isGreaterThan(3_000_000);
assertThat(lines).isEqualTo(50001);
}
}
else
{
assertThatThrownBy(() -> read(client, "/testapp/alpha/views/selected-rows.csv")).isInstanceOf(java.io.IOException.class)
.hasMessageContaining("limit");
}
}
assertThat(h.audit.query("file_gateway_transfer_log", new QQueryFilter())).anySatisfy(row ->
{
assertThat(row.getValueString("objectKind")).isEqualTo("VIEW");
assertThat(row.getValueString("outcome")).isEqualTo(scenario.equals("large") ? "OK" : "LIMIT");
assertThat(row.getValueLong("bytes")).isPositive().isLessThanOrEqualTo(scenario.equals("large") ? 4_000_000L : 1_048_576L);
});
}
}
}
@@ -40,7 +40,7 @@ class EntityContractTest
** Verify annotations, metadata completeness and fluent accessors against QRecord conversion.
*******************************************************************************/
@ParameterizedTest
@ValueSource(strings = {"Account", "SshKey", "Alias", "UploadJob", "OutboxFile", "TransferLog", "ConnectionLog"})
@ValueSource(strings = {"Account", "SshKey", "Alias", "UploadJob", "OutboxFile", "TransferLog", "ConnectionLog", "Server"})
void entityRoundTrip(String suffix) throws Exception
{
Class<?> type = Class.forName("com.kingsrook.qbits.filegateway.model.FileGateway" + suffix);
@@ -0,0 +1,159 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import java.time.Clock;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Random;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import org.apache.sshd.common.config.keys.KeyUtils;
/*******************************************************************************
** Demo server for screenshots: the full test instance with the Next UI on
** port 8099 (or the first argument), MOCK auth granting every permission to
** browser sessions, and realistic gateway data. Test scope only.
*******************************************************************************/
public final class FileGatewayDemoServer
{
private static final List<String> FILES = List.of("orders-2026-10-13.csv", "inventory.xlsx", "returns.csv", "price-update.csv", "partners.csv", "shipments-week41.csv");
/*******************************************************************************
** Utility class.
*******************************************************************************/
private FileGatewayDemoServer()
{
}
/*******************************************************************************
** Start, seed and keep running until killed.
*******************************************************************************/
public static void main(String[] args) throws Exception
{
int port = args.length > 0 ? Integer.parseInt(args[0]) : 8099;
FileGatewayTestServer.TestAuthentication.grantAllToUiSessions = true;
FileGatewayTestServer server = start(port);
System.out.println("File gateway demo: http://localhost:" + port + "/app/file_gateway");
Thread.currentThread().join();
server.close();
}
/*******************************************************************************
** Start the server and seed demo data.
*******************************************************************************/
static FileGatewayTestServer start(int port) throws Exception
{
FileGatewayTestServer server = new FileGatewayTestServer(Clock.systemUTC(), instance ->
{
}, port, true, true);
seed(server);
return (server);
}
/*******************************************************************************
** Seed accounts, keys, endpoints, jobs, transfers, connections and outbox files.
*******************************************************************************/
static void seed(FileGatewayTestServer h) throws Exception
{
Random random = new Random(42);
Instant now = Instant.now();
QRecord acme = h.audit.insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", "acme-orders").withValue("label", "Acme Corp (orders feed)")
.withValue("accountType", "SERVICE").withValue("userId", "acme-svc").withValue("ownerUserId", "jamie").withValue("allowedCidrs", "203.0.113.0/24"));
QRecord globex = h.audit.insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", "globex").withValue("label", "Globex Logistics")
.withValue("accountType", "SERVICE").withValue("userId", "globex-svc").withValue("ownerUserId", "jamie"));
QRecord jamie = h.audit.insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", "jamie").withValue("label", "Jamie Rivera")
.withValue("accountType", "USER").withValue("userId", "jamie"));
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", globex.getValue("id")).withValue("failedLoginCount", 5)
.withValue("lockedUntil", now.plus(25, ChronoUnit.MINUTES)));
List<QRecord> accounts = List.of(acme, globex, jamie);
for(QRecord account : accounts)
{
h.key(account, KeyUtils.generateKeyPair("ssh-ed25519", 256));
}
QRecord inbox = h.audit.insert(FileGatewayTables.ALIAS, new QRecord().withValue("name", "acme-orders-in").withValue("path", "/acme/orders-in")
.withValue("kind", "INBOX").withValue("consumerName", "bulk-load").withValue("accountId", acme.getValue("id")).withValue("description", "Acme drops nightly order files here"));
QRecord outbox = h.audit.insert(FileGatewayTables.ALIAS, new QRecord().withValue("name", "globex-acks").withValue("path", "/globex/acks")
.withValue("kind", "OUTBOX").withValue("accountId", globex.getValue("id")).withValue("description", "Acknowledgements Globex picks up"));
h.audit.insert(FileGatewayTables.ALIAS, new QRecord().withValue("name", "alpha-daily").withValue("path", "/reports/alpha-daily")
.withValue("kind", "VIEW").withValue("tableName", "alpha").withValue("format", "csv").withValue("fileName", "alpha-daily.csv"));
List<String> statuses = List.of("SUCCEEDED", "SUCCEEDED", "SUCCEEDED", "SUCCEEDED", "PARTIAL", "FAILED", "REJECTED", "DUPLICATE", "QUEUED", "RUNNING");
for(int i = 0; i < 40; i++)
{
QRecord account = accounts.get(i % accounts.size());
String status = statuses.get(random.nextInt(statuses.size()));
String file = FILES.get(random.nextInt(FILES.size()));
Instant finished = now.minus(random.nextInt(7 * 24 * 60), ChronoUnit.MINUTES);
int inserted = "FAILED".equals(status) || "REJECTED".equals(status) ? 0 : 50 + random.nextInt(900);
int errors = "PARTIAL".equals(status) ? 1 + random.nextInt(12) : 0;
h.audit.insert(FileGatewayTables.UPLOAD_JOB, new QRecord().withValue("accountId", account.getValue("id"))
.withValue("aliasId", account == acme ? inbox.getValue("id") : null).withValue("tableName", "alpha").withValue("consumerName", "bulk-load")
.withValue("path", "/testapp/alpha/upload/orders/" + file).withValue("fileName", file).withValue("sizeBytes", 20_000L + random.nextInt(4_000_000))
.withValue("sha256", Long.toHexString(random.nextLong()) + Long.toHexString(random.nextLong())).withValue("status", status)
.withValue("insertedCount", inserted).withValue("errorCount", errors).withValue("remoteAddress", "203.0.113." + (10 + random.nextInt(40)))
.withValue("errorMessage", "FAILED".equals(status) ? "this profile needs value mappings saved in it" : "REJECTED".equals(status) ? "Virus scan: infected" : null)
.withValue("queuedAt", finished.minusSeconds(40)).withValue("startedAt", finished.minusSeconds(30))
.withValue("finishedAt", List.of("QUEUED", "RUNNING").contains(status) ? null : finished).withValue("attempts", 1));
}
for(int i = 0; i < 60; i++)
{
QRecord account = accounts.get(random.nextInt(accounts.size()));
String op = List.of("LIST", "UPLOAD", "DOWNLOAD", "LIST", "DOWNLOAD").get(random.nextInt(5));
h.seedTransfer(account, op, "LIST".equals(op) ? 0 : 10_000L + random.nextInt(8_000_000), now.minus(random.nextInt(6 * 24 * 60), ChronoUnit.MINUTES));
}
for(int i = 0; i < 12; i++)
{
QRecord account = accounts.get(i % accounts.size());
boolean live = i < 2;
Instant start = now.minus(live ? 5 + i * 7 : 60 + random.nextInt(5000), ChronoUnit.MINUTES);
h.audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("accountId", account.getValue("id")).withValue("username", account.getValueString("username"))
.withValue("remoteIp", "203.0.113." + (10 + i)).withValue("clientVersion", "SSH-2.0-OpenSSH_9.8").withValue("outcome", i == 5 ? "AUTH_FAILED" : "CONNECTED")
.withValue("sessionId", "demo-" + i).withValue("podName", "file-gateway-" + (i % 2 == 0 ? "7f9c4-abcde" : "7f9c4-fghij")).withValue("connectedAt", start)
.withValue("lastSeenAt", live ? now : start.plus(4, ChronoUnit.MINUTES)).withValue("disconnectedAt", live ? null : start.plus(4, ChronoUnit.MINUTES))
.withValue("bytesIn", (long) random.nextInt(9_000_000)).withValue("bytesOut", (long) random.nextInt(2_000_000)).withValue("operationCount", (long) (3 + random.nextInt(40)))
.withValue("clientChannel", "sftp"));
}
for(String name : List.of("ack-1041.csv", "ack-1042.csv", "ack-1043.csv"))
{
h.store.put("file-gateway/outbox/" + outbox.getValue("id") + "/" + name, new java.io.ByteArrayInputStream("order,status\n1041,received\n".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
h.audit.insert(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("aliasId", outbox.getValue("id")).withValue("fileName", name).withValue("sizeBytes", 26L)
.withValue("storageKey", "file-gateway/outbox/" + outbox.getValue("id") + "/" + name).withValue("status", name.endsWith("3.csv") ? "READY" : "DOWNLOADED"));
}
}
}
@@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** The demo server seeds and serves (smoke test; the demo itself is manual).
*******************************************************************************/
class FileGatewayDemoServerTest
{
/*******************************************************************************
** Reset the demo-only grant.
*******************************************************************************/
@AfterEach
void reset()
{
FileGatewayTestServer.TestAuthentication.grantAllToUiSessions = false;
}
/*******************************************************************************
** Start on a free port, check seeded data, stop.
*******************************************************************************/
@Test
void startsAndSeeds() throws Exception
{
try(FileGatewayTestServer server = FileGatewayDemoServer.start(0))
{
assertThat(server.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).hasSize(3);
assertThat(server.audit.query(FileGatewayTables.UPLOAD_JOB, new QQueryFilter())).hasSize(40);
assertThat(server.audit.query(FileGatewayTables.OUTBOX_FILE, new QQueryFilter())).hasSize(3);
}
}
}
@@ -22,6 +22,8 @@ package com.kingsrook.qbits.filegateway;
import java.util.List;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData;
import org.junit.jupiter.api.Test;
@@ -52,8 +54,8 @@ class FileGatewayQBitProducerTest
assertThat(instance.getApp("file_gateway").getSections().stream().flatMap(s -> s.getTables().stream()).toList())
.containsExactlyElementsOf(FileGatewayTables.MAIN);
assertThat(instance.getJoins()).hasSize(6);
assertThat(instance.getWidgets()).hasSize(6);
assertThat(instance.getPossibleValueSources()).hasSize(19);
assertThat(instance.getWidgets()).hasSize(16);
assertThat(instance.getPossibleValueSources()).hasSize(20);
instance.getTables().values().forEach(t ->
{
assertThat(t.getBackendName()).isEqualTo("host-db");
@@ -73,6 +75,21 @@ class FileGatewayQBitProducerTest
/*******************************************************************************
** The three gateway permission names reach QQQ's catalog, so identity
** policies can grant them.
*******************************************************************************/
@Test
void permissionsAreInTheAvailableCatalog() throws Exception
{
var instance = new QInstance();
var config = new FileGatewayQBitConfig().withBackendName("host-db");
new FileGatewayQBitProducer().withQBitConfig(config).withNamespace("test").produce(instance).addSelfToInstance(instance);
assertThat(PermissionsHelper.getAllAvailablePermissionNames(instance)).containsAll(FileGatewayPermissions.ALL);
}
/*******************************************************************************
** Package scanning never enables an unconfigured producer.
*******************************************************************************/
@@ -100,6 +117,8 @@ class FileGatewayQBitProducerTest
void contingentPickerMetadata(String source) throws Exception
{
var instance = new QInstance();
instance.addPossibleValueSource(new com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource().withName(source)
.withType(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType.TABLE).withTableName(source));
new FileGatewayQBitProducer().withQBitConfig(new FileGatewayQBitConfig()).produce(instance);
var field = instance.getTable(FileGatewayTables.ALIAS).getField(source + "Id");
assertThat(field.getPossibleValueSourceName()).isEqualTo(source);
@@ -112,4 +131,27 @@ class FileGatewayQBitProducerTest
});
assertThat(filter.getSubFilters()).isNullOrEmpty();
}
/*******************************************************************************
** A host without saved views, reports or bulk-load profiles still validates:
** those pickers become plain id fields; present sources keep their pickers.
*******************************************************************************/
@Test
void dropsPickersForMissingHostSources() throws Exception
{
QInstance instance = new QInstance();
com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData alias = FileGatewayTables.tables(new FileGatewayQBitConfig()).stream()
.filter(t -> t.getName().equals(FileGatewayTables.ALIAS)).findFirst().orElseThrow();
instance.addPossibleValueSource(new com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource().withName("savedView")
.withType(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType.TABLE).withTableName("savedView"));
FileGatewayQBitProducer.dropMissingPickers(instance, alias);
assertThat(alias.getField("savedViewId").getPossibleValueSourceName()).isEqualTo("savedView");
assertThat(alias.getField("savedViewId").getPossibleValueSourceFilter()).isNotNull();
assertThat(alias.getField("savedReportId").getPossibleValueSourceName()).isNull();
assertThat(alias.getField("savedReportId").getPossibleValueSourceFilter()).isNull();
assertThat(alias.getField("savedBulkLoadProfileId").getPossibleValueSourceName()).isNull();
assertThat(alias.getField("tableName").getPossibleValueSourceName()).isEqualTo("tables");
}
}
@@ -85,6 +85,7 @@ public class FileGatewayTestServer implements AutoCloseable
public final FileGatewayRuntime runtime;
public final SshClient client;
public final java.util.List<ApiCall> calls = new java.util.concurrent.CopyOnWriteArrayList<>();
public volatile int exportFailureStatus;
/*******************************************************************************
** Non-secret request evidence for the invoker contract assertions.
@@ -112,6 +113,29 @@ public class FileGatewayTestServer implements AutoCloseable
*******************************************************************************/
public FileGatewayTestServer(Clock clock) throws Exception
{
this(clock, instance ->
{
// Most acceptance tests use the default application metadata.
});
}
/*******************************************************************************
** Customize application metadata before validation and server startup.
*******************************************************************************/
public FileGatewayTestServer(Clock clock, java.util.function.Consumer<QInstance> customize) throws Exception
{
this(clock, customize, 0, false, true);
}
/*******************************************************************************
** Demo and web-only tests can serve Next without starting an SFTP listener.
*******************************************************************************/
public FileGatewayTestServer(Clock clock, java.util.function.Consumer<QInstance> customize, int port, boolean frontend, boolean sftp) throws Exception
{
config.withSftpEnabled(sftp);
ConnectionManager.resetConnectionProviders();
GRANTS.clear();
GRANTS.put("A", Set.of("file_gateway.connect", "alpha.read", "alpha.insert", "beta.read", "beta.insert"));
@@ -167,9 +191,11 @@ public class FileGatewayTestServer implements AutoCloseable
}
instance.addBackend(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData()
.withName("memory").withBackendType(com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule.class));
instance.getTable("reportStorage").setBackendName("memory");
instance.addTable(new QTableMetaData().withName("testUploadArchive").withBackendName("memory").withPrimaryKeyField("reference")
.withField(new QFieldMetaData("reference", QFieldType.STRING)).withField(new QFieldMetaData("contents", QFieldType.BLOB)));
instance.withSupplementalMetaData(new com.kingsrook.qqq.middleware.javalin.QJavalinMetaData().withUploadedFileArchiveTableName("testUploadArchive"));
customize.accept(instance);
new QInstanceEnricher(instance).enrich();
new QInstanceValidator().validate(instance);
QContext.init(instance, new QSystemUserSession(instance));
@@ -183,17 +209,68 @@ public class FileGatewayTestServer implements AutoCloseable
{
return (instance);
}
}).withPort(0).withServeFrontendMaterialDashboard(false).withServeFrontendNext(false)
.withJavalinConfigCustomizer(j -> j.routes.before(ctx -> calls.add(new ApiCall(ctx.method().name(), ctx.path(), ctx.header("X-QQQ-Client"), ctx.header("X-QQQ-Invoked-By")))))
}).withPort(port).withServeFrontendMaterialDashboard(false).withServeFrontendNext(frontend)
.withJavalinConfigCustomizer(j -> j.routes.before(ctx ->
{
calls.add(new ApiCall(ctx.method().name(), ctx.path(), ctx.header("X-QQQ-Client"), ctx.header("X-QQQ-Invoked-By")));
if(exportFailureStatus != 0 && ctx.path().endsWith("/export"))
{
throw (new io.javalin.http.HttpResponseException(exportFailureStatus, "Export failure test"));
}
}))
.withStrictUiApiRoutes(true).withMiddlewareVersionList(List.of(new MiddlewareVersionV1()));
http.start();
runtime = FileGatewayRuntime.startForTest(instance, http, config, KeyUtils.generateKeyPair("ecdsa-sha2-nistp256", 256), clock, Duration.ofHours(1), store);
runtime = sftp ? FileGatewayRuntime.startForTest(instance, http, config, KeyUtils.generateKeyPair("ecdsa-sha2-nistp256", 256), clock, Duration.ofHours(1), store) : null;
if(!sftp)
{
com.kingsrook.qbits.filegateway.api.FileGatewayOutbox.setProvider(new com.kingsrook.qbits.filegateway.outbox.OutboxService(instance, store, config, clock));
}
client = SshClient.setUpDefaultClient();
client.setServerKeyVerifier(AcceptAllServerKeyVerifier.INSTANCE);
client.start();
}
/*******************************************************************************
** Seed many export rows without generating unrelated insert audits.
*******************************************************************************/
public void seedRows(int count) throws Exception
{
try(var statement = keeper.prepareStatement("INSERT INTO alpha (id, name, quantity) VALUES (?, ?, ?)"))
{
for(int i = 1; i <= count; i++)
{
statement.setInt(1, i);
statement.setString(2, "Row " + i + " " + "x".repeat(50));
statement.setInt(3, i);
statement.addBatch();
if(i % 1000 == 0)
{
statement.executeBatch();
}
}
statement.executeBatch();
}
}
/*******************************************************************************
** Seed quota history with an exact UTC timestamp rather than the insertion clock.
*******************************************************************************/
public void seedTransfer(QRecord account, String operation, long bytes, java.time.Instant when) throws Exception
{
QRecord row = audit.insert("file_gateway_transfer_log", new QRecord().withValue("accountId", account.getValue("id"))
.withValue("operation", operation).withValue("bytes", bytes).withValue("outcome", "OK")
.withValue("path", "/history.csv").withValue("clientChannel", "sftp/history"));
try(var statement = keeper.prepareStatement("UPDATE file_gateway_transfer_log SET create_date = ? WHERE id = ?"))
{
statement.setObject(1, when.atOffset(java.time.ZoneOffset.UTC).toLocalDateTime());
statement.setInt(2, row.getValueInteger("id"));
statement.executeUpdate();
}
}
/*******************************************************************************
** Create core test tables using their actual metadata.
@@ -293,7 +370,10 @@ public class FileGatewayTestServer implements AutoCloseable
public void close() throws Exception
{
client.stop();
runtime.stop();
if(runtime != null)
{
runtime.stop();
}
http.stop();
keeper.close();
QContext.clear();
@@ -306,6 +386,27 @@ public class FileGatewayTestServer implements AutoCloseable
*******************************************************************************/
public static class TestAuthentication implements QAuthenticationModuleCustomizerInterface
{
/** Demo only: browser (MOCK) sessions get every permission in the instance. */
public static volatile boolean grantAllToUiSessions = false;
/*******************************************************************************
** Browser sessions: nothing, unless the demo server switches it on.
*******************************************************************************/
@Override
public void customizeSession(QInstance instance, QSession session, Map<String, Object> context)
{
if(grantAllToUiSessions)
{
java.util.Set<String> all = new java.util.HashSet<>(com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper.getAllAvailablePermissionNames(instance));
all.addAll(com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions.ALL);
session.setPermissions(all);
}
}
/*******************************************************************************
** Customize automated session for user.
*******************************************************************************/
@@ -0,0 +1,404 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import java.security.KeyPair;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.preferences.FileTransferPreferencesProvider;
import com.kingsrook.qqq.backend.core.actions.preferences.UserPreferencesAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionResult;
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferencesMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInProcessSessions;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInvoker;
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
import com.kingsrook.qqq.openapi.model.HttpMethod;
import org.apache.sshd.common.config.keys.KeyUtils;
import org.apache.sshd.common.config.keys.PublicKeyEntry;
import org.apache.sshd.common.digest.BuiltinDigests;
import org.apache.sshd.sftp.client.SftpClientFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Self-service acceptance through core, strict HTTP routes and real SSH clients.
*******************************************************************************/
@ExtendWith(NoDiskExtension.class)
class FileTransferPreferencesTest
{
private static final String SECTION = FileTransferPreferencesProvider.SECTION_NAME;
private final UserPreferencesAction preferences = new UserPreferencesAction();
/*******************************************************************************
** Connect permission alone reveals the section, independently of the admin app.
*******************************************************************************/
@Test
void visibilityAndEmptyAccount() throws Exception
{
try(var h = new FileGatewayTestServer())
{
user(h, "A");
assertThat(preferences.listSections()).hasSize(1);
var section = preferences.getSection(SECTION);
assertThat(section.getSection().getPermissionName()).isEqualTo("file_gateway.connect");
assertThat(section.getSection().getAppName()).isNull();
assertThat(section.getSection().getOrder()).isEqualTo(60);
assertThat(section.getValues().getValues().get("username")).isEqualTo("");
assertThat(section.getValues().getLists().get("keys")).isEmpty();
assertThat(section.getSection().getLists().getFirst().getDescription()).contains("first key creates");
assertThat(section.getSection().getFields()).noneMatch(field -> "host".equals(field.getName()));
assertThat(section.getValues().getValues().get("port")).isEqualTo(h.runtime.getProtocol().getPort());
assertThat(section.getValues().getValues().get("hostKeyFingerprint")).isNotNull();
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).isEmpty();
user(h, "NONE");
assertThat(preferences.listSections()).isEmpty();
assertThatThrownBy(() -> preferences.getSection(SECTION)).isInstanceOf(QNotFoundException.class);
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of())).isInstanceOf(QNotFoundException.class);
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of())).isInstanceOf(QNotFoundException.class);
}
}
/*******************************************************************************
** Host is optional; heartbeat data comes from the most recent server.
*******************************************************************************/
@Test
void publicHostAndLatestHeartbeat() throws Exception
{
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance -> { }, 0, false, false))
{
assertThat(h.config.getPublicHostName()).isNull();
UserPreferencesMetaData.of(h.instance).getSections().clear();
FileTransferPreferencesProvider.register(h.instance, h.config.withPublicHostName("sftp.example.test"));
user(h, "A");
assertThat(preferences.getSection(SECTION).getValues().getValues()).containsEntry("host", "sftp.example.test").containsEntry("port", null);
h.audit.insert(FileGatewayTables.SERVER, new QRecord().withValue("podName", "new").withValue("port", 2223)
.withValue("lastSeenAt", Instant.now()).withValue("hostKeyFingerprint", "SHA256:new"));
h.audit.insert(FileGatewayTables.SERVER, new QRecord().withValue("podName", "old").withValue("port", 2222)
.withValue("lastSeenAt", Instant.now().minusSeconds(60)).withValue("hostKeyFingerprint", "SHA256:old"));
assertThat(preferences.getSection(SECTION).getValues().getValues()).containsEntry("port", 2223).containsEntry("hostKeyFingerprint", "SHA256:new");
}
}
/*******************************************************************************
** Creation uses schema defaults, reuses the account and preserves audit identity.
*******************************************************************************/
@Test
void createsAndReusesAccountWithAudits() throws Exception
{
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance ->
{
for(String table : List.of(FileGatewayTables.ACCOUNT, FileGatewayTables.SSH_KEY))
{
instance.getTable(table).withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD));
}
instance.getTable(FileGatewayTables.ACCOUNT).getField("maxFileSizeMb").setDefaultValue(321);
instance.setEnforcePermissionsInCoreActions(true);
}))
{
user(h, "A");
var key = pair();
var first = add(key, "Laptop");
assertThat(first.getMessage()).contains(KeyUtils.getFingerPrint(BuiltinDigests.sha256, key.getPublic()));
assertThat(first.getSection().getValues().getValues().get("username")).isEqualTo("user-a");
assertThat(first.getSection().getValues().getLists().get("keys").getFirst().getValues()).containsEntry("keyType", "ssh-ed25519").doesNotContainKey("publicKey");
add(pair(), "Backup");
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).singleElement().satisfies(account ->
{
assertThat(account.getValueString("userId")).isEqualTo("A");
assertThat(account.getValueString("accountType")).isEqualTo("USER");
assertThat(account.getValueString("rootMode")).isEqualTo("FULL_TREE");
assertThat(account.getValueString("status")).isEqualTo("ACTIVE");
assertThat(account.getValueInteger("maxFileSizeMb")).isEqualTo(321);
assertThat(account.getValueInteger("maxConcurrentSessions")).isEqualTo(4);
});
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).hasSize(2);
var audits = h.audit.system(() -> new QueryAction().execute(new QueryInput("audit")).getRecords());
var auditTables = h.audit.system(() -> new QueryAction().execute(new QueryInput("auditTable")).getRecords());
var auditUsers = h.audit.system(() -> new QueryAction().execute(new QueryInput("auditUser")).getRecords());
Object tableId = auditTables.stream().filter(row -> FileGatewayTables.SSH_KEY.equals(row.getValueString("name"))).findFirst().orElseThrow().getValue("id");
Object userId = auditUsers.stream().filter(row -> "Test user A".equals(row.getValueString("name"))).findFirst().orElseThrow().getValue("id");
assertThat(audits).anySatisfy(audit ->
{
assertThat(audit.getValue("auditTableId")).isEqualTo(tableId);
assertThat(audit.getValue("auditUserId")).isEqualTo(userId);
assertThat(audit.getValueString("clientChannel")).isEqualTo("next-ui/preferences-test");
});
assertThat(QContext.getQSession().getUser().getIdReference()).isEqualTo("A");
}
}
/*******************************************************************************
** Invalid first keys roll back the account; duplicates retain table messages.
*******************************************************************************/
@Test
void rejectsWeakDuplicateAndMissingKeys() throws Exception
{
try(var h = new FileGatewayTestServer())
{
user(h, "A");
assertThatThrownBy(() -> add(KeyUtils.generateKeyPair("ssh-rsa", 2048), "Weak"))
.isInstanceOf(QBadRequestException.class).hasMessageContaining("at least 3072 bits");
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).isEmpty();
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of()))
.isInstanceOf(QBadRequestException.class).hasMessageContaining("Public key: is required").hasMessageContaining("Label: is required");
var key = pair();
add(key, "First");
assertThatThrownBy(() -> add(key, "Duplicate")).isInstanceOf(QBadRequestException.class).hasMessageContaining("Fingerprint");
user(h, "B");
assertThatThrownBy(() -> add(key, "Other user duplicate")).isInstanceOf(QBadRequestException.class).hasMessageContaining("Fingerprint");
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).hasSize(1);
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter())).hasSize(1);
}
}
/*******************************************************************************
** Normalization, truncation and suffixes obey the account validator.
*******************************************************************************/
@ParameterizedTest
@ValueSource(strings = {"Alice@EXAMPLE.COM", "A", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"})
void uniquifiesUsername(String id) throws Exception
{
try(var h = new FileGatewayTestServer())
{
user(h, id);
String base = id.toLowerCase(java.util.Locale.ROOT).replaceAll("[^a-z0-9._-]", "-");
base = base.length() < 3 ? "user-" + base : base.substring(0, Math.min(64, base.length()));
h.account(base, "B");
String second = base.substring(0, Math.min(62, base.length())) + "-2";
h.account(second, "B");
var added = add(pair(), "Key");
assertThat(added.getSection().getValues().getValues().get("username")).isEqualTo(base.substring(0, Math.min(62, base.length())) + "-3");
}
}
/*******************************************************************************
** Lists and mutations exclude other users, revoked keys and service accounts.
*******************************************************************************/
@Test
void isolatesKeysAndRejectsIdentityInjection() throws Exception
{
try(var h = new FileGatewayTestServer())
{
var other = h.key(h.account("bob", "B"), pair());
var account = h.account("alice", "A");
var revoked = h.key(account, pair());
h.audit.update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", revoked.getValue("id")).withValue("revokedAt", Instant.now()));
var service = h.account("service", "A");
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", service.getValue("id")).withValue("accountType", "SERVICE"));
var serviceKey = h.key(service, pair());
user(h, "A");
var own = add(pair(), "Own");
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).extracting(row -> row.getKey()).containsExactly(own.getFocusRowKey());
for(QRecord excluded : List.of(other, revoked, serviceKey))
{
assertThatThrownBy(() -> preferences.runAction(SECTION, "revokeKey", Map.of("rowKey", excluded.getValueString("id"))))
.isInstanceOf(QBadRequestException.class);
assertThatThrownBy(() -> patch(excluded.getValueString("id"), Map.of("label", "Stolen")))
.isInstanceOf(QBadRequestException.class).hasMessageContaining("isn't a setting here");
}
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of("userId", "B"))).isInstanceOf(QBadRequestException.class);
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of("userId", "B"))).isInstanceOf(QBadRequestException.class);
assertThatThrownBy(() -> patch(own.getFocusRowKey(), Map.of("accountId", other.getValue("accountId"))))
.isInstanceOf(QBadRequestException.class);
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter())).noneMatch(row -> "Stolen".equals(row.getValueString("label")));
}
}
/*******************************************************************************
** PATCH changes only label/expiry, supports clearing expiry and rejects readonly fields.
*******************************************************************************/
@Test
void editsOnlyMutableFields() throws Exception
{
try(var h = new FileGatewayTestServer())
{
user(h, "A");
String key = add(pair(), "Laptop").getFocusRowKey();
Instant expires = Instant.parse("2030-01-01T00:00:00Z");
patch(key, Map.of("label", "Work laptop", "expiresAt", expires.toString()));
var values = preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues();
assertThat(values).containsEntry("label", "Work laptop").containsEntry("expiresAt", expires);
Map<String, Object> clear = new HashMap<>();
clear.put("expiresAt", null);
patch(key, clear);
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues()).containsEntry("expiresAt", null);
for(String field : List.of("fingerprint", "keyType", "lastUsedAt"))
{
assertThatThrownBy(() -> patch(key, Map.of(field, "2030-01-01T00:00:00Z"))).isInstanceOf(QBadRequestException.class).hasMessageContaining("can't be changed");
}
for(String field : List.of("username", "port", "hostKeyFingerprint"))
{
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of("values", Map.of(field, "22"))))
.isInstanceOf(QBadRequestException.class).hasMessageContaining("can't be changed");
}
assertThatThrownBy(() -> patch(key, Map.of("label", " "))).isInstanceOf(QBadRequestException.class).hasMessageContaining("Label: is required");
assertThatThrownBy(() -> patch(key, Map.of("publicKey", "replacement"))).isInstanceOf(QBadRequestException.class);
}
}
/*******************************************************************************
** A self-service key signs in and lists SFTP; revocation closes and refuses it.
*******************************************************************************/
@Test
void addedKeyLogsInAndRevokedKeyCannot() throws Exception
{
try(var h = new FileGatewayTestServer())
{
user(h, "A");
var pair = pair();
var added = add(pair, "Laptop");
String username = added.getSection().getValues().getValues().get("username").toString();
try(var session = h.login(username, pair); var client = SftpClientFactory.instance().createSftpClient(session))
{
assertThat(client.readDir("/")).anyMatch(entry -> "testapp".equals(entry.getFilename()));
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues().get("lastUsedAt")).isNotNull();
assertThat(preferences.runAction(SECTION, "revokeKey", Map.of("rowKey", added.getFocusRowKey())).getMessage()).isEqualTo("Key revoked.");
h.runtime.getProtocol().getAuthenticator().poll();
session.waitFor(Set.of(org.apache.sshd.client.session.ClientSession.ClientSessionEvent.CLOSED), Duration.ofSeconds(5));
assertThat(session.isClosed()).isTrue();
}
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter()).getFirst().getValue("revokedAt")).isNotNull();
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).isEmpty();
assertThatThrownBy(() -> h.login(username, pair)).isInstanceOf(Exception.class);
}
}
/*******************************************************************************
** The strict UI route exposes actions and translates domain failures to 400.
*******************************************************************************/
@Test
void preferencesThroughStrictUiApi() throws Exception
{
try(var h = new FileGatewayTestServer())
{
QSession session = user(h, "A");
String token = UiApiInProcessSessions.register(session, Duration.ofMinutes(1));
try
{
var invoker = UiApiInvoker.forServer(h.http, "next-ui");
var get = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath("/preferences")
.withCredentials(UiApiCredentials.ofInProcessSession(token)));
assertThat(get.getStatus()).isEqualTo(200);
assertThat(get.getJson().toString()).contains("file-transfer", "addKey", "revokeKey");
var added = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/preferences/{sectionName}/actions/{actionName}")
.withPathParam("sectionName", SECTION).withPathParam("actionName", "addKey").withCredentials(UiApiCredentials.ofInProcessSession(token))
.withJsonBody(Map.of("values", Map.of("publicKey", PublicKeyEntry.toString(pair().getPublic()), "label", "HTTP key"))));
assertThat(added.getStatus()).as(added.getBody()).isEqualTo(200);
assertThat(added.getJson().toString()).contains("SHA256:", "HTTP key");
var invalid = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/preferences/{sectionName}/actions/{actionName}")
.withPathParam("sectionName", SECTION).withPathParam("actionName", "addKey").withCredentials(UiApiCredentials.ofInProcessSession(token))
.withJsonBody(Map.of("values", Map.of("publicKey", "invalid", "label", "Invalid"))));
assertThat(invalid.getStatus()).isEqualTo(400);
assertThat(invalid.getBody()).contains("at least 3072 bits");
}
finally
{
UiApiInProcessSessions.revoke(token);
}
}
}
/*******************************************************************************
** Set a real non-system session with only the user's normal permissions.
*******************************************************************************/
private QSession user(FileGatewayTestServer h, String id) throws Exception
{
QSession session = h.audit.system(() -> new QAuthenticationModuleDispatcher().getQModule(h.instance.getAuthentication()).createAutomatedSessionForUser(h.instance, id));
session.setPermissions(FileGatewayTestServer.GRANTS.getOrDefault(id, Set.of("file_gateway.connect")));
session.setClientChannel("next-ui/preferences-test");
QContext.init(h.instance, session);
return (session);
}
/*******************************************************************************
** Generate a supported public key entirely in memory.
*******************************************************************************/
private KeyPair pair() throws Exception
{
return (KeyUtils.generateKeyPair("ssh-ed25519", 256));
}
/*******************************************************************************
** Exercise registered action input validation rather than calling the provider.
*******************************************************************************/
private UserPreferenceActionResult add(KeyPair key, String label) throws Exception
{
return (preferences.runAction(SECTION, "addKey", Map.of("values", Map.of("publicKey", PublicKeyEntry.toString(key.getPublic()), "label", label))));
}
/*******************************************************************************
** Send the same partial row body as a preferences client.
*******************************************************************************/
private void patch(String key, Map<String, ?> values) throws Exception
{
preferences.patchSection(SECTION, Map.of("lists", Map.of("keys", List.of(Map.of("key", key, "values", values)))));
}
}
@@ -0,0 +1,199 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.NonWritableChannelException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import com.kingsrook.qbits.filegateway.storage.GeneratedReadChannel;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Fixed-window stream semantics independently of network packet boundaries and file size.
*******************************************************************************/
class GeneratedReadChannelTest
{
/*******************************************************************************
** Capture charges and finalization without involving a database.
*******************************************************************************/
private GeneratedReadChannel.Audit audit(AtomicInteger bytes, AtomicReference<IOException> failure, AtomicInteger closes)
{
return (new GeneratedReadChannel.Audit()
{
/*******************************************************************************
** Track each newly generated byte once, including forward read-ahead.
*******************************************************************************/
public void charge(int count)
{
bytes.addAndGet(count);
}
/*******************************************************************************
** Capture final outcome.
*******************************************************************************/
public void finish(IOException error)
{
failure.set(error);
closes.incrementAndGet();
}
});
}
/*******************************************************************************
** Forward gaps and backward reads inside the window reuse bytes without regeneration.
*******************************************************************************/
@Test
void reorderedReadsAndEviction() throws Exception
{
byte[] source = new byte[80];
for(int i = 0; i < source.length; i++)
{
source[i] = (byte) i;
}
AtomicInteger bytes = new AtomicInteger();
AtomicInteger closes = new AtomicInteger();
AtomicReference<IOException> failure = new AtomicReference<>();
try(var channel = new GeneratedReadChannel(new ByteArrayInputStream(source), 16, audit(bytes, failure, closes)))
{
assertThat(channel.size()).isZero();
channel.position(8);
ByteBuffer data = ByteBuffer.allocate(4);
assertThat(channel.read(data)).isEqualTo(4);
assertThat(data.array()).containsExactly(8, 9, 10, 11);
channel.position(0);
data = ByteBuffer.allocate(8);
assertThat(channel.read(data)).isEqualTo(8);
assertThat(data.array()).containsExactly(0, 1, 2, 3, 4, 5, 6, 7);
assertThat(bytes.get()).isEqualTo(12);
channel.position(12);
assertThat(channel.read(ByteBuffer.allocate(20))).isEqualTo(20);
assertThat(channel.position()).isEqualTo(32);
channel.position(16);
assertThat(channel.read(ByteBuffer.allocate(4))).isEqualTo(4);
channel.position(15);
assertThatThrownBy(() -> channel.read(ByteBuffer.allocate(1))).isInstanceOf(IOException.class).hasMessageContaining("download sequentially");
assertThatThrownBy(() -> channel.read(ByteBuffer.allocate(1))).isInstanceOf(IOException.class);
}
assertThat(closes.get()).isEqualTo(1);
assertThat(failure.get()).hasMessageContaining("download sequentially");
assertThat(bytes.get()).isEqualTo(32);
}
/*******************************************************************************
** Tiny upstream packets cannot falsely signal EOF; only one fixed window is retained.
*******************************************************************************/
@Test
void fullReadsEofAndLifecycle() throws Exception
{
AtomicInteger closes = new AtomicInteger();
AtomicInteger streamCloses = new AtomicInteger();
AtomicInteger bytes = new AtomicInteger();
AtomicReference<IOException> failure = new AtomicReference<>();
InputStream input = new InputStream()
{
private int remaining = 100003;
/*******************************************************************************
** Supply bytes one at a time to simulate fragmented HTTP reads.
*******************************************************************************/
public int read()
{
return (remaining-- > 0 ? 7 : -1);
}
/*******************************************************************************
** Verify stream ownership.
*******************************************************************************/
public void close()
{
streamCloses.incrementAndGet();
}
};
var channel = new GeneratedReadChannel(input, 17, audit(bytes, failure, closes));
ByteBuffer buffer = ByteBuffer.allocate(1024);
long total = 0;
int count;
while((count = channel.read(buffer.clear())) >= 0)
{
total += count;
assertThat(count).isEqualTo(total == 100003 ? 675 : 1024);
}
assertThat(total).isEqualTo(100003);
assertThat(bytes.get()).isEqualTo(100003);
assertThat(channel.read(ByteBuffer.allocate(0))).isZero();
assertThatThrownBy(() -> channel.position(-1)).isInstanceOf(IllegalArgumentException.class);
assertThatThrownBy(() -> channel.write(buffer)).isInstanceOf(NonWritableChannelException.class);
assertThatThrownBy(() -> channel.truncate(0)).isInstanceOf(NonWritableChannelException.class);
channel.close();
channel.close();
assertThat(channel.isOpen()).isFalse();
assertThat(closes.get()).isEqualTo(1);
assertThat(streamCloses.get()).isEqualTo(1);
assertThat(failure.get()).isNull();
assertThatThrownBy(channel::position).isInstanceOf(ClosedChannelException.class);
assertThatThrownBy(channel::size).isInstanceOf(ClosedChannelException.class);
assertThatThrownBy(() -> channel.read(buffer)).isInstanceOf(ClosedChannelException.class);
}
/*******************************************************************************
** Invalid windows, excessive forward seeks and underlying read failures are explicit.
*******************************************************************************/
@Test
void failuresCloseAndAudit() throws Exception
{
AtomicInteger closes = new AtomicInteger();
AtomicReference<IOException> failure = new AtomicReference<>();
var audit = audit(new AtomicInteger(), failure, closes);
assertThatThrownBy(() -> new GeneratedReadChannel(InputStream.nullInputStream(), 0, audit)).isInstanceOf(IllegalArgumentException.class);
try(var channel = new GeneratedReadChannel(InputStream.nullInputStream(), 8, audit))
{
channel.position(8);
assertThatThrownBy(() -> channel.read(ByteBuffer.allocate(1))).hasMessageContaining("sequentially");
}
try(var channel = new GeneratedReadChannel(new InputStream()
{
/*******************************************************************************
** Simulate an HTTP transport failure.
*******************************************************************************/
public int read() throws IOException
{
throw (new IOException("HTTP connection interrupted"));
}
}, 8, audit))
{
assertThatThrownBy(() -> channel.read(ByteBuffer.allocate(1))).hasMessageContaining("HTTP connection interrupted");
}
assertThat(failure.get()).hasMessageContaining("HTTP connection interrupted");
assertThat(closes.get()).isEqualTo(2);
}
}
@@ -0,0 +1,157 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.filegateway.maintenance.GatewayMaintenance;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayProcesses;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.apache.sshd.common.config.keys.KeyUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Retention and administrative effects operate on durable shared database state.
*******************************************************************************/
@ExtendWith(NoDiskExtension.class)
class MaintenanceIntegrationTest
{
@Test
void everyRetentionRuleAndFreshRowsSurvive() throws Exception
{
Instant now = Instant.parse("2026-10-13T12:00:00Z");
try(var h = new FileGatewayTestServer(Clock.fixed(now, ZoneOffset.UTC)))
{
var account = h.account("alice", "A");
for(String status : List.of("SUCCEEDED", "RECEIVING", "RUNNING", "QUEUED"))
{
String key = "retention/" + status;
h.store.put(key, new java.io.ByteArrayInputStream(new byte[]{1}));
h.store.put(key + "-results", new java.io.ByteArrayInputStream(new byte[]{2}));
h.store.put(key + "-plain", new java.io.ByteArrayInputStream(new byte[]{3}));
var job = h.audit.insert(FileGatewayTables.UPLOAD_JOB, new QRecord().withValue("accountId", account.getValue("id"))
.withValue("status", status).withValue("fileName", status)
.withValue("storageKey", key).withValue("resultsStorageKey", key + "-results").withValue("plaintextStorageKey", key + "-plain"));
backdate(h, FileGatewayTables.UPLOAD_JOB, job, now.minusSeconds(100 * 86400L));
}
for(String table : List.of(FileGatewayTables.TRANSFER_LOG, FileGatewayTables.CONNECTION_LOG))
{
backdate(h, table, h.audit.insert(table, new QRecord().withValue("lastSeenAt", now.minusSeconds(181 * 86400L))), now.minusSeconds(181 * 86400L));
backdate(h, table, h.audit.insert(table, new QRecord().withValue("lastSeenAt", now)), now);
}
h.audit.insert(FileGatewayTables.SERVER, new QRecord().withValue("podName", "dead-pod").withValue("lastSeenAt", now.minusSeconds(86401)));
var alias = PhaseEIntegrationTest.alias(h, "out", "/out", "OUTBOX", account);
h.store.put("outbox/old", new java.io.ByteArrayInputStream(new byte[]{4}));
var out = h.audit.insert(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("aliasId", alias.getValue("id")).withValue("status", "READY")
.withValue("storageKey", "outbox/old"));
backdate(h, FileGatewayTables.OUTBOX_FILE, out, now.minusSeconds(31 * 86400L));
GatewayMaintenance.maintain(h.instance, h.config, h.store, Clock.fixed(now, ZoneOffset.UTC));
var jobs = UploadIntegrationTest.jobs(h);
assertThat(jobs).anySatisfy(row ->
{
assertThat(row.getValueString("status")).isEqualTo("ABORTED");
assertThat(row.getValue("storageKey")).isNull();
});
assertThat(h.store.objects).doesNotContainKeys("retention/SUCCEEDED", "retention/SUCCEEDED-results", "retention/SUCCEEDED-plain", "retention/RECEIVING", "outbox/old")
.containsKeys("retention/RUNNING", "retention/QUEUED");
assertThat(h.audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", out.getValue("id"))).getFirst().getValueString("status")).isEqualTo("EXPIRED");
assertThat(h.audit.query(FileGatewayTables.TRANSFER_LOG, new QQueryFilter())).hasSize(1);
assertThat(h.audit.query(FileGatewayTables.CONNECTION_LOG, new QQueryFilter())).hasSize(1);
assertThat(h.audit.query(FileGatewayTables.SERVER, new QQueryFilter())).hasSize(1);
GatewayMaintenance.maintain(h.instance, h.config, h.store, Clock.fixed(now, ZoneOffset.UTC));
assertThat(GatewayMaintenance.metadata(h.config.withSchedulerName("scheduler")).getSchedule().getRepeatSeconds()).isEqualTo(86400);
}
}
/*******************************************************************************
** Inserts stamp createDate with the real time; move a row's createDate after.
*******************************************************************************/
private static void backdate(FileGatewayTestServer h, String table, QRecord row, Instant createDate) throws Exception
{
h.audit.update(table, new QRecord().withValue("id", row.getValue("id")).withValue("createDate", createDate));
assertThat(h.audit.query(table, UploadJobs.equal("id", row.getValue("id"))).getFirst().getValueInstant("createDate")).isEqualTo(createDate);
}
@Test
void reviewResultAndLiveRevocation() throws Exception
{
try(var h = new FileGatewayTestServer())
{
var account = h.account("alice", "A");
var keyPair = KeyUtils.generateKeyPair("ssh-ed25519", 256);
var key = h.key(account, keyPair);
QContext.init(h.instance, new QSession().withPermissions(Set.of("file_gateway.admin")));
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("lockedUntil", Instant.now().plusSeconds(600)).withValue("failedLoginCount", 5));
apply(h, FileGatewayProcesses.UNLOCK, account);
var unlocked = h.audit.query(FileGatewayTables.ACCOUNT, UploadJobs.equal("id", account.getValue("id"))).getFirst();
assertThat(unlocked.getValue("lockedUntil")).isNull();
assertThat(unlocked.getValueInteger("failedLoginCount")).isZero();
try(var ssh = h.login("alice", keyPair))
{
var row = h.audit.query(FileGatewayTables.CONNECTION_LOG, UploadJobs.equal("accountId", account.getValue("id"))).getFirst();
apply(h, FileGatewayProcesses.END_SESSION, row);
assertThat(h.audit.query(FileGatewayTables.CONNECTION_LOG, UploadJobs.equal("id", row.getValue("id"))).getFirst().getValue("disconnectedAt")).isNull();
h.runtime.getProtocol().getAuthenticator().poll();
ssh.waitFor(java.util.EnumSet.of(org.apache.sshd.client.session.ClientSession.ClientSessionEvent.CLOSED), java.time.Duration.ofSeconds(5));
assertThat(ssh.isClosed()).isTrue();
}
try(var ssh = h.login("alice", keyPair))
{
apply(h, FileGatewayProcesses.DISABLE, account);
h.runtime.getProtocol().getAuthenticator().poll();
ssh.waitFor(java.util.EnumSet.of(org.apache.sshd.client.session.ClientSession.ClientSessionEvent.CLOSED), java.time.Duration.ofSeconds(5));
assertThat(ssh.isClosed()).isTrue();
}
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("status", "ACTIVE"));
apply(h, FileGatewayProcesses.REVOKE, key);
assertThatThrownBy(() -> h.login("alice", keyPair)).isInstanceOf(Exception.class);
}
}
private void apply(FileGatewayTestServer h, String action, QRecord row) throws Exception
{
QContext.init(h.instance, new QSession().withPermissions(Set.of("file_gateway.admin")));
var input = new RunBackendStepInput().withProcessName(action).withRecords(List.of(row));
var review = new FileGatewayProcesses.Review();
var preview = new RunBackendStepOutput();
review.runOnePage(input, preview);
assertThat(review.getProcessSummary(preview, false)).hasSize(1);
assertThat(review.getProcessSummary(preview, true)).hasSize(1);
var result = new RunBackendStepOutput();
new FileGatewayProcesses.Apply().runOnePage(input.withRecords(preview.getRecords()), result);
assertThat(result.getRecords()).hasSize(1);
}
}
@@ -0,0 +1,196 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.security.KeyPairGenerator;
import java.util.Date;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.pgp.PgpStreams;
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import org.apache.sshd.common.config.keys.KeyUtils;
import org.apache.sshd.sftp.client.SftpClientFactory;
import org.bouncycastle.bcpg.ArmoredOutputStream;
import org.bouncycastle.bcpg.HashAlgorithmTags;
import org.bouncycastle.openpgp.PGPKeyRingGenerator;
import org.bouncycastle.openpgp.PGPPublicKey;
import org.bouncycastle.openpgp.PGPSignature;
import org.bouncycastle.openpgp.operator.bc.BcPGPContentSignerBuilder;
import org.bouncycastle.openpgp.operator.bc.BcPGPDigestCalculatorProvider;
import org.bouncycastle.openpgp.operator.jcajce.JcaPGPKeyPair;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Real transfer encryption/decryption with ephemeral test-only OpenPGP keys.
*******************************************************************************/
@ExtendWith(NoDiskExtension.class)
class PgpIntegrationTest
{
record Keys(String publicKey, String privateKey)
{
}
static Keys keys() throws Exception
{
var generator = KeyPairGenerator.getInstance("RSA");
generator.initialize(2048);
var signing = new JcaPGPKeyPair(PGPPublicKey.RSA_SIGN, generator.generateKeyPair(), new Date());
var encryption = new JcaPGPKeyPair(PGPPublicKey.RSA_ENCRYPT, generator.generateKeyPair(), new Date());
var rings = new PGPKeyRingGenerator(PGPSignature.POSITIVE_CERTIFICATION, signing, "ephemeral-test",
new BcPGPDigestCalculatorProvider().get(HashAlgorithmTags.SHA1), null, null,
new BcPGPContentSignerBuilder(PGPPublicKey.RSA_SIGN, HashAlgorithmTags.SHA256), null);
rings.addSubKey(encryption);
ByteArrayOutputStream publicBytes = new ByteArrayOutputStream();
try(var armor = new ArmoredOutputStream(publicBytes))
{
rings.generatePublicKeyRing().encode(armor);
}
ByteArrayOutputStream privateBytes = new ByteArrayOutputStream();
try(var armor = new ArmoredOutputStream(privateBytes))
{
rings.generateSecretKeyRing().encode(armor);
}
return (new Keys(publicBytes.toString(StandardCharsets.UTF_8), privateBytes.toString(StandardCharsets.UTF_8)));
}
@Test
void encryptedBulkUploadInvalidCiphertextAndAccountValidation() throws Exception
{
Keys keys = keys();
try(var h = new FileGatewayTestServer())
{
UploadIntegrationTest.profile(h);
var account = h.account("alice", "A");
assertThatThrownBy(() -> h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id"))
.withValue("pgpPublicKey", "invalid"))).isInstanceOf(Exception.class);
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("pgpDecryptUploads", true));
h.config.withPgpPrivateKeyEnvVarName("TEST_PGP").withPgpPassphraseEnvVarName("TEST_PASS");
h.runtime.getWorker().withEnvironment(name -> "TEST_PGP".equals(name) ? keys.privateKey() : "");
var sshKey = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(account, sshKey);
try(var ssh = h.login("alice", sshKey); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
{
byte[] ciphertext;
try(var input = PgpStreams.encrypt(new ByteArrayInputStream(UploadIntegrationTest.CSV), keys.publicKey(), "orders.csv"))
{
ciphertext = input.readAllBytes();
}
try(var output = sftp.write(UploadIntegrationTest.FOLDER + "orders.csv.pgp"))
{
output.write(ciphertext);
}
h.runtime.getWorker().runOnce();
var job = UploadIntegrationTest.jobs(h).getFirst();
assertThat(job.getValueString("status")).isEqualTo("SUCCEEDED");
assertThat(job.getValueInteger("insertedCount")).isEqualTo(2);
assertThat(job.getValueString("plaintextSha256")).isNotBlank().isNotEqualTo(job.getValueString("sha256"));
assertThat(h.store.objects.get(job.getValueString("plaintextStorageKey"))).isEqualTo(UploadIntegrationTest.CSV);
assertThat(h.store.objects.get(job.getValueString("storageKey"))).isEqualTo(ciphertext);
try(var output = sftp.write(UploadIntegrationTest.FOLDER + "bad.csv.gpg"))
{
output.write(new byte[]{1, 2, 3});
}
h.runtime.getWorker().runOnce();
assertThat(UploadIntegrationTest.jobs(h)).anySatisfy(row ->
{
assertThat(row.getValueString("status")).isEqualTo("REJECTED");
assertThat(row.getValueString("errorMessage")).contains("PGP decryption failed");
});
}
}
}
@Test
void encryptedGeneratedAndOutboxDownloadsMatchPlaintext() throws Exception
{
Keys keys = keys();
try(var h = new FileGatewayTestServer())
{
h.seedRows(2);
h.saved("savedView", "alpha", "Daily");
var account = h.account("alice", "A");
var sshKey = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(account, sshKey);
byte[] plain;
try(var ssh = h.login("alice", sshKey); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
{
plain = PhaseEIntegrationTest.read(sftp, "/testapp/alpha/views/daily.csv");
}
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id"))
.withValue("pgpPublicKey", keys.publicKey()).withValue("pgpEncryptDownloads", true));
var alias = PhaseEIntegrationTest.alias(h, "out", "/out", "OUTBOX", account);
h.audit.update(FileGatewayTables.ALIAS, new QRecord().withValue("id", alias.getValue("id")).withValue("consumerName", null));
com.kingsrook.qbits.filegateway.api.FileGatewayOutbox.put("out", "reply.csv", new ByteArrayInputStream(plain), plain.length);
try(var ssh = h.login("alice", sshKey); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
{
assertThat(PhaseEIntegrationTest.names(sftp, "/testapp/alpha/views")).contains("daily.csv.pgp");
for(String path : java.util.List.of("/testapp/alpha/views/daily.csv.pgp", "/out/reply.csv.pgp"))
{
byte[] encrypted = PhaseEIntegrationTest.read(sftp, path);
try(var decrypted = PgpStreams.decrypt(new ByteArrayInputStream(encrypted), keys.privateKey(), new char[0], 1_000_000))
{
assertThat(decrypted.readAllBytes()).isEqualTo(plain);
}
}
assertThat(h.audit.query(FileGatewayTables.OUTBOX_FILE, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()).getFirst().getValueString("status"))
.isEqualTo("DOWNLOADED");
}
}
}
@Test
void integrityAndPlaintextSizeAreEnforced() throws Exception
{
Keys keys = keys();
byte[] bytes = new byte[160000];
new java.util.Random(42).nextBytes(bytes);
byte[] encrypted;
try(var input = PgpStreams.encrypt(new ByteArrayInputStream(bytes), keys.publicKey(), "file.bin"))
{
encrypted = input.readAllBytes();
}
try(var input = PgpStreams.decrypt(new ByteArrayInputStream(encrypted), keys.privateKey(), new char[0], bytes.length))
{
assertThat(input.readAllBytes()).isEqualTo(bytes);
}
assertThatThrownBy(() ->
{
try(var input = PgpStreams.decrypt(new ByteArrayInputStream(encrypted), keys.privateKey(), new char[0], 10))
{
input.readAllBytes();
}
}).isInstanceOf(java.io.IOException.class);
encrypted[encrypted.length - 1] ^= 1;
assertThatThrownBy(() ->
{
try(var input = PgpStreams.decrypt(new ByteArrayInputStream(encrypted), keys.privateKey(), new char[0], bytes.length))
{
input.readAllBytes();
}
}).isInstanceOf(java.io.IOException.class);
}
}
@@ -0,0 +1,318 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.filegateway;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.time.Clock;
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.filegateway.api.FileAcceptance;
import com.kingsrook.qbits.filegateway.api.FileGatewayConsumer;
import com.kingsrook.qbits.filegateway.api.FileGatewayConsumersMetaData;
import com.kingsrook.qbits.filegateway.api.FileGatewayOutbox;
import com.kingsrook.qbits.filegateway.api.InboundFile;
import com.kingsrook.qbits.filegateway.api.OutboxFileRef;
import com.kingsrook.qbits.filegateway.maintenance.GatewayMaintenance;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayHome;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.outbox.OutboxService;
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.apache.sshd.common.config.keys.KeyUtils;
import org.apache.sshd.sftp.client.SftpClient;
import org.apache.sshd.sftp.client.SftpClientFactory;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Phase E acceptance uses real MINA sessions, strict UI API and the adopter DDL.
*******************************************************************************/
@ExtendWith(NoDiskExtension.class)
class PhaseEIntegrationTest
{
static InboundFile accepted;
static byte[] acceptedBytes;
static int downloaded;
static boolean failCallback;
public static class Consumer implements FileGatewayConsumer
{
@Override
public String getName()
{
return ("partner");
}
@Override
public FileAcceptance accept(InboundFile file) throws Exception
{
accepted = file;
try(var stream = file.openStream())
{
acceptedBytes = stream.readAllBytes();
}
return (FileAcceptance.accepted("receipt"));
}
@Override
public void onDownloaded(OutboxFileRef ref) throws Exception
{
if(failCallback)
{
throw (new Exception("retry test"));
}
downloaded++;
}
}
static QRecord alias(FileGatewayTestServer h, String name, String path, String kind, QRecord account) throws Exception
{
return (h.audit.insert(FileGatewayTables.ALIAS, new QRecord().withValue("name", name).withValue("path", path).withValue("kind", kind)
.withValue("accountId", account == null ? null : account.getValue("id")).withValue("consumerName", "partner")));
}
static List<String> names(SftpClient sftp, String path) throws Exception
{
List<String> names = new ArrayList<>();
for(var item : sftp.readDir(path))
{
if(!List.of(".", "..").contains(item.getFilename()))
{
names.add(item.getFilename());
}
}
return (names);
}
static byte[] read(SftpClient sftp, String path) throws Exception
{
try(var input = sftp.read(path))
{
return (input.readAllBytes());
}
}
@Test
void endpointsChrootVisibilityBulkAndInbox() throws Exception
{
try(var h = new FileGatewayTestServer())
{
FileGatewayConsumersMetaData.ofOrWithNew(h.instance).withConsumer(new QCodeReference(Consumer.class));
var profile = UploadIntegrationTest.profile(h);
var service = h.audit.insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", "service").withValue("userId", "A").withValue("accountType", "SERVICE"));
assertThat(service.getValueString("rootMode")).isEqualTo("ENDPOINTS_ONLY");
var user = h.account("alice", "A");
assertThat(user.getValueString("rootMode")).isEqualTo("FULL_TREE");
var denied = h.account("bob", "B");
var bulk = alias(h, "orders", "/acme/orders", "BULK_LOAD", null);
h.audit.update(FileGatewayTables.ALIAS, new QRecord().withValue("id", bulk.getValue("id")).withValue("tableName", "alpha")
.withValue("savedBulkLoadProfileId", profile.getValue("id")));
alias(h, "inbox", "/acme/inbox", "INBOX", service);
alias(h, "private", "/private", "INBOX", user);
alias(h, "shared", "/shared", "OUTBOX", null);
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(service, key);
var userKey = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(user, userKey);
var deniedKey = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(denied, deniedKey);
try(var ssh = h.login("service", key); var sftp = SftpClientFactory.instance().createSftpClient(ssh);
var aliceSsh = h.login("alice", userKey); var alice = SftpClientFactory.instance().createSftpClient(aliceSsh);
var bobSsh = h.login("bob", deniedKey); var bob = SftpClientFactory.instance().createSftpClient(bobSsh))
{
assertThat(names(sftp, "/")).containsExactlyInAnyOrder("acme", "shared");
assertThat(names(alice, "/")).contains("testapp", "acme", "private", "shared");
assertThat(names(bob, "/")).doesNotContain("acme", "private");
assertThatThrownBy(() -> sftp.stat("/testapp")).isInstanceOf(Exception.class);
try(var output = sftp.write("/acme/orders/order.csv"))
{
output.write(UploadIntegrationTest.CSV);
}
h.runtime.getWorker().runOnce();
assertThat(UploadIntegrationTest.jobs(h).getFirst().getValueString("status")).isEqualTo("SUCCEEDED");
try(var output = sftp.write("/acme/inbox/message.txt"))
{
output.write("hello partner".getBytes(StandardCharsets.UTF_8));
}
h.runtime.getWorker().runOnce();
assertThat(accepted.endpointName()).isEqualTo("inbox");
assertThat(accepted.remoteAddress()).isEqualTo("127.0.0.1");
assertThat(acceptedBytes).isEqualTo("hello partner".getBytes(StandardCharsets.UTF_8));
}
}
}
@Test
void endpointValidationAndView() throws Exception
{
try(var h = new FileGatewayTestServer())
{
var user = h.account("alice", "A");
for(String path : List.of("relative", "/../escape", "/UPPER", "/a//b", "/a/./b", "/a/", "/testapp/in"))
{
assertThatThrownBy(() -> alias(h, "invalid", path, "INBOX", user)).isInstanceOf(Exception.class);
}
var mount = alias(h, "good", "/good", "INBOX", user);
assertThatThrownBy(() -> h.audit.update(FileGatewayTables.ALIAS, new QRecord().withValue("id", mount.getValue("id")).withValue("path", "/testapp"))).isInstanceOf(Exception.class);
assertThatThrownBy(() -> alias(h, "duplicate", "/good", "INBOX", user)).isInstanceOf(Exception.class);
h.seedRows(2);
var saved = h.saved("savedView", "alpha", "Daily Rows");
var view = alias(h, "view", "/download", "VIEW", user);
h.audit.update(FileGatewayTables.ALIAS, new QRecord().withValue("id", view.getValue("id")).withValue("tableName", "alpha")
.withValue("savedViewId", saved.getValue("id")).withValue("format", "csv"));
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(user, key);
try(var ssh = h.login("alice", key); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
{
assertThat(names(sftp, "/download")).containsExactly("daily-rows.csv");
assertThat(read(sftp, "/download/daily-rows.csv")).isEqualTo(read(sftp, "/testapp/alpha/views/daily-rows.csv"));
}
}
}
@Test
void outboxLifecycleAndRetry() throws Exception
{
try(var h = new FileGatewayTestServer())
{
downloaded = 0;
failCallback = false;
FileGatewayConsumersMetaData.ofOrWithNew(h.instance).withConsumer(new QCodeReference(Consumer.class));
var account = h.account("alice", "A");
alias(h, "out", "/out", "OUTBOX", account);
alias(h, "in", "/in", "INBOX", account);
assertThatThrownBy(() -> FileGatewayOutbox.put("missing", "x", new ByteArrayInputStream(new byte[0]), 0)).isInstanceOf(Exception.class);
assertThatThrownBy(() -> FileGatewayOutbox.put("in", "x", new ByteArrayInputStream(new byte[0]), 0)).isInstanceOf(Exception.class);
byte[] bytes = new byte[200000];
new java.util.Random(7).nextBytes(bytes);
var ref = FileGatewayOutbox.put("out", "reply.bin", new ByteArrayInputStream(bytes), bytes.length);
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
h.key(account, key);
try(var ssh = h.login("alice", key); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
{
assertThat(names(sftp, "/out")).containsExactly("reply.bin");
try(var handle = sftp.open("/out/reply.bin"))
{
byte[] partial = new byte[100];
for(int i = 0; i < 5; i++)
{
sftp.read(handle, 0, partial);
}
}
assertThat(downloaded).isZero();
assertThat(read(sftp, "/out/reply.bin")).isEqualTo(bytes);
assertThat(downloaded).isEqualTo(1);
var row = h.audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", ref.outboxFileId())).getFirst();
assertThat(row.getValueString("status")).isEqualTo("DOWNLOADED");
assertThat(row.getValue("firstDownloadedAt")).isNotNull();
assertThat(row.getValueInteger("downloadCount")).isEqualTo(1);
read(sftp, "/out/reply.bin");
assertThat(downloaded).isEqualTo(1);
sftp.remove("/out/reply.bin");
assertThat(names(sftp, "/out")).isEmpty();
assertThat(h.audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", ref.outboxFileId())).getFirst().getValueString("status")).isEqualTo("REMOVED");
failCallback = true;
FileGatewayOutbox.put("out", "retry.bin", new ByteArrayInputStream(bytes), bytes.length);
read(sftp, "/out/retry.bin");
assertThat(downloaded).isEqualTo(1);
failCallback = false;
GatewayMaintenance.maintain(h.instance, h.config, h.store, Clock.systemUTC());
assertThat(downloaded).isEqualTo(2);
}
}
}
@Test
void outboxWithoutRunningSftpAndHomeAndIntegrations() throws Exception
{
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance -> { }, 0, false, false))
{
assertThat(FileGatewayOutbox.isAvailable()).isTrue();
FileGatewayOutbox.setProvider(new OutboxService(h.instance, h.store, h.config, Clock.systemUTC()));
var account = h.account("partner", "A");
alias(h, "out", "/out", "OUTBOX", account);
FileGatewayOutbox.put("out", "file.txt", new ByteArrayInputStream(new byte[]{1}), 1);
Instant now = Instant.now();
h.seedTransfer(account, "UPLOAD", 2097152L, now);
h.seedTransfer(account, "DOWNLOAD", 1048576L, now);
h.audit.insert(FileGatewayTables.UPLOAD_JOB, new QRecord().withValue("accountId", account.getValue("id")).withValue("fileName", "orders.csv")
.withValue("status", "FAILED").withValue("finishedAt", now).withValue("errorMessage", "Invalid quantity"));
QContext.init(h.instance, new QSession().withPermissions(Set.of("file_gateway.view")));
FileGatewayHome home = new FileGatewayHome();
assertThat(((Number) ((StatisticsData) home.data("transfers", now)).getCount()).intValue()).isEqualTo(2);
assertThat(((Number) ((StatisticsData) home.data("in", now)).getCount()).doubleValue()).isEqualTo(2);
assertThat(((Number) ((StatisticsData) home.data("out", now)).getCount()).doubleValue()).isEqualTo(1);
assertThat(((Number) ((StatisticsData) home.data("failures", now)).getCount()).intValue()).isEqualTo(1);
assertThat(((TableData) home.data("recent", now)).getRows()).hasSize(1);
///////////////////////////////////////////////////////////////////////////////
// direction: failures and locks are bad when they grow; volume is not bad //
///////////////////////////////////////////////////////////////////////////////
assertThat(((StatisticsData) home.data("failures", now)).getIncreaseIsGood()).as("failures").isFalse();
assertThat(((StatisticsData) home.data("locked", now)).getIncreaseIsGood()).as("locked").isFalse();
for(String key : List.of("transfers", "in", "out", "sessions"))
{
assertThat(((StatisticsData) home.data(key, now)).getIncreaseIsGood()).as(key).isTrue();
}
///////////////////////////////////////////////////////////////////////////////
// every home table says why it is empty //
///////////////////////////////////////////////////////////////////////////////
for(String key : List.of("recent", "partners", "active", "servers"))
{
assertThat(((TableData) home.data(key, now)).getNoRowsFoundHTML()).as(key).isNotBlank();
}
assertThat(((TableData) home.data("active", now)).getRows()).isEmpty();
assertThat(((TableData) home.data("active", now)).getNoRowsFoundHTML()).contains("No one is connected");
//////////////////////////////////////////////////////////////////
// readable at all widths: home tables declare no column widths //
//////////////////////////////////////////////////////////////////
assertThat(((TableData) home.data("recent", now)).getColumns()).allSatisfy(column -> assertThat(column.getWidth()).isNull());
for(String key : List.of("recent", "partners", "active", "servers"))
{
////////////////////////////////////////////////////////////////////////
// at most four columns, so tables stay readable at phone widths //
////////////////////////////////////////////////////////////////////////
assertThat(((TableData) home.data(key, now)).getColumns()).as(key).hasSizeLessThanOrEqualTo(4)
.allSatisfy(column -> assertThat(column.getWidth()).isNull());
}
assertThat(((TableData) home.data("partners", now)).getRows()).hasSize(1);
for(String widget : FileGatewayHome.KEYS)
{
assertThat(home.data(widget, now).getType()).isIn("statistics", "table");
}
assertThat(h.instance.getWidgets().values()).allSatisfy(widget -> assertThat(widget.getType()).isNotEqualTo("html"));
assertThat(FileGatewayIntegrations.esbTableNames()).containsExactly(FileGatewayTables.ACCOUNT, FileGatewayTables.ALIAS, FileGatewayTables.UPLOAD_JOB);
assertThat(FileGatewayIntegrations.apiTableNames()).containsExactlyElementsOf(FileGatewayTables.ALL);
assertThat(FileGatewayIntegrations.quickSearchTables()).hasSize(3);
}
}
}
@@ -78,6 +78,35 @@ class SftpIntegrationTest
/*******************************************************************************
** A TCP connection that never signs in (a health probe or port scan) leaves
** no connection-log row and no failure; a real sign-in attempt does.
*******************************************************************************/
@Test
void tcpProbeLeavesNoConnectionRow() throws Exception
{
try(var h = new FileGatewayTestServer())
{
for(int i = 0; i < 3; i++)
{
try(java.net.Socket socket = new java.net.Socket("127.0.0.1", h.runtime.getProtocol().getPort()))
{
socket.setSoTimeout(2000);
socket.getInputStream().read();
}
}
Thread.sleep(300);
assertThat(h.audit.query("file_gateway_connection_log", new QQueryFilter())).isEmpty();
KeyPair unknown = pair();
org.assertj.core.api.Assertions.assertThatThrownBy(() -> h.login("nobody", unknown)).isInstanceOf(Exception.class);
assertThat(h.audit.query("file_gateway_connection_log", new QQueryFilter())).singleElement()
.satisfies(row -> assertThat(row.getValueString("outcome")).isEqualTo("AUTH_FAILED"));
}
}
/*******************************************************************************
** Permission filtered listings and audit.
*******************************************************************************/
@@ -109,7 +138,10 @@ class SftpIntegrationTest
assertThat(names(ca, "/testapp/alpha/views")).containsExactlyInAnyOrder("all-orders.csv", "all-orders.xlsx");
assertThat(names(ca, "/testapp/alpha/reports")).contains("daily-orders.csv", "daily-orders.xlsx");
assertThat(ca.stat("/testapp/alpha/views/all-orders.csv").getSize()).isZero();
assertThatThrownBy(() -> ca.read("/testapp/alpha/views/all-orders.csv")).isInstanceOf(SftpException.class);
try(var download = ca.read("/testapp/alpha/views/all-orders.csv"))
{
assertThat(download.readAllBytes()).isNotEmpty();
}
assertThatThrownBy(() -> ca.write("/testapp/alpha/test.csv")).isInstanceOf(SftpException.class);
assertThatThrownBy(() -> ca.setStat("/testapp/alpha/views/all-orders.csv", new SftpClient.Attributes().modifyTime(0))).isInstanceOfSatisfying(SftpException.class,
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_PERMISSION_DENIED));
@@ -123,7 +155,7 @@ class SftpIntegrationTest
}
assertThat(harness.calls).isNotEmpty().allSatisfy(call ->
{
String template = call.path().substring("/qqq/v1".length()).replaceAll("/table/[^/]+/query", "/table/{tableName}/query");
String template = call.path().substring("/qqq/v1".length()).replaceAll("/table/[^/]+/(query|export)", "/table/{tableName}/$1");
assertThat(harness.runtime.getInvoker().findOperation(HttpMethod.valueOf(call.method()), template)).isPresent();
assertThat(call.channel()).startsWith("sftp/");
assertThat(call.invokedBy()).isEqualTo("file-gateway");