Compare commits
61
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
79d241e1f6
|
||
|
|
c47d5a1c18 | ||
|
|
677aa54801
|
||
|
|
44a463e4d9
|
||
|
|
32f91b75c2
|
||
|
|
32239cd57d
|
||
|
|
950a8018c7
|
||
|
|
43bfe48feb
|
||
|
|
ead0c904ce
|
||
|
|
579946d62c
|
||
|
|
4f53c29318
|
||
|
|
cd3c0393a2
|
||
|
|
d9f74cb576
|
||
|
|
d12a7777af | ||
|
|
5f6a5022ce | ||
|
|
3c9a4d229b | ||
|
|
c5e1dd7e14 | ||
|
|
d6ebb60de6 | ||
|
|
e41bf1147b | ||
|
|
d4ca427445 | ||
|
|
2542dc0f89
|
||
|
|
c6954d86f1 | ||
|
|
05bfcd8d4b
|
||
|
|
d3cc594b0e | ||
|
|
ff182b6d7f
|
||
|
|
7482050234 | ||
|
|
27324a4159
|
||
|
|
37ee9028f2 | ||
|
|
c8a231f5f8
|
||
|
|
0a8c95e0ad | ||
|
|
f0e9836a89
|
||
|
|
dd4643e60d | ||
|
|
483c856594
|
||
|
|
275cc26b6b | ||
|
|
3e5cec20d2
|
||
|
|
1713d18fd7 | ||
|
|
6073091434 | ||
|
|
db241b03bc
|
||
|
|
10eb8807ba
|
||
|
|
ac55e437bd | ||
|
|
3ba219d668 | ||
|
|
d0572402a9
|
||
|
|
e6ff62181b
|
||
|
|
dc8ac5f089
|
||
|
|
7a5e42495d
|
||
|
|
5098152460 | ||
|
|
ab64cbf040 | ||
|
|
6261e197d0 | ||
|
|
3672bb513d | ||
|
|
182fb46a66
|
||
|
|
516c250d4f
|
||
|
|
c0646db521
|
||
|
|
5fcf0d1f8d
|
||
|
|
c1b78d3077
|
||
|
|
00a20eb5c8
|
||
|
|
24f1cbec33
|
||
|
|
415d0bdd0a
|
||
|
|
89e468ca45
|
||
|
|
3296cb9ca4
|
||
|
|
852d474722 | ||
|
|
ec8092e468 |
No files matched your search
+10
-61
@@ -1,65 +1,14 @@
|
||||
version: 2.1
|
||||
|
||||
setup: true
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
workflows:
|
||||
|
||||
test_only:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release/.*|integration.*)/
|
||||
tags:
|
||||
only: []
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- develop
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /release\/.*/
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [ /^v\d+\.\d+\.\d+$/ ]
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
@@ -0,0 +1,7 @@
|
||||
name: build
|
||||
on:
|
||||
push:
|
||||
jobs:
|
||||
maven:
|
||||
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
|
||||
secrets: inherit
|
||||
@@ -0,0 +1,23 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -1,25 +1,74 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
description: Plan a cut without changing release state
|
||||
type: boolean
|
||||
default: false
|
||||
finalize:
|
||||
description: Retry finalization of an existing cut; do not cut another RC
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Recover the same published RC after a deployment timeout; never
|
||||
reupload
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
description: Existing cut merge_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
bump_sha:
|
||||
description: Existing cut bump_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
branch:
|
||||
description: Existing cut branch (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
version:
|
||||
description: Existing cut version (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
types:
|
||||
- closed
|
||||
branches:
|
||||
- develop
|
||||
repository_dispatch:
|
||||
types:
|
||||
- qbit-rc-published
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
actions: read
|
||||
checks: read
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|
||||
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
|
||||
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
|
||||
'release:rc'))
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
|
||||
}}
|
||||
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
|
||||
== 'workflow_dispatch' && inputs.finalize) }}
|
||||
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
|
||||
}}
|
||||
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|
||||
|| '' }}
|
||||
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|
||||
|| inputs.merge_sha || '' }}
|
||||
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|
||||
|| inputs.bump_sha || '' }}
|
||||
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|
||||
|| inputs.branch || '' }}
|
||||
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|
||||
|| inputs.version || '' }}
|
||||
secrets: inherit
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release-no-integration
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
asciidoc: false
|
||||
release_completion: true
|
||||
@@ -14,13 +14,12 @@ code and QInstance. Design: `docs/DESIGN.md`.
|
||||
## Build commands
|
||||
|
||||
```bash
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT compile
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT test
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
```
|
||||
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd` and quote the `-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT` argument in PowerShell.
|
||||
Keep the POM default at `4.1.0-SNAPSHOT` for CI; the private version prevents local artifact replacement.
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -51,12 +50,30 @@ Claims compare expected status under an account-row transaction; core's optimist
|
||||
alone does not add a SQL WHERE guard. Fresh worker user sessions outlive SSH connections and are
|
||||
revoked after delivery. Bulk process multipart field: `theFile`; mapping frontend step: `fileMapping`
|
||||
(resumes `receiveFileMapping`). RC.2 object storage owns S3 part size at connection level. The
|
||||
private invoker selects multipart only when a file part is present, so the mapping request
|
||||
restreams the file with its values. See README adoption prerequisites and compatibility notes.
|
||||
private invoker supports multipart process values without a file part, so the mapping request
|
||||
sends only its values. See README adoption prerequisites and compatibility notes.
|
||||
|
||||
Saved-view/report downloads, endpoint routing/chroot, outbox delivery, Quick Search/ESB,
|
||||
preferences, retention and the app home belong to later phases. The core-valid alert type is `filegateway.lockout`; the design's
|
||||
underscore-only name is rejected by core validation.
|
||||
Phase (c) adds `GeneratedDownloads` (user-scoped export/render/download API flow and durable
|
||||
account-locked download quotas) and `GeneratedReadChannel` (fixed rolling window, no spool).
|
||||
Saved-view JSON uses `queryFilter.orderBys` and `queryColumns.columns` visibility/order. Generated
|
||||
stat size is zero; reads fill requests until real EOF to avoid MINA's short-read EOF inference.
|
||||
Reports list only with a configured render-process storage table. FieldSecurityLock DENY tests
|
||||
require `enforcePermissionsInCoreActions=true` and match direct same-user UI export bytes.
|
||||
Core `RenderSavedReportExecuteStep` stages a rendered report in a JVM temp file and deletes it
|
||||
in try-with-resources; that is the bounded temp area the owner rule allows (in-memory `/tmp`).
|
||||
The core-valid alert type is `filegateway.lockout` (dotted names only).
|
||||
|
||||
Phase (e) adds endpoints and chroot (`AliasCustomizer` validates paths), `outbox/` (the
|
||||
`FileGatewayOutboxProvider`, registered in every deployment), `pgp/` (BouncyCastle stream
|
||||
decrypt/encrypt, keys in memory), `maintenance/` (daily retention process and record-action
|
||||
effects), `FileGatewayHome` (typed STATISTICS and TABLE widgets only; guardrail G5),
|
||||
`FileGatewayIntegrations` (ESB, Quick Search, API lists) and `file_gateway_server` heartbeats.
|
||||
The `file-transfer` Preferences section requires only `file_gateway.connect`. Its provider uses
|
||||
core's trusted scope with the original session, derives identity only from the request context,
|
||||
and manages the caller's unrevoked USER-account keys through normal table actions. Add key
|
||||
creates an account on first use; Revoke sets `revoked_at`; PATCH edits labels and expiry.
|
||||
Optional `publicHostName` advertises the hostname; port/fingerprint use the latest heartbeat.
|
||||
The k8s example is in `docs/k8s/`.
|
||||
|
||||
Tests run a strict UI API server, H2 from the adopter DDL and real MINA clients. NoDiskExtension
|
||||
snapshots the working tree (excluding target) and isolated test home, and asserts the test temp
|
||||
@@ -78,7 +95,7 @@ directory is empty. The host supplies RDBMS; `qqq-backend-module-rdbms` must rem
|
||||
|
||||
## Dependencies
|
||||
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-SNAPSHOT` until 4.1.0-RC.6.
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.7` (first release with QRun-IO/qqq#1170); never a SNAPSHOT.
|
||||
Only allow-listed qbit API artifacts (`qbit-object-storage` `api` package) may be depended on.
|
||||
Notifications go through core `QAlerts.notify`, never qbit-alerts classes.
|
||||
|
||||
|
||||
+37
-6
@@ -4,8 +4,30 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Upload-job status transitions no longer use `status` as the table's record-version field.
|
||||
QQQ 4.1.0-RC.26 advances a configured version field on every update (a STRING version
|
||||
becomes a random UUID), which overwrote upload status. The guarded transition now re-reads
|
||||
status inside the existing cross-pod account-lock transaction. The upload-job table no longer
|
||||
declares a record-version field; no schema or public API change.
|
||||
|
||||
- Endpoint pickers for saved views, saved reports and saved bulk-load profiles are dropped when
|
||||
the host has not registered those possible-value sources, so hosts without them validate.
|
||||
|
||||
### Added
|
||||
|
||||
- File transfer Preferences with connect-permission visibility, read-only connection details,
|
||||
optional public hostname, self-service Add key/Revoke actions and editable key labels/expiry.
|
||||
- First-key USER account creation with configured defaults and unique usernames; transactional
|
||||
ownership checks, friendly duplicate-key validation, core/HTTP tests and real SFTP acceptance.
|
||||
|
||||
- Phase (c) generated saved-view and saved-report downloads through the user-scoped UI API,
|
||||
CSV/XLSX/JSON account formats, a bounded rolling read window and HTTP stream cleanup.
|
||||
- Generated-file and UTC daily-download quotas, persisted byte charges under account locks,
|
||||
per-file VIEW/REPORT transfer auditing and actionable SFTP API/limit errors.
|
||||
- Real MINA download tests for filters/column order, POI XLSX validation, JSON, FieldSecurityLock
|
||||
DENY parity, report rendering, hidden tables/storage, large streaming exports and quotas.
|
||||
- Phase (b) bounded streaming uploads through the object-storage system API, SHA-256, concurrent
|
||||
byte quotas, temporary-name promotion, dedupe and transfer auditing.
|
||||
- Bounded upload workers with guarded claims, fresh user sessions, scanner/quarantine hooks,
|
||||
@@ -29,12 +51,21 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
|
||||
- Isolated consumer/outbox API contract and complete configuration for subsequent phases.
|
||||
- Real SFTP/strict UI API acceptance tests, no-disk guard, schema round trips and adoption docs.
|
||||
|
||||
- Endpoints (aliases) with path validation and ENDPOINTS_ONLY chroot; BULK_LOAD, INBOX, VIEW,
|
||||
REPORT and OUTBOX kinds.
|
||||
- The outbox provider (`FileGatewayOutbox.put`), outbox files, pickup tracking and `onDownloaded`.
|
||||
- Optional per-account PGP decryption of uploads and encryption of downloads.
|
||||
- The typed app home, `file_gateway_server` heartbeats, Unlock/Disable/Revoke/End session record
|
||||
actions, the daily retention process and `FileGatewayIntegrations`.
|
||||
- Kubernetes example manifests in `docs/k8s/`.
|
||||
|
||||
### Compatibility
|
||||
|
||||
- Local invoker integration builds use `4.1.0-FG-SNAPSHOT`; CI retains the POM's
|
||||
`4.1.0-SNAPSHOT` default. RDBMS remains a host-provided runtime dependency.
|
||||
- Builds on QQQ `4.1.0-RC.7` (the first release with QRun-IO/qqq#1170). RDBMS remains a
|
||||
host-provided runtime dependency.
|
||||
- Alert type `filegateway.lockout` uses the dotted syntax required by core validation.
|
||||
- Saved-view/report downloads, endpoint routing/chroot, outbox delivery and app home remain later
|
||||
phases. Object-storage RC.2 controls S3 part size through its connection configuration.
|
||||
- With the private core build, the file-mapping request resends its file to select multipart
|
||||
encoding; URL-encoded process values are not accepted by the v1 route.
|
||||
- Object-storage RC.2 controls S3 part size through its connection configuration.
|
||||
- Core #1170 sends multipart process values without re-uploading the original file.
|
||||
- FieldSecurityLock requires core permission enforcement enabled. Saved-report rendering still
|
||||
uses a temporary file inside core; fully disk-free report generation needs a core change.
|
||||
The report folder is hidden without the render process's configured storage table.
|
||||
@@ -1,4 +1,4 @@
|
||||
# CLAUDE.md
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for coding agents working in this repository.
|
||||
|
||||
@@ -14,13 +14,12 @@ code and QInstance. Design: `docs/DESIGN.md`.
|
||||
## Build commands
|
||||
|
||||
```bash
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT compile
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT test
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
```
|
||||
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd` and quote the `-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT` argument in PowerShell.
|
||||
Keep the POM default at `4.1.0-SNAPSHOT` for CI; the private version prevents local artifact replacement.
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -51,12 +50,30 @@ Claims compare expected status under an account-row transaction; core's optimist
|
||||
alone does not add a SQL WHERE guard. Fresh worker user sessions outlive SSH connections and are
|
||||
revoked after delivery. Bulk process multipart field: `theFile`; mapping frontend step: `fileMapping`
|
||||
(resumes `receiveFileMapping`). RC.2 object storage owns S3 part size at connection level. The
|
||||
private invoker selects multipart only when a file part is present, so the mapping request
|
||||
restreams the file with its values. See README adoption prerequisites and compatibility notes.
|
||||
private invoker supports multipart process values without a file part, so the mapping request
|
||||
sends only its values. See README adoption prerequisites and compatibility notes.
|
||||
|
||||
Saved-view/report downloads, endpoint routing/chroot, outbox delivery, Quick Search/ESB,
|
||||
preferences, retention and the app home belong to later phases. The core-valid alert type is `filegateway.lockout`; the design's
|
||||
underscore-only name is rejected by core validation.
|
||||
Phase (c) adds `GeneratedDownloads` (user-scoped export/render/download API flow and durable
|
||||
account-locked download quotas) and `GeneratedReadChannel` (fixed rolling window, no spool).
|
||||
Saved-view JSON uses `queryFilter.orderBys` and `queryColumns.columns` visibility/order. Generated
|
||||
stat size is zero; reads fill requests until real EOF to avoid MINA's short-read EOF inference.
|
||||
Reports list only with a configured render-process storage table. FieldSecurityLock DENY tests
|
||||
require `enforcePermissionsInCoreActions=true` and match direct same-user UI export bytes.
|
||||
Core `RenderSavedReportExecuteStep` stages a rendered report in a JVM temp file and deletes it
|
||||
in try-with-resources; that is the bounded temp area the owner rule allows (in-memory `/tmp`).
|
||||
The core-valid alert type is `filegateway.lockout` (dotted names only).
|
||||
|
||||
Phase (e) adds endpoints and chroot (`AliasCustomizer` validates paths), `outbox/` (the
|
||||
`FileGatewayOutboxProvider`, registered in every deployment), `pgp/` (BouncyCastle stream
|
||||
decrypt/encrypt, keys in memory), `maintenance/` (daily retention process and record-action
|
||||
effects), `FileGatewayHome` (typed STATISTICS and TABLE widgets only; guardrail G5),
|
||||
`FileGatewayIntegrations` (ESB, Quick Search, API lists) and `file_gateway_server` heartbeats.
|
||||
The `file-transfer` Preferences section requires only `file_gateway.connect`. Its provider uses
|
||||
core's trusted scope with the original session, derives identity only from the request context,
|
||||
and manages the caller's unrevoked USER-account keys through normal table actions. Add key
|
||||
creates an account on first use; Revoke sets `revoked_at`; PATCH edits labels and expiry.
|
||||
Optional `publicHostName` advertises the hostname; port/fingerprint use the latest heartbeat.
|
||||
The k8s example is in `docs/k8s/`.
|
||||
|
||||
Tests run a strict UI API server, H2 from the adopter DDL and real MINA clients. NoDiskExtension
|
||||
snapshots the working tree (excluding target) and isolated test home, and asserts the test temp
|
||||
@@ -78,7 +95,7 @@ directory is empty. The host supplies RDBMS; `qqq-backend-module-rdbms` must rem
|
||||
|
||||
## Dependencies
|
||||
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-SNAPSHOT` until 4.1.0-RC.6.
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.7` (first release with QRun-IO/qqq#1170); never a SNAPSHOT.
|
||||
Only allow-listed qbit API artifacts (`qbit-object-storage` `api` package) may be depended on.
|
||||
Notifications go through core `QAlerts.notify`, never qbit-alerts classes.
|
||||
|
||||
|
||||
@@ -4,11 +4,11 @@ An SSH-key-only SFTP server that presents a QQQ application's permitted apps and
|
||||
virtual folder tree. Application metadata and saved objects are read through the spec-defined
|
||||
`/qqq/v1` UI API as the signed-in user, using `UiApiInvoker` on channel `sftp`.
|
||||
|
||||
Phases (a) and (b) implement authentication, listing/stat, account administration, uploads into
|
||||
saved bulk-load profiles, background consumer delivery, and readable sidecars/results/archive.
|
||||
Saved-view/report downloads, endpoint routing/chroot, outbox delivery, preferences and the app
|
||||
home follow in later phases. No native filesystem, host-key file, home directory or spool is
|
||||
used by the gateway. See [the approved design](docs/DESIGN.md).
|
||||
Phases (a) through (c) implement authentication, listing/stat, account administration, uploads into
|
||||
saved bulk-load profiles, background consumer delivery, readable sidecars/results/archive and
|
||||
saved-view/report downloads. Phase (e) adds endpoint routing/chroot, outbox delivery, PGP,
|
||||
Preferences, retention and the app home. No native filesystem, host-key file, home directory
|
||||
or spool is used by the gateway. See [the approved design](docs/DESIGN.md).
|
||||
|
||||
## Requirements
|
||||
|
||||
@@ -17,8 +17,8 @@ backend, qbit-object-storage 0.1.0-RC.2, and core saved bulk-load profile, saved
|
||||
report metadata. The gateway's
|
||||
RDBMS module dependency is test-scoped. Maven 3.9.11 is pinned by the wrapper.
|
||||
|
||||
Until the invoker core PR merges, local builds use the private `4.1.0-FG-SNAPSHOT` artifacts.
|
||||
The POM default remains `4.1.0-SNAPSHOT` for CI, to be replaced by 4.1.0-RC.6 when released.
|
||||
QQQ is pinned to `4.1.0-RC.7`, the first release with the UiApiInvoker in-process sessions,
|
||||
streams and multipart (QRun-IO/qqq#1170).
|
||||
|
||||
## Install
|
||||
|
||||
@@ -36,6 +36,10 @@ Apply [docs/ddl/file_gateway.sql](docs/ddl/file_gateway.sql) to the host's datab
|
||||
H2 and PostgreSQL and stores UTC timestamps without a zone, matching QQQ's RDBMS binding.
|
||||
The gateway does not create or migrate tables at runtime.
|
||||
|
||||
Register core's saved views, saved reports and saved bulk-load profiles (if the app uses them)
|
||||
before the gateway producer: endpoint pickers for a source the app hasn't registered become plain
|
||||
id fields, so the instance still validates.
|
||||
|
||||
Register metadata while building the QInstance:
|
||||
|
||||
```java
|
||||
@@ -68,6 +72,25 @@ least 3072 bits. Public-key validation runs on insert and update and computes it
|
||||
fingerprint, type and size. The host authentication module must assign the user's permissions
|
||||
when `createAutomatedSessionForUser` is called, including `file_gateway.connect`.
|
||||
|
||||
## Preferences
|
||||
|
||||
Users with `file_gateway.connect` manage their own SSH keys in **Preferences → File transfer**.
|
||||
They need no File Gateway admin or table permissions. **Add key** accepts an OpenSSH public key,
|
||||
a required label and optional expiry. The first successful key creates a USER account with
|
||||
FULL_TREE access, the configured account defaults and a unique login derived from the user ID.
|
||||
The section displays that login, plus the port and host key fingerprint from the latest gateway
|
||||
heartbeat. Set `.withPublicHostName("sftp.example.com")` on the shared config to display the
|
||||
public hostname; it defaults to null and is omitted when unset. The heartbeat port is the gateway
|
||||
listener port; deployments publishing a different external port must communicate that mapping.
|
||||
|
||||
The keys list contains only the user's unrevoked keys. Users can edit labels and expiry dates,
|
||||
or choose **Revoke**. Revoked keys cannot sign in; existing sessions close at the next revocation
|
||||
poll (normally within 60 seconds). Fingerprints, key types and last-used times are read-only.
|
||||
Ed25519, ECDSA and RSA of at least 3072 bits are supported. Duplicate keys are rejected with a
|
||||
validation message. Normal table customizers and configured audits apply, attributed to the
|
||||
signed-in user and client channel. Administrators manage service accounts and other users' keys
|
||||
through the standard account and SSH-key tables.
|
||||
|
||||
## Tables and permissions
|
||||
|
||||
All seven tables are registered, with camelCase QQQ fields mapped to snake_case database columns.
|
||||
@@ -109,12 +132,12 @@ The root lists visible apps; apps contain child apps and readable tables. Table
|
||||
|
||||
- `upload/<profile-slug>/` when the user has insert permission, a visible profile and a bulk-insert process;
|
||||
- `views/<slug>.csv` and `.xlsx` (or account-enabled JSON);
|
||||
- `reports/<slug>.<format>` when the saved-report table and render process are available;
|
||||
- `reports/<slug>.<format>` when the saved-report table, render process and its configured storage table are available;
|
||||
- account-scoped `results/<jobId>-<file>.results.csv` and `archive/<yyyy-mm-dd>/<jobId>-<file>`.
|
||||
|
||||
Hidden paths return no-such-file. Slug collisions append record IDs. Saved-object queries are
|
||||
paged. Metadata and saved objects are cached per connection for 60 and 30 seconds respectively.
|
||||
Saved-view/report files report size zero and remain unreadable until phase (c). Stored artifacts
|
||||
Saved-view/report files report size zero (unknown); clients read until EOF. Stored artifacts
|
||||
report their actual lengths and support ranged reads.
|
||||
|
||||
Only public-key auth is advertised. Password, keyboard-interactive, GSSAPI, host-based auth,
|
||||
@@ -132,6 +155,46 @@ Lockouts raise core `QAlerts` signals. **Core compatibility:** the alert type is
|
||||
`filegateway.lockout`, because the core validator rejects the design's underscore-only
|
||||
`file_gateway_lockout`. Without an alert sink the core logging fallback is used.
|
||||
|
||||
## Generated downloads
|
||||
|
||||
Opening `views/<slug>.<format>` queries the saved view again as the account user and starts
|
||||
`POST /table/{table}/export` through `UiApiInvoker.invokeForStream`. The saved view's
|
||||
`queryFilter` (including `orderBys`) and visible `queryColumns.columns` are preserved in order;
|
||||
the UI selection column and pagination are excluded. Legacy explicit `fieldNames` are also
|
||||
supported. CSV includes a header; CSV, XLSX and JSON are offered only when enabled in the
|
||||
account's `formats` (default `csv,xlsx`).
|
||||
|
||||
Opening `reports/<slug>.<format>` selects that saved report through the `renderSavedReport`
|
||||
record action, submits `reportFormat`, polls asynchronous status and streams the returned
|
||||
`/download/{file}` storage reference. The process's `storageTableName` input default must name
|
||||
an existing storage-capable table, backed by object storage in production. Without that table,
|
||||
`reports/` is hidden. Reports requiring additional interactive values fail with the API's error.
|
||||
|
||||
Each generated handle retains at most `downloadReadAheadMb` MiB (default 16). Sequential reads
|
||||
and reordered reads within this rolling window work; a seek outside it fails with a message
|
||||
to download sequentially. The HTTP stream closes with the SFTP handle. Read requests are filled
|
||||
until EOF so MINA does not mistake a short HTTP packet for the end of an unknown-size file.
|
||||
|
||||
`maxGeneratedFileMb` (default 1024) and `max_download_mb_per_day` are enforced as generated
|
||||
bytes enter the window. The daily total queries DOWNLOAD transfer rows for this account since
|
||||
UTC midnight, including failed and in-progress transfers; updates hold the account row lock so
|
||||
concurrent generated downloads share the cap. A rejected chunk is not exposed. Re-reading cached
|
||||
bytes does not charge them twice. One DOWNLOAD row per handle records the view/report ID, format,
|
||||
bytes, duration and outcome: OK, LIMIT, ERROR or DENIED for an HTTP 403. API error text reaches
|
||||
the SFTP client.
|
||||
|
||||
**Field security:** configure `QInstance.enforcePermissionsInCoreActions=true` when using this
|
||||
private core build's `FieldSecurityLock` enforcement. Acceptance tests use a DENY lock and compare
|
||||
the generated CSV byte-for-byte with a direct UI export by the same user: default columns omit
|
||||
the denied field. Explicitly selecting a denied field fails as it does through the UI API.
|
||||
|
||||
**Core filesystem limitation:** the current private core's `RenderSavedReportExecuteStep`
|
||||
creates and deletes a temporary file before publishing to storage. The gateway does not spool,
|
||||
and the existing no-disk snapshot/cleanup guard passes, but that guard cannot detect transient
|
||||
files. Fully disk-free saved-report generation remains blocked on replacing core's temporary
|
||||
file with storage streaming. CSV/XLSX/JSON table exports use core's streaming exporters; the
|
||||
50,000-row CSV acceptance test consumes the result with a fixed buffer.
|
||||
|
||||
## Uploads and object storage
|
||||
|
||||
Configure an object-storage bucket alias matching `config.fileGatewayBucketAlias` (default
|
||||
@@ -203,6 +266,7 @@ present without enabling those features.
|
||||
| Options | Defaults |
|
||||
|---|---|
|
||||
| `backendName`, `sftpEnabled`, `sftpPort` | `rdbms`, `false`, `2222` |
|
||||
| `publicHostName` | null (omit host from Preferences) |
|
||||
| `hostKeyEnvVarName`, `allowGeneratedHostKey` | `FILE_GATEWAY_SSH_HOST_KEY`, `false` |
|
||||
| `lockoutThreshold`, `lockoutWindow`, `lockoutDuration` | 5, 15 minutes, 30 minutes |
|
||||
| `maxConnectionsPerIpPerMinute` | 30 |
|
||||
@@ -223,19 +287,75 @@ The isolated `com.kingsrook.qbits.filegateway.api` package contains `FileGateway
|
||||
`InboundFile` (including remote address and a fresh stream supplier), `FileAcceptance`,
|
||||
`OutboxFileRef`, and the consumer metadata registry. `FileGatewayOutbox` delegates to a registered
|
||||
`FileGatewayOutboxProvider`; without one it reports unavailable and throws
|
||||
`FileGatewayUnavailableException`. The consumer pipeline is active in phase (b); endpoint binding
|
||||
and the outbox provider remain phase (e).
|
||||
`FileGatewayUnavailableException`. INBOX endpoints bind a consumer by `consumer_name`; the built-in
|
||||
`bulk-load` consumer serves `upload/<profile>/` folders and BULK_LOAD endpoints. The outbox
|
||||
provider is registered by the producer in every deployment (it needs only the database and object
|
||||
storage), so other qbits can call `FileGatewayOutbox.put(...)` from web pods too.
|
||||
|
||||
## Endpoints and chroot
|
||||
|
||||
`file_gateway_alias` rows ("Endpoints") mount partner-friendly paths such as `/acme/orders-in`:
|
||||
|
||||
| Kind | Behaviour |
|
||||
|---|---|
|
||||
| `BULK_LOAD` | upload folder that bulk-loads with the endpoint's saved profile |
|
||||
| `INBOX` | upload folder whose files go to the `consumer_name` consumer |
|
||||
| `VIEW`, `REPORT` | one generated file (`file_name`, or `<slug>.<format>`) |
|
||||
| `OUTBOX` | files put by `FileGatewayOutbox`; partners download and delete them |
|
||||
|
||||
Paths are absolute, `[a-z0-9._/-]`, unique, and never collide with app names. An endpoint is
|
||||
listed only when it is active, bound to the account (or shared), and its target is reachable by
|
||||
the user through `/qqq/v1`; endpoints grant nothing. `root_mode` FULL_TREE (default for users)
|
||||
shows the app tree plus endpoints; ENDPOINTS_ONLY (default for service accounts) is the chroot.
|
||||
|
||||
## Outbox
|
||||
|
||||
`put` streams to `file-gateway/outbox/{aliasId}/{uuid}/{fileName}` with a SHA-256 and a
|
||||
`file_gateway_outbox_file` row (READY). The first complete download marks it DOWNLOADED and calls
|
||||
`onDownloaded` on the endpoint's consumer (at-least-once); `rm` marks it REMOVED; retention marks
|
||||
old files EXPIRED and deletes the object.
|
||||
|
||||
## PGP (optional, per account)
|
||||
|
||||
`pgp_decrypt_uploads`: uploads named `*.pgp`/`*.gpg` are decrypted stream to stream into a new
|
||||
object before the consumer runs, with the key and passphrase from the env vars named by
|
||||
`pgpPrivateKeyEnvVarName`/`pgpPassphraseEnvVarName` (in memory only); a bad file is REJECTED.
|
||||
`pgp_encrypt_downloads`: downloads are encrypted on the fly to the account's `pgp_public_key`
|
||||
(validated on save) and listed with `.pgp` appended.
|
||||
|
||||
## App home, actions and retention
|
||||
|
||||
The `file_gateway` app home uses typed widgets only: a KPI row (transfers today, failed jobs in
|
||||
24 hours, active sessions, MB in and out today, locked accounts), and full-width tables for recent
|
||||
failures, top partners (7 days), active sessions and gateway servers (host key fingerprint, port,
|
||||
pod, last seen, from `file_gateway_server` heartbeats). Record actions (standard Review/Result,
|
||||
never in the nav): Unlock account, Disable account, Revoke key, End session, Retry upload. A daily
|
||||
scheduled retention process applies the `*Retention` settings and aborts stale RECEIVING uploads.
|
||||
|
||||
## Integrations
|
||||
|
||||
`FileGatewayIntegrations` lists what a host wires in without this qbit depending on those modules:
|
||||
`esbTableNames()` (accounts, endpoints, upload jobs), `quickSearchTables()` (accounts by username
|
||||
and label, endpoints by name and path, upload jobs by file name) and `apiTableNames()` (all tables,
|
||||
with the admin's rules).
|
||||
|
||||
## Kubernetes
|
||||
|
||||
`docs/k8s/gateway-deployment.yaml` and `docs/k8s/gateway-service.yaml`: the same image as the web
|
||||
Deployment with `FILE_GATEWAY_SFTP_ENABLED=true`, the host key from a Secret, a read-only root
|
||||
filesystem with an in-memory `/tmp`, 2 replicas and a PDB, and a LoadBalancer Service exposing
|
||||
port 22 only (no passive range: v1 is SFTP only) with `externalTrafficPolicy: Local`.
|
||||
|
||||
## Build and acceptance tests
|
||||
|
||||
```bash
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify
|
||||
./mvnw verify
|
||||
```
|
||||
|
||||
PowerShell:
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o '-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT' verify
|
||||
./mvnw.cmd verify
|
||||
```
|
||||
|
||||
The tests use H2 from the adopter DDL, a strict UI API server, generated in-memory SSH keys and
|
||||
|
||||
+44
-18
@@ -165,7 +165,7 @@ exists on any disk.
|
||||
| table | the table is in the app and in `metaData.tables` (read permission) |
|
||||
| `upload/` | the table has the bulk-insert process (`<table>.bulkInsert`) in metaData, and at least one saved bulk-load profile is visible (`POST /table/savedBulkLoadProfile/query` filtered by `tableName`) |
|
||||
| `views/` | the user can query `savedView` |
|
||||
| `reports/` | the user can query `savedReport` and has the `renderSavedReport` process |
|
||||
| `reports/` | the user can query `savedReport`, has the `renderSavedReport` process, and that process's `storageTableName` input default names an existing storage-capable table |
|
||||
| `results/`, `archive/` | always (they hold only this account's jobs) |
|
||||
|
||||
- **Caching.** Metadata responses are cached in memory per SFTP session for 60 seconds. Saved-object
|
||||
@@ -250,8 +250,8 @@ in-process session token; there is never a service session.
|
||||
(`ProcessSpecUtilsV1.addUploadedFiles`).
|
||||
|
||||
### Shipping
|
||||
§3.1–3.3 go in one core PR to `QRun-IO/qqq` `develop`, for 4.1.0-RC.6. Until then the qbit builds
|
||||
on `4.1.0-SNAPSHOT`, with the pom marked `STAND-IN(removeBy=4.1.0-RC.6)`.
|
||||
§3.1–3.3 shipped in QRun-IO/qqq#1170 (merged 2026-10-14), first released in QQQ 4.1.0-RC.7, which
|
||||
the qbit pins.
|
||||
|
||||
### 3.4 Permissions for automated sessions (qbit-authentik)
|
||||
`AuthentikGroupPermissionMapper` builds permissions from token claims, and an automated session
|
||||
@@ -265,13 +265,30 @@ has no token.
|
||||
- **Who needs it.** It is a qbit-authentik PR. The gateway works without it under MOCK auth and in
|
||||
tests, but kof22 needs it before going live.
|
||||
|
||||
### 3.5 Preferences (QRun-IO/qqq#1162)
|
||||
The gateway registers a "File transfer" section with the provider `FileTransferPreferencesProvider`.
|
||||
### 3.5 Preferences
|
||||
The gateway registers `file-transfer` ("File transfer", order 60) with
|
||||
`FileTransferPreferencesProvider`, using core's user-preferences API. Only users with
|
||||
`file_gateway.connect` see it; no admin-app or backing-table permissions are needed.
|
||||
|
||||
- **Scalar fields:** the SFTP username (read-only), "Add a public key" (TEXT) and "Key label".
|
||||
- **The `keys` list:** label, fingerprint (read-only), last used (read-only), expires and revoked.
|
||||
- **Behaviour.** The first key creates the user's `USER` account. Revoking a key sets `revoked_at`.
|
||||
- **Until #1162 is in the baseline,** admins add keys on the account record.
|
||||
- **Read-only connection details:** the account username, plus port and host key fingerprint from
|
||||
the most recent `file_gateway_server` heartbeat. The optional `publicHostName` config supplies the host; when
|
||||
unset, the host field is omitted. Before the first key, username is blank and the keys list
|
||||
explains that adding a key creates the account.
|
||||
- **The `keys` list:** only the caller's unrevoked USER-account keys. Label and expiry are editable
|
||||
through PATCH; fingerprint, key type and last-used time are read-only. Public-key text is never
|
||||
returned in preference values.
|
||||
- **Add key (`addKey`, LIST action):** requires OpenSSH public-key text and a label, with optional
|
||||
expiry. The first successful addition creates a USER account with FULL_TREE and configured
|
||||
account defaults. Its login is derived from the session user ID, normalized to lowercase
|
||||
`[a-z0-9._-]`, at most 64 characters; short IDs receive a `user-` prefix. Collisions receive
|
||||
`-2`, `-3`, etc. Subsequent keys reuse that account. The result names the computed fingerprint.
|
||||
- **Revoke (`revokeKey`, ROW action):** confirms revocation, sets `revoked_at`, and removes the
|
||||
key from the list. New logins fail immediately; existing sessions end on the next gateway poll.
|
||||
- **Persistence:** ordinary table actions retain `SshKeyCustomizer` validation and configured
|
||||
audits under the original user/client channel. Weak and duplicate keys return actionable 400
|
||||
messages. Failed first-key inserts roll back account creation. Mutations lock and recheck
|
||||
ownership; another user's key is not a setting here. Identity comes only from core's request
|
||||
context, never from the request body. Service-account keys remain administrator-managed.
|
||||
|
||||
## 4. Upload
|
||||
1. **Receive.** The SFTP write streams straight to object storage:
|
||||
@@ -333,6 +350,11 @@ The gateway registers a "File transfer" section with the provider `FileTransferP
|
||||
## 5. Download
|
||||
Everything is streamed. Nothing is written to disk.
|
||||
|
||||
The gateway itself streams without disk. Core's `RenderSavedReportExecuteStep` stages the
|
||||
rendered report in a JVM temp file and deletes it in try-with-resources before the process step
|
||||
returns. That is the bounded temp area the owner rule allows, so the `emptyDir` temp mount must be
|
||||
sized for the largest report. The no-disk test asserts the temp area is empty after each transfer.
|
||||
|
||||
- **Saved view** (`views/<slug>.<fmt>`):
|
||||
- `POST /table/savedView/query` by table name, as the user;
|
||||
- the filter, visible columns and order are parsed from `viewJson`;
|
||||
@@ -436,9 +458,11 @@ Everything is streamed. Nothing is written to disk.
|
||||
(§15) would add only a server and an authenticator.
|
||||
|
||||
## 7. Audit
|
||||
- **Connections.** Every connection writes `file_gateway_connection_log`. The row is opened at TCP
|
||||
accept and updated at auth and at disconnect. It records the outcome, IP, key fingerprint, SSH
|
||||
client version, byte and operation counts, and channel `sftp`.
|
||||
- **Connections.** Every sign-in attempt writes `file_gateway_connection_log`. The row is opened
|
||||
at the first authentication attempt (or a pre-auth refusal such as the connection rate limit)
|
||||
and updated at auth and at disconnect. It records the outcome, IP, key fingerprint, SSH client
|
||||
version, byte and operation counts, and channel `sftp`. A TCP connection that never tries to
|
||||
sign in, such as a kubelet health probe or a port scan, leaves no row.
|
||||
- **Operations.** Every list, upload, download, rename and remove writes
|
||||
`file_gateway_transfer_log`. It records the operation, path, object kind and id, bytes, sha256,
|
||||
duration, outcome and message, with channel `sftp/<sessionId>`.
|
||||
@@ -460,6 +484,7 @@ the table names.
|
||||
| `file_gateway_outbox_file` (child of endpoint) | files waiting for pickup | `alias_id`, `file_name`, `size_bytes`, `sha256`, `storage_key`, `status` (READY, DOWNLOADED, REMOVED, EXPIRED), `first_downloaded_at`, `download_count`, `removed_at`, `expires_at`, `created_by_consumer` |
|
||||
| `file_gateway_transfer_log` (main, "Transfers") | every operation | `account_id`, `connection_log_id`, `operation` (LIST, UPLOAD, DOWNLOAD, RENAME, REMOVE), `path`, `object_kind` (APP, TABLE, PROFILE, VIEW, REPORT, RESULT, ARCHIVE, ENDPOINT, OUTBOX), `object_id`, `format`, `bytes`, `sha256`, `duration_ms`, `outcome` (OK, DENIED, NOT_FOUND, LIMIT, ERROR), `message`, `client_channel`, `upload_job_id` |
|
||||
| `file_gateway_connection_log` (main, "Connections") | every connection | `account_id`, `username`, `remote_ip`, `client_version`, `key_fingerprint`, `outcome` (CONNECTED, AUTH_FAILED, LOCKED_OUT, IP_DENIED, EXPIRED, DISABLED, LIMIT), `session_id`, `pod_name`, `connected_at`, `last_seen_at`, `disconnected_at`, `disconnect_reason`, `bytes_in`, `bytes_out`, `operation_count`, `client_channel` |
|
||||
| `file_gateway_server` (system, not in nav) | gateway pod heartbeats for the app home | `pod_name` (unique), `host_key_fingerprint`, `port`, `started_at`, `last_seen_at`; rows unseen for a day are deleted by retention |
|
||||
|
||||
- **Child lists:**
|
||||
- an account shows its keys, upload jobs and connections;
|
||||
@@ -563,6 +588,7 @@ FileGatewayConsumersMetaData.ofOrWithNew(qInstance).withConsumer(new QCodeRefere
|
||||
- `sftpEnabled` (default false; true only in the gateway deployment, from
|
||||
`FILE_GATEWAY_SFTP_ENABLED`);
|
||||
- `sftpPort` (2222);
|
||||
- `publicHostName` (null; optional public hostname shown in Preferences);
|
||||
- `hostKeyEnvVarName` (`FILE_GATEWAY_SSH_HOST_KEY`);
|
||||
- `allowGeneratedHostKey` (false).
|
||||
- **Storage and buffers:**
|
||||
@@ -593,8 +619,8 @@ The gateway is a separate, stateless Deployment of the same image. An example is
|
||||
- `FILE_GATEWAY_SSH_HOST_KEY` from a Secret; in kof22, through ESO from Infisical.
|
||||
- 2 replicas, with a PodDisruptionBudget of `minAvailable: 1`.
|
||||
- `readOnlyRootFilesystem: true`. The only writable mount is a small in-memory `emptyDir`
|
||||
(`medium: Memory`, `sizeLimit: 64Mi`) for the JVM temp directory. Nothing is persisted.
|
||||
- Requests of 250m CPU and 768Mi memory; a 1.5Gi memory limit. The in-memory buffers are bounded
|
||||
(`medium: Memory`, `sizeLimit: 512Mi`, counted in the memory limit) for the JVM temp directory, sized for the largest saved report core stages there. Nothing is persisted.
|
||||
- Requests of 250m CPU and 768Mi memory; a 2Gi memory limit. The in-memory buffers are bounded
|
||||
by the config.
|
||||
- **Ports and probes.**
|
||||
- Container ports: 2222 (sftp) and the app's HTTP port, which serves loopback and probes only.
|
||||
@@ -602,7 +628,7 @@ The gateway is a separate, stateless Deployment of the same image. An example is
|
||||
- Liveness: HTTP health.
|
||||
- Startup: allows 2 minutes for the QInstance.
|
||||
- **Service.** `type: LoadBalancer`, with these settings:
|
||||
- the annotation `lbipam.cilium.io/ips: 10.100.22.65` in kof22 (owner, 2026-10-13; .60, .61 and .64 are taken);
|
||||
- the annotation `lbipam.cilium.io/ips` (or the older `io.cilium/lb-ipam-ips`, which k8s-galaxy uses): 10.100.22.65 in kof22 (owner, 2026-10-13; .60, .61 and .64 are taken);
|
||||
- **port 22 → 2222 only (no passive range in v1)**;
|
||||
- `externalTrafficPolicy: Local`, so the pod sees the real client IP for allowlists and
|
||||
lockouts.
|
||||
@@ -672,10 +698,10 @@ All tests run in the qbit's build.
|
||||
`Class.forName` of another qbit. The gateway's own `api` package is the published contract.
|
||||
- **G2.** Server tests run in strict mode.
|
||||
- **G3.** No extension is registered: the gateway is a UI API client.
|
||||
- **G4.** Stand-ins are marked `STAND-IN(core=..., removeBy=...)`: the 4.1.0-SNAPSHOT baseline
|
||||
until RC.6, and the preferences section until #1162.
|
||||
- **G4.** No stand-ins remain: the core baseline is a release, and the preferences section uses
|
||||
core's API from QRun-IO/qqq#1162.
|
||||
- **G5.** No HTML widgets.
|
||||
- **G7.** Pin 4.1.0-RC.6 as soon as it is cut. Until then, develop builds on the SNAPSHOT.
|
||||
- **G7.** Pinned to QQQ 4.1.0-RC.7; never a SNAPSHOT.
|
||||
|
||||
## 14. Phases
|
||||
- **(a)** The SFTP server, the virtual filesystem, and auth:
|
||||
|
||||
@@ -93,6 +93,9 @@ CREATE TABLE file_gateway_upload_job (
|
||||
result_summary TEXT,
|
||||
results_storage_key VARCHAR(1024),
|
||||
results_size_bytes BIGINT,
|
||||
plaintext_storage_key VARCHAR(1024),
|
||||
plaintext_sha256 VARCHAR(1024),
|
||||
plaintext_size_bytes BIGINT,
|
||||
error_message TEXT,
|
||||
attempts INTEGER,
|
||||
next_attempt_at TIMESTAMP,
|
||||
@@ -116,6 +119,7 @@ CREATE TABLE file_gateway_outbox_file (
|
||||
status VARCHAR(1024),
|
||||
first_downloaded_at TIMESTAMP,
|
||||
download_count INTEGER,
|
||||
on_downloaded_pending BOOLEAN DEFAULT FALSE,
|
||||
removed_at TIMESTAMP,
|
||||
expires_at TIMESTAMP,
|
||||
created_by_consumer VARCHAR(1024)
|
||||
@@ -156,9 +160,32 @@ CREATE TABLE file_gateway_connection_log (
|
||||
connected_at TIMESTAMP,
|
||||
last_seen_at TIMESTAMP,
|
||||
disconnected_at TIMESTAMP,
|
||||
end_requested_at TIMESTAMP,
|
||||
disconnect_reason TEXT,
|
||||
bytes_in BIGINT,
|
||||
bytes_out BIGINT,
|
||||
operation_count BIGINT,
|
||||
client_channel VARCHAR(1024)
|
||||
);
|
||||
|
||||
CREATE TABLE file_gateway_server (
|
||||
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
|
||||
create_date TIMESTAMP,
|
||||
modify_date TIMESTAMP,
|
||||
pod_name VARCHAR(1024) NOT NULL UNIQUE,
|
||||
host_key_fingerprint VARCHAR(1024),
|
||||
port INTEGER,
|
||||
started_at TIMESTAMP,
|
||||
last_seen_at TIMESTAMP
|
||||
);
|
||||
|
||||
-- Indexes for the job queue, dedupe, live sessions, home queries and retention.
|
||||
CREATE INDEX file_gateway_transfer_log_account_date ON file_gateway_transfer_log (account_id, create_date);
|
||||
CREATE INDEX file_gateway_transfer_log_date ON file_gateway_transfer_log (create_date);
|
||||
CREATE INDEX file_gateway_connection_log_account_live ON file_gateway_connection_log (account_id, disconnected_at);
|
||||
CREATE INDEX file_gateway_connection_log_date ON file_gateway_connection_log (create_date);
|
||||
CREATE INDEX file_gateway_upload_job_status_next ON file_gateway_upload_job (status, next_attempt_at);
|
||||
CREATE INDEX file_gateway_upload_job_dedupe ON file_gateway_upload_job (account_id, sha256);
|
||||
CREATE INDEX file_gateway_upload_job_date ON file_gateway_upload_job (create_date);
|
||||
CREATE INDEX file_gateway_outbox_file_alias_status ON file_gateway_outbox_file (alias_id, status);
|
||||
CREATE INDEX file_gateway_ssh_key_account ON file_gateway_ssh_key (account_id);
|
||||
@@ -0,0 +1,91 @@
|
||||
# qbit-file-gateway: the SFTP gateway Deployment (docs/DESIGN.md section 11).
|
||||
# Same image and QInstance as the application's web Deployment; only FILE_GATEWAY_SFTP_ENABLED differs.
|
||||
# Stateless: no volumes except an in-memory JVM temp area. State lives in the database and object storage.
|
||||
# Replace every <PLACEHOLDER> and copy the application's usual environment (database, object storage, auth).
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: <APP>-file-gateway
|
||||
labels:
|
||||
app.kubernetes.io/name: <APP>-file-gateway
|
||||
app.kubernetes.io/component: sftp-gateway
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: <APP>-file-gateway
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: <APP>-file-gateway
|
||||
app.kubernetes.io/component: sftp-gateway
|
||||
spec:
|
||||
terminationGracePeriodSeconds: 75 # SIGTERM: stop accepting, drain sessions up to 60 s, requeue claimed jobs
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: app
|
||||
image: <APP_IMAGE> # the same image as the web Deployment
|
||||
env:
|
||||
- name: FILE_GATEWAY_SFTP_ENABLED
|
||||
value: "true"
|
||||
- name: FILE_GATEWAY_SSH_HOST_KEY # OpenSSH private key text; parsed in memory, never written to disk
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: file-gateway-host-key
|
||||
key: ssh_host_key
|
||||
- name: JAVA_TOOL_OPTIONS
|
||||
value: "-Djava.io.tmpdir=/tmp"
|
||||
# ... the application's usual environment (database, object storage, authentication) ...
|
||||
ports:
|
||||
- name: sftp
|
||||
containerPort: 2222
|
||||
- name: http # loopback /qqq/v1 calls and kubelet probes only; not in the Service
|
||||
containerPort: 8000
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: sftp
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: <APP_HEALTH_PATH>
|
||||
port: http
|
||||
periodSeconds: 20
|
||||
failureThreshold: 3
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: <APP_HEALTH_PATH>
|
||||
port: http
|
||||
periodSeconds: 5
|
||||
failureThreshold: 24 # up to 2 minutes for the QInstance
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 768Mi
|
||||
limits:
|
||||
memory: 2Gi # includes the in-memory /tmp below and the bounded transfer buffers
|
||||
securityContext:
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
volumes:
|
||||
- name: tmp # JVM temp only (core stages a rendered report here and deletes it)
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 512Mi
|
||||
---
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: <APP>-file-gateway
|
||||
spec:
|
||||
minAvailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: <APP>-file-gateway
|
||||
@@ -0,0 +1,20 @@
|
||||
# qbit-file-gateway: the SFTP LoadBalancer Service (docs/DESIGN.md section 11).
|
||||
# One port only: v1 is SFTP only, so there is no FTPS passive port range (see DESIGN.md section 15).
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: <APP>-file-gateway
|
||||
annotations:
|
||||
# Cilium LB-IPAM: a fixed address (kof22 on k8s-galaxy: 10.100.22.65, owner decision 2026-10-13).
|
||||
# Cilium 1.15+ reads lbipam.cilium.io/ips; older clusters (and k8s-galaxy manifests) use io.cilium/lb-ipam-ips.
|
||||
lbipam.cilium.io/ips: "10.100.22.65"
|
||||
spec:
|
||||
type: LoadBalancer
|
||||
externalTrafficPolicy: Local # keep the client IP: IP allowlists and lockouts need it
|
||||
selector:
|
||||
app.kubernetes.io/name: <APP>-file-gateway
|
||||
ports:
|
||||
- name: sftp
|
||||
port: 22
|
||||
targetPort: sftp
|
||||
protocol: TCP
|
||||
@@ -22,6 +22,7 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
|
||||
<revision>0.1.0-SNAPSHOT</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
@@ -33,7 +34,7 @@
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>2.2.1</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
@@ -73,10 +74,15 @@
|
||||
</scm>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.bouncycastle</groupId>
|
||||
<artifactId>bcpg-jdk18on</artifactId>
|
||||
<version>1.84</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-object-storage</artifactId>
|
||||
<version>0.1.0-RC.2</version>
|
||||
<version>0.1.0-RC.10</version>
|
||||
</dependency>
|
||||
<!-- MINA's Ed25519 key decoder and signer. -->
|
||||
<dependency>
|
||||
@@ -112,7 +118,7 @@
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-frontend-next</artifactId>
|
||||
<version>1.0.0-RC.14</version>
|
||||
<version>1.0.0-RC.38</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -233,8 +239,8 @@
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<!-- STAND-IN(core=QRun-IO/qqq develop after v4.1.0-RC.5, removeBy=4.1.0-RC.6): UiApiInvoker client channels and QAlerts.notify are on develop only; pin the RC when it is cut -->
|
||||
<qqq.snapshot.version>4.1.0-SNAPSHOT</qqq.snapshot.version>
|
||||
<!-- QQQ 4.1.0-RC.7: the first released core with the UiApiInvoker in-process sessions, streams and multipart (QRun-IO/qqq#1170). Never a SNAPSHOT (guardrail G7). -->
|
||||
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** What a host wires into its ESB, Quick Search and API, without this qbit
|
||||
** depending on those modules (the same opt-in hooks as qbit-sql-workbench).
|
||||
**
|
||||
** <ul>
|
||||
** <li>{@link #esbTableNames()}: publish these tables' inserts, updates and
|
||||
** deletes to an ESB topic (the main tables).</li>
|
||||
** <li>{@link #quickSearchTables()}: index these in Quick Search, with field
|
||||
** weights.</li>
|
||||
** <li>{@link #apiTableNames()}: expose these in the host's QQQ API. The API
|
||||
** gets exactly the admin's rules: the same permissions, customizers and
|
||||
** read-only tables.</li>
|
||||
** </ul>
|
||||
*******************************************************************************/
|
||||
public final class FileGatewayIntegrations
|
||||
{
|
||||
/*******************************************************************************
|
||||
** A table for Quick Search: field weights, the timestamp for incremental
|
||||
** indexing, and the record label.
|
||||
*******************************************************************************/
|
||||
public record SearchableTable(String tableName, Map<String, Integer> fieldWeights, String basepullTimestamp, String recordLabelFormat, List<String> recordLabelFields)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private FileGatewayIntegrations()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tables to publish on the ESB.
|
||||
*******************************************************************************/
|
||||
public static List<String> esbTableNames()
|
||||
{
|
||||
return (List.of(FileGatewayTables.ACCOUNT, FileGatewayTables.ALIAS, FileGatewayTables.UPLOAD_JOB));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tables for the host's QQQ API.
|
||||
*******************************************************************************/
|
||||
public static List<String> apiTableNames()
|
||||
{
|
||||
return (FileGatewayTables.ALL);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tables for Quick Search.
|
||||
*******************************************************************************/
|
||||
public static List<SearchableTable> quickSearchTables()
|
||||
{
|
||||
return (List.of(
|
||||
new SearchableTable(FileGatewayTables.ACCOUNT, Map.of("username", 10, "label", 6), "modifyDate", "%s", List.of("username")),
|
||||
new SearchableTable(FileGatewayTables.ALIAS, Map.of("name", 10, "path", 6), "modifyDate", "%s", List.of("name")),
|
||||
new SearchableTable(FileGatewayTables.UPLOAD_JOB, Map.of("fileName", 10), "modifyDate", "%s", List.of("fileName"))));
|
||||
}
|
||||
}
|
||||
@@ -34,9 +34,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
*******************************************************************************/
|
||||
public class FileGatewayQBitConfig implements QBitConfig
|
||||
{
|
||||
private String schedulerName = null;
|
||||
private Duration serverRetention = Duration.ofDays(1);
|
||||
private String backendName = "rdbms";
|
||||
private boolean sftpEnabled = false;
|
||||
private int sftpPort = 2222;
|
||||
private String publicHostName = null;
|
||||
private String hostKeyEnvVarName = "FILE_GATEWAY_SSH_HOST_KEY";
|
||||
private boolean allowGeneratedHostKey = false;
|
||||
private String fileGatewayBucketAlias = "file-gateway";
|
||||
@@ -121,6 +124,27 @@ public class FileGatewayQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Public SFTP hostname advertised in Preferences, when configured.
|
||||
*******************************************************************************/
|
||||
public String getPublicHostName()
|
||||
{
|
||||
return (publicHostName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set the public SFTP hostname; null omits it from Preferences.
|
||||
*******************************************************************************/
|
||||
public FileGatewayQBitConfig withPublicHostName(String publicHostName)
|
||||
{
|
||||
this.publicHostName = publicHostName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for sftpPort.
|
||||
*******************************************************************************/
|
||||
@@ -738,4 +762,49 @@ public class FileGatewayQBitConfig implements QBitConfig
|
||||
this.tableMetaDataCustomizer = tableMetaDataCustomizer;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for schedulerName.
|
||||
*******************************************************************************/
|
||||
public String getSchedulerName()
|
||||
{
|
||||
return (schedulerName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for schedulerName.
|
||||
*******************************************************************************/
|
||||
public FileGatewayQBitConfig withSchedulerName(String schedulerName)
|
||||
{
|
||||
this.schedulerName = schedulerName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for serverRetention.
|
||||
*******************************************************************************/
|
||||
public Duration getServerRetention()
|
||||
{
|
||||
return (serverRetention);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for serverRetention.
|
||||
*******************************************************************************/
|
||||
public FileGatewayQBitConfig withServerRetention(Duration serverRetention)
|
||||
{
|
||||
this.serverRetention = serverRetention;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
@@ -43,6 +44,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
*******************************************************************************/
|
||||
public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGatewayQBitConfig>
|
||||
{
|
||||
/** Core possible-value sources a host may or may not register. */
|
||||
static final java.util.Set<String> OPTIONAL_HOST_SOURCES = java.util.Set.of("savedView", "savedReport", "savedBulkLoadProfile");
|
||||
|
||||
/*******************************************************************************
|
||||
** The sftp wall owns its list (core's qbit prefixes are transitional and the
|
||||
** assistant may now read audits): partners never reach identity, audits,
|
||||
@@ -149,6 +153,7 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
|
||||
table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
|
||||
}
|
||||
table.setSourceQBitName(qBit.getName());
|
||||
dropMissingPickers(qInstance, table);
|
||||
output.add(table);
|
||||
if(FileGatewayTables.MAIN.contains(table.getName()))
|
||||
{
|
||||
@@ -161,12 +166,17 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
|
||||
.withTables(List.of(FileGatewayTables.UPLOAD_JOB, FileGatewayTables.TRANSFER_LOG, FileGatewayTables.CONNECTION_LOG)));
|
||||
FileGatewayTables.joins().forEach(output::add);
|
||||
FileGatewayTables.childWidgets().forEach(output::add);
|
||||
var home = com.kingsrook.qbits.filegateway.metadata.FileGatewayHome.widgets();
|
||||
home.forEach(output::add);
|
||||
app.withWidgets(home.stream().map(widget -> widget.getName()).toList());
|
||||
output.add(app);
|
||||
FileGatewayProcesses.processes().forEach(output::add);
|
||||
output.add(com.kingsrook.qbits.filegateway.maintenance.GatewayMaintenance.metadata(qBitConfig));
|
||||
output.add(new QAlertTypeMetaData().withName("filegateway.lockout").withLabel("File Gateway Lockout")
|
||||
.withSourceQBitName(qBit.getName()).withSubjectTableName(FileGatewayTables.ACCOUNT)
|
||||
.withVisibilityPermission(FileGatewayPermissions.VIEW));
|
||||
output.addSelfToInstance(qInstance);
|
||||
com.kingsrook.qbits.filegateway.preferences.FileTransferPreferencesProvider.register(qInstance, qBitConfig);
|
||||
ClientChannelsMetaData channels = ClientChannelsMetaData.ofOrWithNew(qInstance)
|
||||
.withChannel("sftp", "SFTP file gateway").withDeniedPattern("sftp", "file_gateway_*")
|
||||
.withDeniedPattern("assistant", "file_gateway_*");
|
||||
@@ -178,6 +188,29 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
|
||||
{
|
||||
channels.withDeniedPattern("sftp", pattern);
|
||||
}
|
||||
com.kingsrook.qbits.filegateway.api.FileGatewayOutbox.setProvider(new com.kingsrook.qbits.filegateway.outbox.OutboxService(qInstance,
|
||||
new com.kingsrook.qbits.filegateway.storage.ObjectStorageGatewayStore(qInstance, qBitConfig.getFileGatewayBucketAlias()), qBitConfig, java.time.Clock.systemUTC()));
|
||||
return (new MetaDataProducerMultiOutput());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Saved views, saved reports and saved bulk-load profiles are optional in a
|
||||
** host. A picker whose possible-value source the host has not registered (yet)
|
||||
** stays a plain id field, so the instance validates; register those core
|
||||
** metadata providers before this producer to get the pickers.
|
||||
*******************************************************************************/
|
||||
static void dropMissingPickers(QInstance qInstance, QTableMetaData table)
|
||||
{
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
String source = field.getPossibleValueSourceName();
|
||||
if(source != null && OPTIONAL_HOST_SOURCES.contains(source) && qInstance.getPossibleValueSource(source) == null)
|
||||
{
|
||||
field.setPossibleValueSourceName(null);
|
||||
field.setPossibleValueSourceFilter(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,7 @@ public final class FileGatewayRuntime implements AutoCloseable
|
||||
return (null);
|
||||
}
|
||||
FileGatewayRuntime runtime = new FileGatewayRuntime(instance, server, config, hostKey(config, environment), Clock.systemUTC(), Duration.ofSeconds(60), environment.apply("HOSTNAME"), new ObjectStorageGatewayStore(instance, config.getFileGatewayBucketAlias()));
|
||||
runtime.worker.withEnvironment(environment);
|
||||
runtime.protocol.start();
|
||||
runtime.worker.start(Duration.ofSeconds(60));
|
||||
return (runtime);
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.maintenance;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
|
||||
import com.kingsrook.qbits.filegateway.FileGatewayQBitProducer;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.outbox.OutboxService;
|
||||
import com.kingsrook.qbits.filegateway.sftp.AuditWriter;
|
||||
import com.kingsrook.qbits.filegateway.storage.GatewayObjectStore;
|
||||
import com.kingsrook.qbits.filegateway.storage.ObjectStorageGatewayStore;
|
||||
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
|
||||
/*******************************************************************************
|
||||
** Idempotent daily cleanup and at-least-once outbox callback retries.
|
||||
*******************************************************************************/
|
||||
public class GatewayMaintenance implements BackendStep
|
||||
{
|
||||
public static final String NAME = "file_gateway_retention";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Hidden host-scheduled process; no navigation or user action is registered.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData metadata(FileGatewayQBitConfig config)
|
||||
{
|
||||
QProcessMetaData process = new QProcessMetaData().withName(NAME).withLabel("File gateway retention").withIsHidden(true)
|
||||
.withPermissionRules(FileGatewayTables.permissionRules(NAME))
|
||||
.withStep(new QBackendStepMetaData().withName("run").withCode(new QCodeReference(GatewayMaintenance.class)));
|
||||
if(config.getSchedulerName() != null)
|
||||
{
|
||||
process.withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName()).withRepeatSeconds(86400).withInitialDelaySeconds(60));
|
||||
}
|
||||
return (process);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve registered host configuration without starting the protocol runtime.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
QInstance instance = QContext.getQInstance();
|
||||
FileGatewayQBitConfig config = (FileGatewayQBitConfig) instance.getQBits().get(FileGatewayQBitProducer.GROUP_ID + ":" + FileGatewayQBitProducer.ARTIFACT_ID).getConfig();
|
||||
maintain(instance, config, new ObjectStorageGatewayStore(instance, config.getFileGatewayBucketAlias()), Clock.systemUTC());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cursor batches remain correct while rows are deleted, and bound memory use.
|
||||
*******************************************************************************/
|
||||
public static void maintain(QInstance instance, FileGatewayQBitConfig config, GatewayObjectStore store, Clock clock) throws QException
|
||||
{
|
||||
AuditWriter audit = new AuditWriter(instance);
|
||||
OutboxService outbox = new OutboxService(instance, store, config, clock);
|
||||
for(String table : List.of(FileGatewayTables.UPLOAD_JOB, FileGatewayTables.OUTBOX_FILE, FileGatewayTables.TRANSFER_LOG, FileGatewayTables.CONNECTION_LOG, FileGatewayTables.SERVER))
|
||||
{
|
||||
int last = 0;
|
||||
while(true)
|
||||
{
|
||||
var rows = audit.query(table, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.GREATER_THAN, last))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)).withLimit(250));
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
last = row.getValueInteger("id");
|
||||
try
|
||||
{
|
||||
if(FileGatewayTables.UPLOAD_JOB.equals(table))
|
||||
{
|
||||
audit.withAccountLock("id", row.getValue("accountId"), () ->
|
||||
{
|
||||
try
|
||||
{
|
||||
cleanUpload(audit, store, audit.query(table, UploadJobs.equal("id", row.getValue("id"))).getFirst(), config, clock.instant());
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new QException("Upload retention storage failure", e));
|
||||
}
|
||||
return (null);
|
||||
});
|
||||
}
|
||||
else if(FileGatewayTables.OUTBOX_FILE.equals(table))
|
||||
{
|
||||
outbox.notifyDownloaded(row);
|
||||
if("REMOVED".equals(row.getValueString("status")) || older(row, "createDate", config.getOutboxRetention(), clock.instant())
|
||||
|| (row.getValueInstant("expiresAt") != null && !row.getValueInstant("expiresAt").isAfter(clock.instant())))
|
||||
{
|
||||
deleteObject(store, row, "storageKey");
|
||||
audit.update(table, new QRecord().withValue("id", row.getValue("id")).withValue("status", "EXPIRED").withValue("storageKey", null));
|
||||
}
|
||||
}
|
||||
else if(FileGatewayTables.SERVER.equals(table))
|
||||
{
|
||||
if(older(row, "lastSeenAt", config.getServerRetention(), clock.instant()))
|
||||
{
|
||||
audit.delete(table, row.getValue("id"));
|
||||
}
|
||||
}
|
||||
else if(older(row, "createDate", config.getLogRetention(), clock.instant())
|
||||
&& (!FileGatewayTables.CONNECTION_LOG.equals(table) || older(row, "lastSeenAt", Duration.ofMinutes(3), clock.instant())))
|
||||
{
|
||||
audit.delete(table, row.getValue("id"));
|
||||
}
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new QException("File gateway retention storage failure", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Never delete an object owned by an active worker; retain rows for the audit trail.
|
||||
*******************************************************************************/
|
||||
private static void cleanUpload(AuditWriter audit, GatewayObjectStore store, QRecord row, FileGatewayQBitConfig config, Instant now) throws IOException, QException
|
||||
{
|
||||
String status = row.getValueString("status");
|
||||
QRecord changes = new QRecord().withValue("id", row.getValue("id"));
|
||||
if("RECEIVING".equals(status) && older(row, "createDate", config.getReceivingStaleAfter(), now))
|
||||
{
|
||||
deleteObject(store, row, "storageKey");
|
||||
changes.withValue("status", "ABORTED").withValue("storageKey", null).withValue("finishedAt", now).withValue("errorMessage", "Incomplete upload expired");
|
||||
}
|
||||
else if(!List.of("RECEIVING", "RUNNING", "QUEUED").contains(status))
|
||||
{
|
||||
if(older(row, "createDate", config.getArchiveRetention(), now))
|
||||
{
|
||||
deleteObject(store, row, "storageKey");
|
||||
deleteObject(store, row, "plaintextStorageKey");
|
||||
changes.withValue("storageKey", null).withValue("plaintextStorageKey", null);
|
||||
}
|
||||
if(older(row, "createDate", config.getResultsRetention(), now))
|
||||
{
|
||||
deleteObject(store, row, "resultsStorageKey");
|
||||
changes.withValue("resultsStorageKey", null).withValue("resultsSizeBytes", null);
|
||||
}
|
||||
}
|
||||
if(changes.getValues().size() > 1)
|
||||
{
|
||||
audit.update(FileGatewayTables.UPLOAD_JOB, changes);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Storage deletes precede database clearing so transient errors can be retried.
|
||||
*******************************************************************************/
|
||||
private static void deleteObject(GatewayObjectStore store, QRecord row, String field) throws IOException
|
||||
{
|
||||
if(row.getValueString(field) != null)
|
||||
{
|
||||
store.delete(row.getValueString(field));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Null timestamps are not evidence that a record is old enough to remove.
|
||||
*******************************************************************************/
|
||||
private static boolean older(QRecord row, String field, Duration age, Instant now)
|
||||
{
|
||||
return (row.getValueInstant(field) != null && !row.getValueInstant(field).plus(age).isAfter(now));
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,27 @@ public class AccountCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
record.addError(new BadInputStatusMessage("Username must be 3 to 64 lowercase letters, digits, dots, underscores or hyphens"));
|
||||
}
|
||||
if(oldRecords.isEmpty() && record.getValue("rootMode") == null)
|
||||
{
|
||||
record.setValue("rootMode", "SERVICE".equals(record.getValueString("accountType")) ? "ENDPOINTS_ONLY" : "FULL_TREE");
|
||||
}
|
||||
String pgpKey = record.getValueString("pgpPublicKey");
|
||||
if(pgpKey != null && !pgpKey.isBlank())
|
||||
{
|
||||
try
|
||||
{
|
||||
com.kingsrook.qbits.filegateway.pgp.PgpStreams.encryptionKey(pgpKey);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
record.addError(new BadInputStatusMessage("PGP public key must parse and contain a valid encryption subkey"));
|
||||
}
|
||||
}
|
||||
if(Boolean.TRUE.equals(record.getValueBoolean("pgpEncryptDownloads")) && (pgpKey == null || pgpKey.isBlank())
|
||||
&& oldRecords.orElse(List.of()).stream().noneMatch(old -> java.util.Objects.equals(old.getValue("id"), record.getValue("id")) && old.getValueString("pgpPublicKey") != null))
|
||||
{
|
||||
record.addError(new BadInputStatusMessage("PGP download encryption requires a public key"));
|
||||
}
|
||||
for(String limit : List.of("maxConcurrentSessions", "maxOperationsPerMinute", "maxFileSizeMb", "maxUploadMbPerDay", "maxDownloadMbPerDay"))
|
||||
{
|
||||
if(record.getValueInteger(limit) != null && record.getValueInteger(limit) < 1)
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.metadata;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qbits.filegateway.sftp.AuditWriter;
|
||||
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
|
||||
/*******************************************************************************
|
||||
** Endpoints are canonical mounts, never a way to shadow application paths.
|
||||
*******************************************************************************/
|
||||
public class AliasCustomizer implements TableCustomizerInterface
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Validate the merged row so sparse updates cannot bypass mount validation.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean preview, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
AuditWriter audit = new AuditWriter(QContext.getQInstance());
|
||||
for(QRecord record : records)
|
||||
{
|
||||
QRecord merged = new QRecord();
|
||||
oldRecords.orElse(List.of()).stream().filter(old -> java.util.Objects.equals(old.getValue("id"), record.getValue("id")))
|
||||
.findFirst().ifPresent(old -> merged.getValues().putAll(old.getValues()));
|
||||
merged.getValues().putAll(record.getValues());
|
||||
String path = merged.getValueString("path");
|
||||
String error = null;
|
||||
if(path == null || !path.matches("/[a-z0-9._/-]+") || path.contains("..") || path.contains("//") || path.endsWith("/") || List.of(path.split("/")).contains("."))
|
||||
{
|
||||
error = "Endpoint path must be absolute, use lowercase letters, digits, dots, underscores, hyphens and slashes, and contain no dot segments or empty segments";
|
||||
}
|
||||
else
|
||||
{
|
||||
boolean fullTree = merged.getValue("accountId") == null;
|
||||
if(!fullTree)
|
||||
{
|
||||
var accounts = audit.query(FileGatewayTables.ACCOUNT, UploadJobs.equal("id", merged.getValue("accountId")));
|
||||
fullTree = accounts.isEmpty() || !"ENDPOINTS_ONLY".equals(accounts.getFirst().getValueString("rootMode"));
|
||||
}
|
||||
if(fullTree && QContext.getQInstance().getApps().containsKey(path.substring(1).split("/")[0]))
|
||||
{
|
||||
error = "Endpoint path collides with an application name for a FULL_TREE account";
|
||||
}
|
||||
for(QRecord other : audit.query(FileGatewayTables.ALIAS, new QQueryFilter()))
|
||||
{
|
||||
String mount = other.getValueString("path");
|
||||
if(!java.util.Objects.equals(other.getValue("id"), merged.getValue("id")) && mount != null
|
||||
&& (path.equals(mount) || path.startsWith(mount + "/") || mount.startsWith(path + "/")))
|
||||
{
|
||||
error = "Endpoint paths must be unique and must not overlap another endpoint mount";
|
||||
}
|
||||
}
|
||||
}
|
||||
String filename = merged.getValueString("fileName");
|
||||
if(filename != null && !validFileName(filename))
|
||||
{
|
||||
error = "Endpoint file name must be a single safe filename";
|
||||
}
|
||||
if(error != null)
|
||||
{
|
||||
record.addError(new BadInputStatusMessage(error));
|
||||
}
|
||||
}
|
||||
return (records);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Object names cannot escape a mount or inject control characters.
|
||||
*******************************************************************************/
|
||||
public static boolean validFileName(String name)
|
||||
{
|
||||
return (name != null && !name.isBlank() && !name.equals(".") && !name.equals("..") && !name.contains("/")
|
||||
&& !name.contains("\\") && !name.contains(":") && name.chars().noneMatch(Character::isISOControl));
|
||||
}
|
||||
}
|
||||
@@ -73,7 +73,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -112,7 +112,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,7 +151,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,7 +190,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,7 +229,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,7 +268,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -307,7 +307,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -346,7 +346,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -385,7 +385,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,7 +424,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -463,7 +463,7 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -502,7 +502,18 @@ public final class FileGatewayEnums
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (value);
|
||||
return (label(value));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain label for a persisted value: ENDPOINTS_ONLY becomes "Endpoints only".
|
||||
*******************************************************************************/
|
||||
static String label(String value)
|
||||
{
|
||||
String words = value.replace('_', ' ').toLowerCase();
|
||||
return (Character.toUpperCase(words.charAt(0)) + words.substring(1));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.metadata;
|
||||
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
|
||||
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBlockData;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextValues;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
|
||||
|
||||
/*******************************************************************************
|
||||
** Typed home widgets over real gateway data, queried as the viewing user.
|
||||
*******************************************************************************/
|
||||
public class FileGatewayHome extends AbstractWidgetRenderer
|
||||
{
|
||||
/** Readable UTC timestamps in home tables (the widget title already names the table). */
|
||||
private static final java.time.format.DateTimeFormatter WHEN = java.time.format.DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm 'UTC'").withZone(ZoneOffset.UTC);
|
||||
|
||||
public static final List<String> KEYS = List.of("transfers", "failures", "sessions", "in", "out", "locked", "recent", "partners", "active", "servers");
|
||||
private static final List<String> LABELS = List.of("Transfers today", "Failed jobs (24 hours)", "Active sessions", "MB in today", "MB out today", "Locked accounts",
|
||||
"Recent failures", "Top partners (7 days)", "Active sessions", "Server");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Six small statistics followed by full-width responsive tables, without HTML.
|
||||
*******************************************************************************/
|
||||
public static List<QWidgetMetaData> widgets()
|
||||
{
|
||||
List<QWidgetMetaData> widgets = new ArrayList<>();
|
||||
for(int i = 0; i < KEYS.size(); i++)
|
||||
{
|
||||
widgets.add(new QWidgetMetaData().withName("file_gateway_home_" + KEYS.get(i)).withLabel(LABELS.get(i))
|
||||
.withType((i < 6 ? WidgetType.STATISTICS : WidgetType.TABLE).getType()).withGridColumns(i < 6 ? 2 : 12)
|
||||
.withIsCard(true).withShowReloadButton(true).withCodeReference(new QCodeReference(FileGatewayHome.class))
|
||||
.withPermissionRules(FileGatewayTables.permissionRules("file_gateway")));
|
||||
}
|
||||
return (widgets);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The same permission is required for direct renderer calls and widget routes.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
return (new RenderWidgetOutput(data(input.getWidgetMetaData().getName().replace("file_gateway_home_", ""), Instant.now())));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Accept an explicit time for deterministic seeded-data assertions.
|
||||
*******************************************************************************/
|
||||
public QWidgetData data(String key, Instant now) throws QException
|
||||
{
|
||||
if(QContext.getQSession() == null || !(QContext.getQSession().hasPermission(FileGatewayPermissions.VIEW) || QContext.getQSession().hasPermission(FileGatewayPermissions.ADMIN)))
|
||||
{
|
||||
throw (new QPermissionDeniedException("Missing file_gateway.view"));
|
||||
}
|
||||
Instant today = now.atZone(ZoneOffset.UTC).toLocalDate().atStartOfDay(ZoneOffset.UTC).toInstant();
|
||||
if(List.of("transfers", "in", "out").contains(key))
|
||||
{
|
||||
var rows = query(FileGatewayTables.TRANSFER_LOG, since("createDate", today));
|
||||
long value = "transfers".equals(key) ? rows.size() : rows.stream()
|
||||
.filter(row -> ("in".equals(key) ? "UPLOAD" : "DOWNLOAD").equals(row.getValueString("operation")))
|
||||
.mapToLong(row -> number(row, "bytes")).sum();
|
||||
return (new StatisticsData("transfers".equals(key) ? value : Math.round(value / 10485.76) / 100.0, null, null).withIncreaseIsGood(true));
|
||||
}
|
||||
if("locked".equals(key))
|
||||
{
|
||||
long count = query(FileGatewayTables.ACCOUNT, new QQueryFilter()).stream().filter(row -> "LOCKED".equals(row.getValueString("status"))
|
||||
|| (row.getValueInstant("lockedUntil") != null && row.getValueInstant("lockedUntil").isAfter(now))).count();
|
||||
return (new StatisticsData(count, null, null).withIncreaseIsGood(false));
|
||||
}
|
||||
if(List.of("failures", "recent").contains(key))
|
||||
{
|
||||
var rows = query(FileGatewayTables.UPLOAD_JOB, since("finishedAt", now.minusSeconds("failures".equals(key) ? 86400 : 7 * 86400))
|
||||
.withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, "FAILED", "PARTIAL", "REJECTED")));
|
||||
if("failures".equals(key))
|
||||
{
|
||||
return (new StatisticsData(rows.size(), null, null).withIncreaseIsGood(false));
|
||||
}
|
||||
rows.sort(java.util.Comparator.comparing((QRecord row) -> row.getValueInstant("finishedAt")).reversed());
|
||||
return (table("Recent failures", FileGatewayTables.UPLOAD_JOB, rows.stream().limit(20).toList(),
|
||||
List.of("fileName", "status", "errorMessage", "finishedAt"), List.of("File", "Status", "Message", "Finished"), "No failed transfers in the last 7 days."));
|
||||
}
|
||||
if(List.of("sessions", "active").contains(key))
|
||||
{
|
||||
var rows = query(FileGatewayTables.CONNECTION_LOG, since("lastSeenAt", now.minusSeconds(180))
|
||||
.withCriteria(new QFilterCriteria("disconnectedAt", QCriteriaOperator.IS_BLANK))
|
||||
.withCriteria(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "CONNECTED")));
|
||||
return ("sessions".equals(key) ? new StatisticsData(rows.size(), null, null).withIncreaseIsGood(true) : table("Active sessions", FileGatewayTables.CONNECTION_LOG, rows,
|
||||
List.of("username", "remoteIp", "connectedAt", "operationCount"), List.of("Username", "IP", "Connected", "Operations"),
|
||||
"No one is connected right now (no session has been seen in the last 3 minutes)."));
|
||||
}
|
||||
if("servers".equals(key))
|
||||
{
|
||||
return (table("Server", FileGatewayTables.SERVER, query(FileGatewayTables.SERVER, since("lastSeenAt", now.minusSeconds(180))),
|
||||
List.of("podName", "hostKeyFingerprint", "port", "lastSeenAt"), List.of("Pod", "Host key fingerprint", "Port", "Last seen"),
|
||||
"No gateway server has reported in during the last 3 minutes."));
|
||||
}
|
||||
Map<Integer, QRecord> partners = new LinkedHashMap<>();
|
||||
for(QRecord account : query(FileGatewayTables.ACCOUNT, new QQueryFilter()))
|
||||
{
|
||||
partners.put(account.getValueInteger("id"), account.withValue("transfers", 0L).withValue("mb", 0.0));
|
||||
}
|
||||
for(QRecord transfer : query(FileGatewayTables.TRANSFER_LOG, since("createDate", now.minusSeconds(7 * 86400))))
|
||||
{
|
||||
QRecord partner = partners.get(transfer.getValueInteger("accountId"));
|
||||
if(partner != null)
|
||||
{
|
||||
partner.withValue("transfers", number(partner, "transfers") + 1)
|
||||
.withValue("mb", ((Number) partner.getValue("mb")).doubleValue() + number(transfer, "bytes") / 1048576.0);
|
||||
}
|
||||
}
|
||||
List<QRecord> rows = partners.values().stream().filter(row -> number(row, "transfers") > 0)
|
||||
.sorted(java.util.Comparator.comparingLong((QRecord row) -> number(row, "transfers")).reversed()).limit(10).toList();
|
||||
return (table("Top partners (7 days)", FileGatewayTables.ACCOUNT, rows, List.of("username", "transfers", "mb"), List.of("Partner", "Transfers", "MB"),
|
||||
"No partner has transferred a file in the last 7 days."));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Typed text blocks carry record links; cells never contain markup.
|
||||
*******************************************************************************/
|
||||
private TableData table(String label, String table, List<QRecord> records, List<String> fields, List<String> labels, String noRowsText)
|
||||
{
|
||||
List<TableData.Column> columns = new ArrayList<>();
|
||||
for(int i = 0; i < fields.size(); i++)
|
||||
{
|
||||
columns.add(new TableData.Column(i == 0 ? "block" : "default", labels.get(i), fields.get(i), null, null));
|
||||
}
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
for(String field : fields)
|
||||
{
|
||||
Object value = record.getValue(field);
|
||||
row.put(field, value == null ? "" : value instanceof Instant instant ? WHEN.format(instant) : value);
|
||||
}
|
||||
row.put(fields.getFirst(), new TextBlockData().withValues(new TextValues(record.getValueString(fields.getFirst())))
|
||||
.withLink("/record/" + table + "/" + record.getValue("id")));
|
||||
rows.add(row);
|
||||
}
|
||||
return (new TableData(null, columns, rows).withRowsPerPage(10).withHidePaginationDropdown(true).withNoRowsFoundHTML(noRowsText));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Queries preserve viewer security and use bounded database pages.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> query(String table, QQueryFilter filter) throws QException
|
||||
{
|
||||
List<QRecord> rows = new ArrayList<>();
|
||||
for(int offset = 0; ; offset += 250)
|
||||
{
|
||||
var page = new QueryAction().execute(new QueryInput(table).withFilter(filter.withSkip(offset).withLimit(250).withOrderBy(new QFilterOrderBy("id", true)))).getRecords();
|
||||
rows.addAll(page);
|
||||
if(page.size() < 250)
|
||||
{
|
||||
return (rows);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** UTC time windows agree with quota and transfer bookkeeping.
|
||||
*******************************************************************************/
|
||||
private QQueryFilter since(String field, Instant from)
|
||||
{
|
||||
return (new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.GREATER_THAN_OR_EQUALS, from)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Older audit rows may have no counters.
|
||||
*******************************************************************************/
|
||||
private long number(QRecord row, String field)
|
||||
{
|
||||
return (row.getValueLong(field) == null ? 0 : row.getValueLong(field));
|
||||
}
|
||||
}
|
||||
@@ -229,7 +229,7 @@ public final class FileGatewayProcesses
|
||||
case END_SESSION ->
|
||||
{
|
||||
table = FileGatewayTables.CONNECTION_LOG;
|
||||
update.withValue("disconnectedAt", Instant.now()).withValue("disconnectReason", "Ended by administrator");
|
||||
update.withValue("endRequestedAt", Instant.now()).withValue("disconnectReason", "Ended by administrator");
|
||||
}
|
||||
default -> throw (new QException("Unsupported gateway action"));
|
||||
}
|
||||
|
||||
@@ -62,9 +62,10 @@ public final class FileGatewayTables
|
||||
public static final String OUTBOX_FILE = "file_gateway_outbox_file";
|
||||
public static final String TRANSFER_LOG = "file_gateway_transfer_log";
|
||||
public static final String CONNECTION_LOG = "file_gateway_connection_log";
|
||||
public static final List<String> ALL = List.of(ACCOUNT, SSH_KEY, ALIAS, UPLOAD_JOB, OUTBOX_FILE, TRANSFER_LOG, CONNECTION_LOG);
|
||||
public static final String SERVER = "file_gateway_server";
|
||||
public static final List<String> ALL = List.of(ACCOUNT, SSH_KEY, ALIAS, UPLOAD_JOB, OUTBOX_FILE, TRANSFER_LOG, CONNECTION_LOG, SERVER);
|
||||
public static final List<String> MAIN = List.of(ACCOUNT, ALIAS, UPLOAD_JOB, TRANSFER_LOG, CONNECTION_LOG);
|
||||
public static final List<String> SYSTEM_WRITTEN = List.of(UPLOAD_JOB, OUTBOX_FILE, TRANSFER_LOG, CONNECTION_LOG);
|
||||
public static final List<String> SYSTEM_WRITTEN = List.of(UPLOAD_JOB, OUTBOX_FILE, TRANSFER_LOG, CONNECTION_LOG, SERVER);
|
||||
|
||||
|
||||
|
||||
@@ -112,12 +113,45 @@ public final class FileGatewayTables
|
||||
|
||||
|
||||
|
||||
/** Plain labels for child lists and their record-screen tabs. */
|
||||
static final java.util.Map<String, String> CHILD_LABELS = java.util.Map.of(
|
||||
SSH_KEY, "SSH keys", UPLOAD_JOB, "Upload jobs", CONNECTION_LOG, "Connections",
|
||||
TRANSFER_LOG, "Transfers", OUTBOX_FILE, "Outbox files");
|
||||
|
||||
/** Plain field labels where the name-derived label reads badly. */
|
||||
static final java.util.Map<String, String> FIELD_LABELS = java.util.Map.ofEntries(
|
||||
java.util.Map.entry("allowedCidrs", "Allowed CIDRs"), java.util.Map.entry("userId", "User"),
|
||||
java.util.Map.entry("ownerUserId", "Owner"), java.util.Map.entry("maxFileSizeMb", "Max file size (MB)"),
|
||||
java.util.Map.entry("maxUploadMbPerDay", "Max upload per day (MB)"), java.util.Map.entry("maxDownloadMbPerDay", "Max download per day (MB)"),
|
||||
java.util.Map.entry("lastLoginIp", "Last login IP"), java.util.Map.entry("lastUsedIp", "Last used IP"),
|
||||
java.util.Map.entry("pgpPublicKey", "PGP public key"), java.util.Map.entry("pgpDecryptUploads", "Decrypt PGP uploads"),
|
||||
java.util.Map.entry("pgpEncryptDownloads", "Encrypt downloads with PGP"), java.util.Map.entry("remoteIp", "Remote IP"),
|
||||
java.util.Map.entry("remoteAddress", "Remote IP"), java.util.Map.entry("sha256", "SHA-256"),
|
||||
java.util.Map.entry("processUuid", "Process UUID"), java.util.Map.entry("savedBulkLoadProfileId", "Bulk load profile"),
|
||||
java.util.Map.entry("savedViewId", "Saved view"), java.util.Map.entry("savedReportId", "Saved report"),
|
||||
java.util.Map.entry("sizeBytes", "Size (bytes)"), java.util.Map.entry("accountId", "Account"),
|
||||
java.util.Map.entry("aliasId", "Endpoint"), java.util.Map.entry("connectionLogId", "Connection"),
|
||||
java.util.Map.entry("hostKeyFingerprint", "Host key fingerprint"), java.util.Map.entry("keyFingerprint", "Key fingerprint"));
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build all seven tables.
|
||||
*******************************************************************************/
|
||||
public static List<QTableMetaData> tables(FileGatewayQBitConfig config)
|
||||
{
|
||||
return (List.of(account(config), ssh_key(config), alias(config), upload_job(config), outbox_file(config), transfer_log(config), connection_log(config)));
|
||||
List<QTableMetaData> tables = List.of(account(config), ssh_key(config), alias(config), upload_job(config), outbox_file(config), transfer_log(config), connection_log(config), server(config));
|
||||
for(QTableMetaData table : tables)
|
||||
{
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
if(FIELD_LABELS.containsKey(field.getName()))
|
||||
{
|
||||
field.setLabel(FIELD_LABELS.get(field.getName()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
@@ -160,7 +194,7 @@ public final class FileGatewayTables
|
||||
.withField(field("userId", QFieldType.STRING).withIsRequired(true))
|
||||
.withField(field("ownerUserId", QFieldType.STRING))
|
||||
.withField(field("status", QFieldType.STRING).withPossibleValueSourceName("file_gateway_account_status").withDefaultValue("ACTIVE").withIsRequired(true))
|
||||
.withField(field("rootMode", QFieldType.STRING).withPossibleValueSourceName("file_gateway_root_mode").withDefaultValue("FULL_TREE").withIsRequired(true))
|
||||
.withField(field("rootMode", QFieldType.STRING).withPossibleValueSourceName("file_gateway_root_mode"))
|
||||
.withField(field("allowedCidrs", QFieldType.TEXT))
|
||||
.withField(field("expiresAt", QFieldType.DATE_TIME))
|
||||
.withField(field("formats", QFieldType.STRING).withDefaultValue("csv,xlsx"))
|
||||
@@ -187,9 +221,9 @@ public final class FileGatewayTables
|
||||
table.withSection(SectionFactory.defaultT1("id", "username", "label", "accountType", "userId", "ownerUserId"));
|
||||
table.withSection(SectionFactory.defaultT2("status", "rootMode", "allowedCidrs", "expiresAt", "formats", "maxConcurrentSessions", "maxOperationsPerMinute", "maxFileSizeMb", "maxUploadMbPerDay", "maxDownloadMbPerDay", "dedupeUploads", "notifyOnSuccess", "pgpPublicKey", "pgpDecryptUploads", "pgpEncryptDownloads", "operationWindowAt", "operationWindowCount", "failedLoginCount", "failedLoginWindowAt", "lockedUntil", "lastLoginAt", "lastLoginIp"));
|
||||
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
|
||||
table.withSection(SectionFactory.customT2(SSH_KEY, new QIcon("list")).withWidgetName(ACCOUNT + "_children_" + SSH_KEY));
|
||||
table.withSection(SectionFactory.customT2(UPLOAD_JOB, new QIcon("list")).withWidgetName(ACCOUNT + "_children_" + UPLOAD_JOB));
|
||||
table.withSection(SectionFactory.customT2(CONNECTION_LOG, new QIcon("list")).withWidgetName(ACCOUNT + "_children_" + CONNECTION_LOG));
|
||||
table.withSection(SectionFactory.customT2(SSH_KEY, new QIcon("list")).withLabel(CHILD_LABELS.get(SSH_KEY)).withWidgetName(ACCOUNT + "_children_" + SSH_KEY));
|
||||
table.withSection(SectionFactory.customT2(UPLOAD_JOB, new QIcon("list")).withLabel(CHILD_LABELS.get(UPLOAD_JOB)).withWidgetName(ACCOUNT + "_children_" + UPLOAD_JOB));
|
||||
table.withSection(SectionFactory.customT2(CONNECTION_LOG, new QIcon("list")).withLabel(CHILD_LABELS.get(CONNECTION_LOG)).withWidgetName(ACCOUNT + "_children_" + CONNECTION_LOG));
|
||||
return (table);
|
||||
}
|
||||
|
||||
@@ -246,11 +280,13 @@ public final class FileGatewayTables
|
||||
.withField(field("description", QFieldType.TEXT));
|
||||
table.withUniqueKey(new UniqueKey("name"));
|
||||
table.withUniqueKey(new UniqueKey("path"));
|
||||
table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(AliasCustomizer.class));
|
||||
table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, new QCodeReference(AliasCustomizer.class));
|
||||
table.withSection(SectionFactory.defaultT1("id", "name", "path", "kind", "accountId", "tableName"));
|
||||
table.withSection(SectionFactory.defaultT2("savedBulkLoadProfileId", "savedViewId", "savedReportId", "format", "fileName", "consumerName", "isActive", "description"));
|
||||
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
|
||||
table.withSection(SectionFactory.customT2(UPLOAD_JOB, new QIcon("list")).withWidgetName(ALIAS + "_children_" + UPLOAD_JOB));
|
||||
table.withSection(SectionFactory.customT2(OUTBOX_FILE, new QIcon("list")).withWidgetName(ALIAS + "_children_" + OUTBOX_FILE));
|
||||
table.withSection(SectionFactory.customT2(UPLOAD_JOB, new QIcon("list")).withLabel(CHILD_LABELS.get(UPLOAD_JOB)).withWidgetName(ALIAS + "_children_" + UPLOAD_JOB));
|
||||
table.withSection(SectionFactory.customT2(OUTBOX_FILE, new QIcon("list")).withLabel(CHILD_LABELS.get(OUTBOX_FILE)).withWidgetName(ALIAS + "_children_" + OUTBOX_FILE));
|
||||
return (table);
|
||||
}
|
||||
|
||||
@@ -261,7 +297,7 @@ public final class FileGatewayTables
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData upload_job(FileGatewayQBitConfig config)
|
||||
{
|
||||
QTableMetaData table = base(config, UPLOAD_JOB, "Upload Jobs").withRecordVersionField("status")
|
||||
QTableMetaData table = base(config, UPLOAD_JOB, "Upload Jobs")
|
||||
.withRecordLabelFormatAndFields("%s", "id")
|
||||
.withField(field("accountId", QFieldType.INTEGER).withPossibleValueSourceName(ACCOUNT))
|
||||
.withField(field("aliasId", QFieldType.INTEGER).withPossibleValueSourceName(ALIAS))
|
||||
@@ -284,6 +320,9 @@ public final class FileGatewayTables
|
||||
.withField(field("resultSummary", QFieldType.TEXT))
|
||||
.withField(field("resultsStorageKey", QFieldType.STRING))
|
||||
.withField(field("resultsSizeBytes", QFieldType.LONG))
|
||||
.withField(field("plaintextStorageKey", QFieldType.STRING))
|
||||
.withField(field("plaintextSha256", QFieldType.STRING))
|
||||
.withField(field("plaintextSizeBytes", QFieldType.LONG))
|
||||
.withField(field("errorMessage", QFieldType.TEXT))
|
||||
.withField(field("attempts", QFieldType.INTEGER))
|
||||
.withField(field("nextAttemptAt", QFieldType.DATE_TIME))
|
||||
@@ -294,9 +333,9 @@ public final class FileGatewayTables
|
||||
.withField(field("finishedAt", QFieldType.DATE_TIME))
|
||||
.withField(field("acknowledgedAt", QFieldType.DATE_TIME));
|
||||
table.withSection(SectionFactory.defaultT1("id", "accountId", "aliasId", "connectionLogId", "remoteAddress", "tableName"));
|
||||
table.withSection(SectionFactory.defaultT2("savedBulkLoadProfileId", "consumerName", "consumerReference", "path", "fileName", "sizeBytes", "sha256", "storageKey", "status", "duplicateOfJobId", "processUuid", "insertedCount", "errorCount", "resultSummary", "resultsStorageKey", "resultsSizeBytes", "errorMessage", "attempts", "nextAttemptAt", "claimedBy", "claimedAt", "queuedAt", "startedAt", "finishedAt", "acknowledgedAt"));
|
||||
table.withSection(SectionFactory.defaultT2("savedBulkLoadProfileId", "consumerName", "consumerReference", "path", "fileName", "sizeBytes", "sha256", "storageKey", "status", "duplicateOfJobId", "processUuid", "insertedCount", "errorCount", "resultSummary", "resultsStorageKey", "resultsSizeBytes", "plaintextStorageKey", "plaintextSha256", "plaintextSizeBytes", "errorMessage", "attempts", "nextAttemptAt", "claimedBy", "claimedAt", "queuedAt", "startedAt", "finishedAt", "acknowledgedAt"));
|
||||
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
|
||||
table.withSection(SectionFactory.customT2(TRANSFER_LOG, new QIcon("list")).withWidgetName(UPLOAD_JOB + "_children_" + TRANSFER_LOG));
|
||||
table.withSection(SectionFactory.customT2(TRANSFER_LOG, new QIcon("list")).withLabel(CHILD_LABELS.get(TRANSFER_LOG)).withWidgetName(UPLOAD_JOB + "_children_" + TRANSFER_LOG));
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
return (table);
|
||||
}
|
||||
@@ -318,11 +357,12 @@ public final class FileGatewayTables
|
||||
.withField(field("status", QFieldType.STRING).withPossibleValueSourceName("file_gateway_outbox_status"))
|
||||
.withField(field("firstDownloadedAt", QFieldType.DATE_TIME))
|
||||
.withField(field("downloadCount", QFieldType.INTEGER))
|
||||
.withField(field("onDownloadedPending", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(field("removedAt", QFieldType.DATE_TIME))
|
||||
.withField(field("expiresAt", QFieldType.DATE_TIME))
|
||||
.withField(field("createdByConsumer", QFieldType.STRING));
|
||||
table.withSection(SectionFactory.defaultT1("id", "aliasId", "fileName", "sizeBytes", "sha256", "storageKey"));
|
||||
table.withSection(SectionFactory.defaultT2("status", "firstDownloadedAt", "downloadCount", "removedAt", "expiresAt", "createdByConsumer"));
|
||||
table.withSection(SectionFactory.defaultT2("status", "firstDownloadedAt", "downloadCount", "onDownloadedPending", "removedAt", "expiresAt", "createdByConsumer"));
|
||||
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
return (table);
|
||||
@@ -378,13 +418,14 @@ public final class FileGatewayTables
|
||||
.withField(field("connectedAt", QFieldType.DATE_TIME))
|
||||
.withField(field("lastSeenAt", QFieldType.DATE_TIME))
|
||||
.withField(field("disconnectedAt", QFieldType.DATE_TIME))
|
||||
.withField(field("endRequestedAt", QFieldType.DATE_TIME))
|
||||
.withField(field("disconnectReason", QFieldType.TEXT))
|
||||
.withField(field("bytesIn", QFieldType.LONG))
|
||||
.withField(field("bytesOut", QFieldType.LONG))
|
||||
.withField(field("operationCount", QFieldType.LONG))
|
||||
.withField(field("clientChannel", QFieldType.STRING));
|
||||
table.withSection(SectionFactory.defaultT1("id", "accountId", "username", "remoteIp", "clientVersion", "keyFingerprint"));
|
||||
table.withSection(SectionFactory.defaultT2("outcome", "sessionId", "podName", "connectedAt", "lastSeenAt", "disconnectedAt", "disconnectReason", "bytesIn", "bytesOut", "operationCount", "clientChannel"));
|
||||
table.withSection(SectionFactory.defaultT2("outcome", "sessionId", "podName", "connectedAt", "lastSeenAt", "disconnectedAt", "endRequestedAt", "disconnectReason", "bytesIn", "bytesOut", "operationCount", "clientChannel"));
|
||||
table.withSection(SectionFactory.defaultT3("createDate", "modifyDate"));
|
||||
table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE);
|
||||
return (table);
|
||||
@@ -392,6 +433,25 @@ public final class FileGatewayTables
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Gateway heartbeat shared with the main web deployment, never a navigation table.
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData server(FileGatewayQBitConfig config)
|
||||
{
|
||||
return (base(config, SERVER, "Servers").withRecordLabelFormatAndFields("%s", "podName")
|
||||
.withField(field("podName", QFieldType.STRING).withIsRequired(true))
|
||||
.withField(field("hostKeyFingerprint", QFieldType.STRING))
|
||||
.withField(field("port", QFieldType.INTEGER))
|
||||
.withField(field("startedAt", QFieldType.DATE_TIME))
|
||||
.withField(field("lastSeenAt", QFieldType.DATE_TIME))
|
||||
.withUniqueKey(new UniqueKey("podName"))
|
||||
.withSection(SectionFactory.defaultT1("id", "podName", "hostKeyFingerprint", "port", "startedAt", "lastSeenAt"))
|
||||
.withSection(SectionFactory.defaultT3("createDate", "modifyDate"))
|
||||
.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE, Capability.TABLE_DELETE));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Parent and child relationships.
|
||||
*******************************************************************************/
|
||||
@@ -423,7 +483,7 @@ public final class FileGatewayTables
|
||||
for(QJoinMetaData join : joins())
|
||||
{
|
||||
widgets.add(ChildRecordListRenderer.widgetMetaDataBuilder(join)
|
||||
.withName(join.getLeftTable() + "_children_" + join.getRightTable()).withLabel(join.getRightTable().replace("file_gateway_", "").replace("_", " "))
|
||||
.withName(join.getLeftTable() + "_children_" + join.getRightTable()).withLabel(CHILD_LABELS.getOrDefault(join.getRightTable(), join.getRightTable()))
|
||||
.withCanAddChildRecord(SSH_KEY.equals(join.getRightTable())).withMaxRows(100)
|
||||
.getWidgetMetaData().withGridColumns(12).withPermissionRules(permissionRules(join.getRightTable())));
|
||||
}
|
||||
|
||||
@@ -22,12 +22,21 @@ package com.kingsrook.qbits.filegateway.metadata;
|
||||
|
||||
|
||||
import java.security.PublicKey;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
@@ -47,8 +56,9 @@ public class SshKeyCustomizer implements TableCustomizerInterface
|
||||
** Parse and validate inserts and partial updates without logging key material.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecords)
|
||||
public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
Set<String> fingerprints = new HashSet<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
QRecord old = oldRecords.orElse(List.of()).stream().filter(candidate -> Objects.equals(candidate.getValue("id"), record.getValue("id"))).findFirst().orElse(null);
|
||||
@@ -69,6 +79,19 @@ public class SshKeyCustomizer implements TableCustomizerInterface
|
||||
catch(Exception e)
|
||||
{
|
||||
record.addError(new BadInputStatusMessage("Use an OpenSSH Ed25519, ECDSA, or RSA key of at least 3072 bits"));
|
||||
continue;
|
||||
}
|
||||
// Core's unique-key check precedes this computed fingerprint.
|
||||
QQueryFilter filter = new QQueryFilter().withCriteria("fingerprint", QCriteriaOperator.EQUALS, record.getValue("fingerprint"));
|
||||
if(old != null)
|
||||
{
|
||||
filter.withCriteria("id", QCriteriaOperator.NOT_EQUALS, old.getValue("id"));
|
||||
}
|
||||
var transaction = input instanceof InsertInput insert ? insert.getTransaction() : ((UpdateInput) input).getTransaction();
|
||||
if(!fingerprints.add(record.getValueString("fingerprint")) || !new QueryAction().execute(new QueryInput(FileGatewayTables.SSH_KEY)
|
||||
.withFilter(filter.withLimit(1)).withTransaction(transaction)).getRecords().isEmpty())
|
||||
{
|
||||
record.addError(new BadInputStatusMessage("Fingerprint: this SSH key is already registered. Use a different key."));
|
||||
}
|
||||
}
|
||||
return (records);
|
||||
|
||||
@@ -94,6 +94,10 @@ public class FileGatewayConnectionLog extends QRecordEntity
|
||||
|
||||
|
||||
|
||||
@QField
|
||||
private Instant endRequestedAt;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create an empty entity.
|
||||
*******************************************************************************/
|
||||
@@ -699,4 +703,35 @@ public class FileGatewayConnectionLog extends QRecordEntity
|
||||
{
|
||||
this.clientChannel = clientChannel;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for endRequestedAt.
|
||||
*******************************************************************************/
|
||||
public Instant getEndRequestedAt()
|
||||
{
|
||||
return (endRequestedAt);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter used by QRecordEntity population.
|
||||
*******************************************************************************/
|
||||
public void setEndRequestedAt(Instant endRequestedAt)
|
||||
{
|
||||
this.endRequestedAt = endRequestedAt;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for endRequestedAt.
|
||||
*******************************************************************************/
|
||||
public FileGatewayConnectionLog withEndRequestedAt(Instant endRequestedAt)
|
||||
{
|
||||
this.endRequestedAt = endRequestedAt;
|
||||
return (this);
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,10 @@ public class FileGatewayOutboxFile extends QRecordEntity
|
||||
|
||||
|
||||
|
||||
@QField
|
||||
private Boolean onDownloadedPending;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create an empty entity.
|
||||
*******************************************************************************/
|
||||
@@ -529,4 +533,35 @@ public class FileGatewayOutboxFile extends QRecordEntity
|
||||
{
|
||||
this.createdByConsumer = createdByConsumer;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for onDownloadedPending.
|
||||
*******************************************************************************/
|
||||
public Boolean getOnDownloadedPending()
|
||||
{
|
||||
return (onDownloadedPending);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter used by QRecordEntity population.
|
||||
*******************************************************************************/
|
||||
public void setOnDownloadedPending(Boolean onDownloadedPending)
|
||||
{
|
||||
this.onDownloadedPending = onDownloadedPending;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for onDownloadedPending.
|
||||
*******************************************************************************/
|
||||
public FileGatewayOutboxFile withOnDownloadedPending(Boolean onDownloadedPending)
|
||||
{
|
||||
this.onDownloadedPending = onDownloadedPending;
|
||||
return (this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.model;
|
||||
|
||||
import java.time.Instant;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QField;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
|
||||
|
||||
/*******************************************************************************
|
||||
** Persisted gateway heartbeat, visible across deployments.
|
||||
*******************************************************************************/
|
||||
public class FileGatewayServer extends QRecordEntity
|
||||
{
|
||||
public static final String TABLE_NAME = "file_gateway_server";
|
||||
|
||||
@QField
|
||||
private Integer id;
|
||||
|
||||
@QField
|
||||
private Instant createDate;
|
||||
|
||||
@QField
|
||||
private Instant modifyDate;
|
||||
|
||||
@QField
|
||||
private String podName;
|
||||
|
||||
@QField
|
||||
private String hostKeyFingerprint;
|
||||
|
||||
@QField
|
||||
private Integer port;
|
||||
|
||||
@QField
|
||||
private Instant startedAt;
|
||||
|
||||
@QField
|
||||
private Instant lastSeenAt;
|
||||
|
||||
/*******************************************************************************
|
||||
** Empty record.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Populate from gateway persistence.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer(QRecord record) throws QException
|
||||
{
|
||||
populateFromQRecord(record);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access id.
|
||||
*******************************************************************************/
|
||||
public Integer getId()
|
||||
{
|
||||
return (id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access id.
|
||||
*******************************************************************************/
|
||||
public void setId(Integer id)
|
||||
{
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access id.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer withId(Integer id)
|
||||
{
|
||||
this.id = id;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access createDate.
|
||||
*******************************************************************************/
|
||||
public Instant getCreateDate()
|
||||
{
|
||||
return (createDate);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access createDate.
|
||||
*******************************************************************************/
|
||||
public void setCreateDate(Instant createDate)
|
||||
{
|
||||
this.createDate = createDate;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access createDate.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer withCreateDate(Instant createDate)
|
||||
{
|
||||
this.createDate = createDate;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access modifyDate.
|
||||
*******************************************************************************/
|
||||
public Instant getModifyDate()
|
||||
{
|
||||
return (modifyDate);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access modifyDate.
|
||||
*******************************************************************************/
|
||||
public void setModifyDate(Instant modifyDate)
|
||||
{
|
||||
this.modifyDate = modifyDate;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access modifyDate.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer withModifyDate(Instant modifyDate)
|
||||
{
|
||||
this.modifyDate = modifyDate;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access podName.
|
||||
*******************************************************************************/
|
||||
public String getPodName()
|
||||
{
|
||||
return (podName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access podName.
|
||||
*******************************************************************************/
|
||||
public void setPodName(String podName)
|
||||
{
|
||||
this.podName = podName;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access podName.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer withPodName(String podName)
|
||||
{
|
||||
this.podName = podName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access hostKeyFingerprint.
|
||||
*******************************************************************************/
|
||||
public String getHostKeyFingerprint()
|
||||
{
|
||||
return (hostKeyFingerprint);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access hostKeyFingerprint.
|
||||
*******************************************************************************/
|
||||
public void setHostKeyFingerprint(String hostKeyFingerprint)
|
||||
{
|
||||
this.hostKeyFingerprint = hostKeyFingerprint;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access hostKeyFingerprint.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer withHostKeyFingerprint(String hostKeyFingerprint)
|
||||
{
|
||||
this.hostKeyFingerprint = hostKeyFingerprint;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access port.
|
||||
*******************************************************************************/
|
||||
public Integer getPort()
|
||||
{
|
||||
return (port);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access port.
|
||||
*******************************************************************************/
|
||||
public void setPort(Integer port)
|
||||
{
|
||||
this.port = port;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access port.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer withPort(Integer port)
|
||||
{
|
||||
this.port = port;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access startedAt.
|
||||
*******************************************************************************/
|
||||
public Instant getStartedAt()
|
||||
{
|
||||
return (startedAt);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access startedAt.
|
||||
*******************************************************************************/
|
||||
public void setStartedAt(Instant startedAt)
|
||||
{
|
||||
this.startedAt = startedAt;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access startedAt.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer withStartedAt(Instant startedAt)
|
||||
{
|
||||
this.startedAt = startedAt;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access lastSeenAt.
|
||||
*******************************************************************************/
|
||||
public Instant getLastSeenAt()
|
||||
{
|
||||
return (lastSeenAt);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access lastSeenAt.
|
||||
*******************************************************************************/
|
||||
public void setLastSeenAt(Instant lastSeenAt)
|
||||
{
|
||||
this.lastSeenAt = lastSeenAt;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Access lastSeenAt.
|
||||
*******************************************************************************/
|
||||
public FileGatewayServer withLastSeenAt(Instant lastSeenAt)
|
||||
{
|
||||
this.lastSeenAt = lastSeenAt;
|
||||
return (this);
|
||||
}
|
||||
}
|
||||
@@ -136,6 +136,16 @@ public class FileGatewayUploadJob extends QRecordEntity
|
||||
|
||||
|
||||
|
||||
@QField
|
||||
private String plaintextStorageKey;
|
||||
|
||||
@QField
|
||||
private String plaintextSha256;
|
||||
|
||||
@QField
|
||||
private Long plaintextSizeBytes;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create an empty entity.
|
||||
*******************************************************************************/
|
||||
@@ -1170,4 +1180,97 @@ public class FileGatewayUploadJob extends QRecordEntity
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for plaintextStorageKey.
|
||||
*******************************************************************************/
|
||||
public String getPlaintextStorageKey()
|
||||
{
|
||||
return (plaintextStorageKey);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter used by QRecordEntity population.
|
||||
*******************************************************************************/
|
||||
public void setPlaintextStorageKey(String plaintextStorageKey)
|
||||
{
|
||||
this.plaintextStorageKey = plaintextStorageKey;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for plaintextStorageKey.
|
||||
*******************************************************************************/
|
||||
public FileGatewayUploadJob withPlaintextStorageKey(String plaintextStorageKey)
|
||||
{
|
||||
this.plaintextStorageKey = plaintextStorageKey;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for plaintextSha256.
|
||||
*******************************************************************************/
|
||||
public String getPlaintextSha256()
|
||||
{
|
||||
return (plaintextSha256);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter used by QRecordEntity population.
|
||||
*******************************************************************************/
|
||||
public void setPlaintextSha256(String plaintextSha256)
|
||||
{
|
||||
this.plaintextSha256 = plaintextSha256;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for plaintextSha256.
|
||||
*******************************************************************************/
|
||||
public FileGatewayUploadJob withPlaintextSha256(String plaintextSha256)
|
||||
{
|
||||
this.plaintextSha256 = plaintextSha256;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for plaintextSizeBytes.
|
||||
*******************************************************************************/
|
||||
public Long getPlaintextSizeBytes()
|
||||
{
|
||||
return (plaintextSizeBytes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Setter used by QRecordEntity population.
|
||||
*******************************************************************************/
|
||||
public void setPlaintextSizeBytes(Long plaintextSizeBytes)
|
||||
{
|
||||
this.plaintextSizeBytes = plaintextSizeBytes;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for plaintextSizeBytes.
|
||||
*******************************************************************************/
|
||||
public FileGatewayUploadJob withPlaintextSizeBytes(Long plaintextSizeBytes)
|
||||
{
|
||||
this.plaintextSizeBytes = plaintextSizeBytes;
|
||||
return (this);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.outbox;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.Clock;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
|
||||
import com.kingsrook.qbits.filegateway.api.FileGatewayConsumer;
|
||||
import com.kingsrook.qbits.filegateway.api.FileGatewayConsumersMetaData;
|
||||
import com.kingsrook.qbits.filegateway.api.FileGatewayOutboxProvider;
|
||||
import com.kingsrook.qbits.filegateway.api.OutboxFileRef;
|
||||
import com.kingsrook.qbits.filegateway.metadata.AliasCustomizer;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.sftp.AuditWriter;
|
||||
import com.kingsrook.qbits.filegateway.storage.GatewayObjectStore;
|
||||
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
|
||||
/*******************************************************************************
|
||||
** Database and object-storage outbox usable in web pods without an SFTP listener.
|
||||
*******************************************************************************/
|
||||
public class OutboxService implements FileGatewayOutboxProvider
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(OutboxService.class);
|
||||
private final QInstance instance;
|
||||
private final AuditWriter audit;
|
||||
private final GatewayObjectStore store;
|
||||
private final FileGatewayQBitConfig config;
|
||||
private final Clock clock;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No runtime, threads or protocol credentials are required.
|
||||
*******************************************************************************/
|
||||
public OutboxService(QInstance instance, GatewayObjectStore store, FileGatewayQBitConfig config, Clock clock)
|
||||
{
|
||||
this.instance = instance;
|
||||
this.audit = new AuditWriter(instance);
|
||||
this.store = store;
|
||||
this.config = config;
|
||||
this.clock = clock;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publish only after the object is complete and its caller-supplied size matches.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public OutboxFileRef put(String endpointName, String fileName, InputStream content, long sizeBytes) throws QException
|
||||
{
|
||||
if(!AliasCustomizer.validFileName(fileName) || sizeBytes < 0)
|
||||
{
|
||||
throw (new QException("Invalid outbox filename or size"));
|
||||
}
|
||||
QRecord alias = endpoint(endpointName);
|
||||
String key = "file-gateway/outbox/" + alias.getValue("id") + "/" + UUID.randomUUID() + "/" + fileName;
|
||||
try
|
||||
{
|
||||
GatewayObjectStore.Receipt receipt = store.put(key, content);
|
||||
if(receipt.size() != sizeBytes)
|
||||
{
|
||||
throw (new QException("Outbox stream size does not match sizeBytes"));
|
||||
}
|
||||
return (audit.withRowLock(FileGatewayTables.ALIAS, "id", alias.getValue("id"), () ->
|
||||
{
|
||||
QRecord current = endpoint(endpointName);
|
||||
for(QRecord existing : list(current.getValueInteger("id")))
|
||||
{
|
||||
if(fileName.equals(existing.getValueString("fileName")))
|
||||
{
|
||||
throw (new QException("An active outbox file already has this filename"));
|
||||
}
|
||||
}
|
||||
QRecord row = audit.insert(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("aliasId", current.getValue("id"))
|
||||
.withValue("fileName", fileName).withValue("storageKey", key).withValue("sizeBytes", receipt.size()).withValue("sha256", receipt.sha256())
|
||||
.withValue("status", "READY").withValue("downloadCount", 0).withValue("onDownloadedPending", false)
|
||||
.withValue("createdByConsumer", current.getValue("consumerName")).withValue("createDate", clock.instant())
|
||||
.withValue("expiresAt", clock.instant().plus(config.getOutboxRetention())));
|
||||
return (reference(row, current));
|
||||
}));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
try
|
||||
{
|
||||
store.delete(key);
|
||||
}
|
||||
catch(IOException cleanup)
|
||||
{
|
||||
e.addSuppressed(cleanup);
|
||||
}
|
||||
throw (new QException("Unable to publish outbox file: " + (e instanceof QException ? e.getMessage() : "storage failed"), e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A missing, inactive or incorrectly typed endpoint cannot accept output.
|
||||
*******************************************************************************/
|
||||
private QRecord endpoint(String name) throws QException
|
||||
{
|
||||
var rows = audit.query(FileGatewayTables.ALIAS, UploadJobs.equal("name", name));
|
||||
if(rows.isEmpty() || !Boolean.TRUE.equals(rows.getFirst().getValueBoolean("isActive")) || !"OUTBOX".equals(rows.getFirst().getValueString("kind")))
|
||||
{
|
||||
throw (new QException("Outbox endpoint is missing, inactive or not OUTBOX"));
|
||||
}
|
||||
return (rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Removed and expired files disappear immediately from the virtual folder.
|
||||
*******************************************************************************/
|
||||
public List<QRecord> list(Integer aliasId) throws QException
|
||||
{
|
||||
return (audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("aliasId", aliasId)).stream()
|
||||
.filter(row -> List.of("READY", "DOWNLOADED").contains(row.getValueString("status"))).toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only a complete read schedules the durable, at-least-once notification.
|
||||
*******************************************************************************/
|
||||
public void downloaded(QRecord row) throws QException
|
||||
{
|
||||
audit.withRowLock(FileGatewayTables.OUTBOX_FILE, "id", row.getValue("id"), () ->
|
||||
{
|
||||
QRecord current = audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", row.getValue("id"))).getFirst();
|
||||
if(List.of("READY", "DOWNLOADED").contains(current.getValueString("status")))
|
||||
{
|
||||
QRecord changes = new QRecord().withValue("id", current.getValue("id")).withValue("status", "DOWNLOADED")
|
||||
.withValue("downloadCount", current.getValueInteger("downloadCount") + 1);
|
||||
if(current.getValue("firstDownloadedAt") == null)
|
||||
{
|
||||
changes.withValue("firstDownloadedAt", clock.instant()).withValue("onDownloadedPending", true);
|
||||
}
|
||||
audit.update(FileGatewayTables.OUTBOX_FILE, changes);
|
||||
}
|
||||
return (null);
|
||||
});
|
||||
notifyDownloaded(row);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Holding the row lock prevents duplicate concurrent delivery; crash retries remain possible.
|
||||
*******************************************************************************/
|
||||
public void notifyDownloaded(QRecord row) throws QException
|
||||
{
|
||||
audit.withRowLock(FileGatewayTables.OUTBOX_FILE, "id", row.getValue("id"), () ->
|
||||
{
|
||||
QRecord current = audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", row.getValue("id"))).getFirst();
|
||||
if(!Boolean.TRUE.equals(current.getValueBoolean("onDownloadedPending")))
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
try
|
||||
{
|
||||
QRecord alias = audit.query(FileGatewayTables.ALIAS, UploadJobs.equal("id", current.getValue("aliasId"))).getFirst();
|
||||
String name = alias.getValueString("consumerName");
|
||||
if(name != null && !name.isBlank())
|
||||
{
|
||||
FileGatewayConsumer selected = null;
|
||||
for(var code : FileGatewayConsumersMetaData.ofOrWithNew(instance).getConsumers())
|
||||
{
|
||||
FileGatewayConsumer consumer = QCodeLoader.getAdHoc(FileGatewayConsumer.class, code);
|
||||
if(name.equals(consumer.getName()))
|
||||
{
|
||||
selected = consumer;
|
||||
}
|
||||
}
|
||||
if(selected == null)
|
||||
{
|
||||
throw (new QException("Outbox consumer is not registered"));
|
||||
}
|
||||
selected.onDownloaded(reference(current, alias));
|
||||
}
|
||||
audit.update(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("id", row.getValue("id")).withValue("onDownloadedPending", false));
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LOG.warn("Outbox download notification failed; maintenance will retry");
|
||||
}
|
||||
return (null);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Acknowledgment retains audit state until object retention runs.
|
||||
*******************************************************************************/
|
||||
public void remove(QRecord row) throws QException
|
||||
{
|
||||
audit.withRowLock(FileGatewayTables.OUTBOX_FILE, "id", row.getValue("id"), () ->
|
||||
{
|
||||
audit.update(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("id", row.getValue("id"))
|
||||
.withValue("status", "REMOVED").withValue("removedAt", clock.instant()));
|
||||
return (null);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Consumer contract contains no implementation types or storage credentials.
|
||||
*******************************************************************************/
|
||||
private OutboxFileRef reference(QRecord row, QRecord alias)
|
||||
{
|
||||
return (new OutboxFileRef(row.getValueInteger("id"), alias.getValueString("name"), alias.getValueString("path") + "/" + row.getValueString("fileName"),
|
||||
row.getValueString("fileName"), row.getValueLong("sizeBytes"), row.getValueString("sha256"), row.getValueInstant("createDate")));
|
||||
}
|
||||
}
|
||||
+43
@@ -22,14 +22,19 @@ package com.kingsrook.qbits.filegateway.permissions;
|
||||
|
||||
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.AvailablePermission;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
@@ -60,4 +65,42 @@ public class FileGatewayPermissionChecker implements CustomPermissionChecker
|
||||
throw (new QPermissionDeniedException("Missing file gateway permission"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** This checker gates on the three gateway permission names, not on per-object
|
||||
** names, so it tells the catalog (getAllAvailablePermissions) those names.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean handlesBuildAvailablePermission()
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Collapse every gateway object onto file_gateway.view, .admin or .connect.
|
||||
** The object name and type are fixed so the catalog (a set) lists each name
|
||||
** once: reads and the app need view, writes and processes need admin, and the
|
||||
** app also stands for SFTP sign-in (connect), which the gateway grants to
|
||||
** sessions rather than checking on an object.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public AvailablePermission buildAvailablePermission(QPermissionRules rules, PermissionSubType permissionSubType, String baseName, MetaDataWithName metaDataWithName, String objectType)
|
||||
{
|
||||
AvailablePermission base = buildBaseAvailablePermission(rules, permissionSubType, metaDataWithName, objectType);
|
||||
if(base == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
boolean write = "WRITE".equals(base.getPermissionType()) || "Process".equals(objectType);
|
||||
String name = write ? FileGatewayPermissions.ADMIN : FileGatewayPermissions.VIEW;
|
||||
if(metaDataWithName instanceof QAppMetaData)
|
||||
{
|
||||
name = FileGatewayPermissions.CONNECT;
|
||||
}
|
||||
return (base.withName(name).withObjectName("File Gateway").withObjectType("App").withPermissionType(name.substring(name.indexOf('.') + 1)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.pgp;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.FilterInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.io.OutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.Date;
|
||||
import org.bouncycastle.bcpg.SymmetricKeyAlgorithmTags;
|
||||
import org.bouncycastle.openpgp.PGPCompressedData;
|
||||
import org.bouncycastle.openpgp.PGPEncryptedDataGenerator;
|
||||
import org.bouncycastle.openpgp.PGPEncryptedDataList;
|
||||
import org.bouncycastle.openpgp.PGPException;
|
||||
import org.bouncycastle.openpgp.PGPLiteralData;
|
||||
import org.bouncycastle.openpgp.PGPLiteralDataGenerator;
|
||||
import org.bouncycastle.openpgp.PGPObjectFactory;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
import org.bouncycastle.openpgp.PGPPublicKeyEncryptedData;
|
||||
import org.bouncycastle.openpgp.PGPPublicKeyRingCollection;
|
||||
import org.bouncycastle.openpgp.PGPSecretKeyRingCollection;
|
||||
import org.bouncycastle.openpgp.PGPUtil;
|
||||
import org.bouncycastle.openpgp.operator.bc.BcKeyFingerprintCalculator;
|
||||
import org.bouncycastle.openpgp.operator.bc.BcPBESecretKeyDecryptorBuilder;
|
||||
import org.bouncycastle.openpgp.operator.bc.BcPGPDataEncryptorBuilder;
|
||||
import org.bouncycastle.openpgp.operator.bc.BcPGPDigestCalculatorProvider;
|
||||
import org.bouncycastle.openpgp.operator.bc.BcPublicKeyDataDecryptorFactory;
|
||||
import org.bouncycastle.openpgp.operator.bc.BcPublicKeyKeyEncryptionMethodGenerator;
|
||||
|
||||
/*******************************************************************************
|
||||
** Bounded, in-memory OpenPGP streams; no key files, providers or background threads.
|
||||
*******************************************************************************/
|
||||
public final class PgpStreams
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Static stream factories only.
|
||||
*******************************************************************************/
|
||||
private PgpStreams()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Require a usable encryption subkey, rather than accepting signing-only keys.
|
||||
*******************************************************************************/
|
||||
public static PGPPublicKey encryptionKey(String text) throws IOException, PGPException
|
||||
{
|
||||
if(text == null || text.isBlank())
|
||||
{
|
||||
throw (new PGPException("PGP public key is required"));
|
||||
}
|
||||
var rings = new PGPPublicKeyRingCollection(PGPUtil.getDecoderStream(new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8))), new BcKeyFingerprintCalculator());
|
||||
for(var ring : rings)
|
||||
{
|
||||
for(var key : ring)
|
||||
{
|
||||
if(!key.isMasterKey() && key.isEncryptionKey() && !key.hasRevocation()
|
||||
&& (key.getValidSeconds() == 0 || key.getCreationTime().toInstant().plusSeconds(key.getValidSeconds()).isAfter(java.time.Instant.now())))
|
||||
{
|
||||
return (key);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw (new PGPException("PGP public key must contain an unrevoked, unexpired encryption subkey"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** AES-256 binary packets with integrity protection and bounded partial packets.
|
||||
*******************************************************************************/
|
||||
public static InputStream encrypt(InputStream source, String keyText, String filename) throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
ByteArrayOutputStream pending = new ByteArrayOutputStream(131072);
|
||||
var generator = new PGPEncryptedDataGenerator(new BcPGPDataEncryptorBuilder(SymmetricKeyAlgorithmTags.AES_256)
|
||||
.setWithIntegrityPacket(true).setSecureRandom(new SecureRandom()));
|
||||
generator.addMethod(new BcPublicKeyKeyEncryptionMethodGenerator(encryptionKey(keyText)));
|
||||
OutputStream encrypted = generator.open(pending, new byte[32768]);
|
||||
var literalGenerator = new PGPLiteralDataGenerator();
|
||||
OutputStream literal = literalGenerator.open(encrypted, PGPLiteralData.BINARY, filename, new Date(0), new byte[32768]);
|
||||
return (new InputStream()
|
||||
{
|
||||
private InputStream available = InputStream.nullInputStream();
|
||||
private boolean finished;
|
||||
private boolean closed;
|
||||
|
||||
@Override
|
||||
public int read() throws IOException
|
||||
{
|
||||
byte[] one = new byte[1];
|
||||
return (read(one, 0, 1) < 0 ? -1 : one[0] & 255);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] bytes, int offset, int length) throws IOException
|
||||
{
|
||||
java.util.Objects.checkFromIndexSize(offset, length, bytes.length);
|
||||
if(closed)
|
||||
{
|
||||
throw (new IOException("PGP stream closed"));
|
||||
}
|
||||
if(length == 0)
|
||||
{
|
||||
return (0);
|
||||
}
|
||||
while(true)
|
||||
{
|
||||
int count = available.read(bytes, offset, length);
|
||||
if(count >= 0)
|
||||
{
|
||||
return (count);
|
||||
}
|
||||
if(pending.size() > 0)
|
||||
{
|
||||
available = new ByteArrayInputStream(pending.toByteArray());
|
||||
pending.reset();
|
||||
}
|
||||
else if(finished)
|
||||
{
|
||||
return (-1);
|
||||
}
|
||||
else
|
||||
{
|
||||
byte[] chunk = source.readNBytes(32768);
|
||||
if(chunk.length == 0)
|
||||
{
|
||||
literal.close();
|
||||
encrypted.close();
|
||||
finished = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
literal.write(chunk);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException
|
||||
{
|
||||
closed = true;
|
||||
source.close();
|
||||
}
|
||||
});
|
||||
}
|
||||
catch(PGPException e)
|
||||
{
|
||||
throw (new IOException("Unable to encrypt PGP download", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify integrity at EOF before object storage can complete its plaintext upload.
|
||||
*******************************************************************************/
|
||||
public static InputStream decrypt(InputStream source, String secret, char[] passphrase, long maximum) throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
if(secret == null)
|
||||
{
|
||||
throw (new PGPException("PGP private key environment variable is missing"));
|
||||
}
|
||||
var keys = new PGPSecretKeyRingCollection(PGPUtil.getDecoderStream(new ByteArrayInputStream(secret.getBytes(StandardCharsets.UTF_8))), new BcKeyFingerprintCalculator());
|
||||
var factory = new PGPObjectFactory(PGPUtil.getDecoderStream(source), new BcKeyFingerprintCalculator());
|
||||
Object packet = factory.nextObject();
|
||||
if(!(packet instanceof PGPEncryptedDataList))
|
||||
{
|
||||
packet = factory.nextObject();
|
||||
}
|
||||
if(!(packet instanceof PGPEncryptedDataList encrypted))
|
||||
{
|
||||
throw (new PGPException("Expected PGP encrypted data"));
|
||||
}
|
||||
for(var item : encrypted)
|
||||
{
|
||||
if(!(item instanceof PGPPublicKeyEncryptedData data))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
var key = keys.getSecretKey(data.getKeyID());
|
||||
if(key == null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
var privateKey = key.extractPrivateKey(new BcPBESecretKeyDecryptorBuilder(new BcPGPDigestCalculatorProvider()).build(passphrase));
|
||||
InputStream clear = data.getDataStream(new BcPublicKeyDataDecryptorFactory(privateKey));
|
||||
Object message = new PGPObjectFactory(clear, new BcKeyFingerprintCalculator()).nextObject();
|
||||
for(int depth = 0; message instanceof PGPCompressedData compressed && depth < 4; depth++)
|
||||
{
|
||||
message = new PGPObjectFactory(compressed.getDataStream(), new BcKeyFingerprintCalculator()).nextObject();
|
||||
}
|
||||
if(!(message instanceof PGPLiteralData literal) || !data.isIntegrityProtected())
|
||||
{
|
||||
throw (new PGPException("Expected integrity-protected literal PGP content"));
|
||||
}
|
||||
return (new FilterInputStream(literal.getInputStream())
|
||||
{
|
||||
private long consumed;
|
||||
private boolean verified;
|
||||
|
||||
@Override
|
||||
public int read() throws IOException
|
||||
{
|
||||
byte[] one = new byte[1];
|
||||
return (read(one, 0, 1) < 0 ? -1 : one[0] & 255);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] bytes, int offset, int length) throws IOException
|
||||
{
|
||||
int count = in.read(bytes, offset, length);
|
||||
if(count > 0 && (consumed += count) > maximum)
|
||||
{
|
||||
throw (new IOException("Decrypted upload exceeds file size limit"));
|
||||
}
|
||||
if(count < 0 && !verified)
|
||||
{
|
||||
try
|
||||
{
|
||||
clear.transferTo(OutputStream.nullOutputStream());
|
||||
if(!data.verify())
|
||||
{
|
||||
throw (new IOException("PGP integrity check failed"));
|
||||
}
|
||||
verified = true;
|
||||
}
|
||||
catch(PGPException e)
|
||||
{
|
||||
throw (new IOException("PGP integrity check failed", e));
|
||||
}
|
||||
}
|
||||
return (count);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
super.close();
|
||||
}
|
||||
finally
|
||||
{
|
||||
source.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
throw (new PGPException("No matching PGP private key"));
|
||||
}
|
||||
catch(PGPException | RuntimeException e)
|
||||
{
|
||||
throw (new IOException("Unable to decrypt PGP upload", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
+411
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.preferences;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionScope;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceChange;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceListMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRequestContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRow;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRowChange;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceSectionMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceValues;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferencesMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Self-service SSH keys. Core supplies the trusted scope and retains the user
|
||||
** and client channel; this provider never switches to a system session.
|
||||
*******************************************************************************/
|
||||
public class FileTransferPreferencesProvider implements UserPreferenceProviderInterface
|
||||
{
|
||||
public static final String SECTION_NAME = "file-transfer";
|
||||
private static final List<String> KEY_FIELDS = List.of("label", "fingerprint", "keyType", "lastUsedAt", "expiresAt");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register without an app scope: connect permission is sufficient.
|
||||
*******************************************************************************/
|
||||
public static void register(QInstance instance, FileGatewayQBitConfig config)
|
||||
{
|
||||
UserPreferenceSectionMetaData section = new UserPreferenceSectionMetaData()
|
||||
.withName(SECTION_NAME).withLabel("File transfer").withIcon(new QIcon("swap_vert")).withOrder(60)
|
||||
.withDescription("Sign in to the SFTP file gateway with your SSH keys.")
|
||||
.withPermissionName(FileGatewayPermissions.CONNECT).withProvider(new QCodeReference(FileTransferPreferencesProvider.class))
|
||||
.withField(new QFieldMetaData("username", QFieldType.STRING).withLabel("Username").withIsEditable(false)
|
||||
.withHelpContent(new QHelpContent("Adding your first key creates your SFTP account and username.")))
|
||||
.withField(new QFieldMetaData("port", QFieldType.INTEGER).withLabel("Port").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("hostKeyFingerprint", QFieldType.STRING).withLabel("Host key fingerprint").withIsEditable(false))
|
||||
.withList(new UserPreferenceListMetaData().withName("keys").withLabel("SSH keys").withRowLabel("Key")
|
||||
.withDescription("Adding your first key creates your SFTP account.")
|
||||
.withField(new QFieldMetaData("label", QFieldType.STRING).withLabel("Label").withIsRequired(true).withMaxLength(1024))
|
||||
.withField(new QFieldMetaData("fingerprint", QFieldType.STRING).withLabel("Fingerprint").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("keyType", QFieldType.STRING).withLabel("Key type").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("lastUsedAt", QFieldType.DATE_TIME).withLabel("Last used").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("expiresAt", QFieldType.DATE_TIME).withLabel("Expires at")))
|
||||
.withAction(new UserPreferenceActionMetaData().withName("addKey").withLabel("Add key")
|
||||
.withScope(UserPreferenceActionScope.LIST).withListName("keys").withOrder(10)
|
||||
.withInputField(new QFieldMetaData("publicKey", QFieldType.TEXT).withLabel("Public key").withIsRequired(true))
|
||||
.withInputField(new QFieldMetaData("label", QFieldType.STRING).withLabel("Label").withIsRequired(true).withMaxLength(1024))
|
||||
.withInputField(new QFieldMetaData("expiresAt", QFieldType.DATE_TIME).withLabel("Expires at")))
|
||||
.withAction(new UserPreferenceActionMetaData().withName("revokeKey").withLabel("Revoke")
|
||||
.withScope(UserPreferenceActionScope.ROW).withListName("keys").withOrder(20).withIsDestructive(true)
|
||||
.withConfirmationText("Revoke this SSH key? It will no longer be able to sign in."));
|
||||
if(config.getPublicHostName() != null && !config.getPublicHostName().isBlank())
|
||||
{
|
||||
section.withField(new QFieldMetaData("host", QFieldType.STRING).withLabel("Host").withIsEditable(false).withDefaultValue(config.getPublicHostName()));
|
||||
}
|
||||
UserPreferencesMetaData.ofOrWithNew(instance).withSection(section);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publish only the user's active key metadata, never the public-key text.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public UserPreferenceValues load(UserPreferenceRequestContext context) throws QException
|
||||
{
|
||||
QRecord account = account(context, null);
|
||||
UserPreferenceValues result = new UserPreferenceValues();
|
||||
result.getValues().put("username", account == null ? "" : account.getValue("username"));
|
||||
List<QRecord> servers = query(FileGatewayTables.SERVER, new QQueryFilter()
|
||||
.withOrderBy(new QFilterOrderBy("lastSeenAt", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(1), null);
|
||||
if(!servers.isEmpty())
|
||||
{
|
||||
result.getValues().put("port", servers.getFirst().getValue("port"));
|
||||
result.getValues().put("hostKeyFingerprint", servers.getFirst().getValue("hostKeyFingerprint"));
|
||||
}
|
||||
List<UserPreferenceRow> rows = new ArrayList<>();
|
||||
if(account != null)
|
||||
{
|
||||
for(QRecord key : query(FileGatewayTables.SSH_KEY, keysFilter(account), null))
|
||||
{
|
||||
Map<String, Serializable> values = new LinkedHashMap<>();
|
||||
KEY_FIELDS.forEach(field -> values.put(field, key.getValue(field)));
|
||||
rows.add(new UserPreferenceRow().withKey(key.getValueString("id")).withLabel(key.getValueString("label")).withValues(values));
|
||||
}
|
||||
}
|
||||
result.getLists().put("keys", rows);
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Core validates types; reject blank labels as well as missing ones.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<String> validate(UserPreferenceRequestContext context, UserPreferenceChange change)
|
||||
{
|
||||
List<String> errors = new ArrayList<>();
|
||||
for(UserPreferenceRowChange row : change.getLists().getOrDefault("keys", List.of()))
|
||||
{
|
||||
if(row.getValues().containsKey("label") && (row.getValues().get("label") == null || row.getValues().get("label").toString().isBlank()))
|
||||
{
|
||||
errors.add("Label: is required");
|
||||
}
|
||||
}
|
||||
return (errors);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply partial changes atomically, checking ownership again under row locks.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void store(UserPreferenceRequestContext context, UserPreferenceChange change) throws QException
|
||||
{
|
||||
try(var tx = QBackendTransaction.openFor(new QueryInput(FileGatewayTables.ACCOUNT)))
|
||||
{
|
||||
try
|
||||
{
|
||||
QRecord account = lockedAccount(context, tx);
|
||||
for(UserPreferenceRowChange row : change.getLists().getOrDefault("keys", List.of()))
|
||||
{
|
||||
QRecord key = lockedKey(account, row.getKey(), tx);
|
||||
QRecord update = new QRecord().withValue("id", key.getValue("id"));
|
||||
for(String field : List.of("label", "expiresAt"))
|
||||
{
|
||||
if(row.getValues().containsKey(field))
|
||||
{
|
||||
update.setValue(field, row.getValues().get(field));
|
||||
}
|
||||
}
|
||||
update(FileGatewayTables.SSH_KEY, update, tx);
|
||||
}
|
||||
tx.commit();
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
tx.rollback();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add through the normal key customizer, or revoke an owned, unrevoked row.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public UserPreferenceActionResult runAction(UserPreferenceRequestContext context, UserPreferenceActionRequest request) throws QException
|
||||
{
|
||||
if(!List.of("addKey", "revokeKey").contains(request.getActionName()))
|
||||
{
|
||||
throw (new QBadRequestException("This action isn't available."));
|
||||
}
|
||||
if("addKey".equals(request.getActionName()))
|
||||
{
|
||||
List<String> errors = new ArrayList<>();
|
||||
for(String field : List.of("publicKey", "label"))
|
||||
{
|
||||
if(request.getValues().get(field) == null || request.getValues().get(field).toString().isBlank())
|
||||
{
|
||||
errors.add(("publicKey".equals(field) ? "Public key" : "Label") + ": is required");
|
||||
}
|
||||
}
|
||||
if(!errors.isEmpty())
|
||||
{
|
||||
throw (new QBadRequestException(String.join("\n", errors)));
|
||||
}
|
||||
}
|
||||
try(var tx = QBackendTransaction.openFor(new QueryInput(FileGatewayTables.ACCOUNT)))
|
||||
{
|
||||
try
|
||||
{
|
||||
QRecord account = lockedAccount(context, tx);
|
||||
UserPreferenceActionResult result;
|
||||
if("addKey".equals(request.getActionName()))
|
||||
{
|
||||
if(account == null)
|
||||
{
|
||||
account = insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", availableUsername(context.getUserId(), tx))
|
||||
.withValue("userId", context.getUserId()).withValue("accountType", "USER").withValue("rootMode", "FULL_TREE"), tx);
|
||||
}
|
||||
QRecord key = insert(FileGatewayTables.SSH_KEY, new QRecord().withValue("accountId", account.getValue("id"))
|
||||
.withValue("publicKey", request.getValues().get("publicKey")).withValue("label", request.getValues().get("label"))
|
||||
.withValue("expiresAt", request.getValues().get("expiresAt")), tx);
|
||||
result = new UserPreferenceActionResult().withMessage("Added key " + key.getValueString("fingerprint") + ".").withFocusRowKey(key.getValueString("id"));
|
||||
}
|
||||
else
|
||||
{
|
||||
QRecord key = lockedKey(account, request.getRowKey(), tx);
|
||||
update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", key.getValue("id")).withValue("revokedAt", Instant.now()), tx);
|
||||
result = new UserPreferenceActionResult().withMessage("Key revoked.");
|
||||
}
|
||||
tx.commit();
|
||||
return (result);
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
tx.rollback();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use only the session identity; service accounts stay in the admin tables.
|
||||
*******************************************************************************/
|
||||
private QRecord account(UserPreferenceRequestContext context, QBackendTransaction tx) throws QException
|
||||
{
|
||||
List<QRecord> rows = query(FileGatewayTables.ACCOUNT, new QQueryFilter()
|
||||
.withCriteria("userId", QCriteriaOperator.EQUALS, context.getUserId())
|
||||
.withCriteria("accountType", QCriteriaOperator.EQUALS, "USER").withOrderBy(new QFilterOrderBy("id", true)).withLimit(1), tx);
|
||||
return (rows.isEmpty() ? null : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A transactional timestamp update acquires the same lock used by gateway auth.
|
||||
** Re-read after the lock in case an administrator changed ownership meanwhile.
|
||||
*******************************************************************************/
|
||||
private QRecord lockedAccount(UserPreferenceRequestContext context, QBackendTransaction tx) throws QException
|
||||
{
|
||||
QRecord account = account(context, tx);
|
||||
if(account != null)
|
||||
{
|
||||
update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("modifyDate", Instant.now()), tx);
|
||||
QRecord current = account(context, tx);
|
||||
if(current == null || !Objects.equals(current.getValue("id"), account.getValue("id")))
|
||||
{
|
||||
throw (new QBadRequestException("Account: isn't a setting here"));
|
||||
}
|
||||
return (current);
|
||||
}
|
||||
return (null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scope every key lookup to the caller's account and unrevoked rows.
|
||||
*******************************************************************************/
|
||||
private QQueryFilter keysFilter(QRecord account)
|
||||
{
|
||||
return (new QQueryFilter().withCriteria("accountId", QCriteriaOperator.EQUALS, account.getValue("id"))
|
||||
.withCriteria("revokedAt", QCriteriaOperator.IS_BLANK).withOrderBy(new QFilterOrderBy("id", true)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check ownership both before and after locking the key against reassignment.
|
||||
*******************************************************************************/
|
||||
private QRecord lockedKey(QRecord account, String rowKey, QBackendTransaction tx) throws QException
|
||||
{
|
||||
QRecord key = ownKey(account, rowKey, tx);
|
||||
update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", key.getValue("id")).withValue("modifyDate", Instant.now()), tx);
|
||||
return (ownKey(account, rowKey, tx));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Do not interpret an arbitrary caller-supplied key as an unrestricted ID.
|
||||
*******************************************************************************/
|
||||
private QRecord ownKey(QRecord account, String rowKey, QBackendTransaction tx) throws QException
|
||||
{
|
||||
if(account != null)
|
||||
{
|
||||
for(QRecord key : query(FileGatewayTables.SSH_KEY, keysFilter(account), tx))
|
||||
{
|
||||
if(key.getValueString("id").equals(rowKey))
|
||||
{
|
||||
return (key);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw (new QBadRequestException("Key: isn't a setting here"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Normalize opaque user IDs to the account validator's 3-64 character syntax.
|
||||
** Truncate before suffixing, so even a 64-character collision stays valid.
|
||||
*******************************************************************************/
|
||||
private String availableUsername(String userId, QBackendTransaction tx) throws QException
|
||||
{
|
||||
String base = userId.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]", "-");
|
||||
if(base.length() < 3)
|
||||
{
|
||||
base = "user-" + base;
|
||||
}
|
||||
base = base.substring(0, Math.min(64, base.length()));
|
||||
String username = base;
|
||||
int suffix = 2;
|
||||
while(!query(FileGatewayTables.ACCOUNT, new QQueryFilter().withCriteria("username", QCriteriaOperator.EQUALS, username).withLimit(1), tx).isEmpty())
|
||||
{
|
||||
String ending = "-" + suffix++;
|
||||
username = base.substring(0, Math.min(base.length(), 64 - ending.length())) + ending;
|
||||
}
|
||||
return (username);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard actions retain customizers, configured audits and the caller's session.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> query(String table, QQueryFilter filter, QBackendTransaction tx) throws QException
|
||||
{
|
||||
return (new QueryAction().execute(new QueryInput(table).withFilter(filter).withTransaction(tx)).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve domain validation messages while withholding internal database errors.
|
||||
*******************************************************************************/
|
||||
private QRecord checked(QRecord record) throws QException
|
||||
{
|
||||
if(record.getErrors() != null && !record.getErrors().isEmpty())
|
||||
{
|
||||
List<String> messages = record.getErrors().stream().map(error -> error instanceof BadInputStatusMessage ? error.getMessage() : "Could not save this setting.").toList();
|
||||
throw (new QBadRequestException(String.join("\n", messages)));
|
||||
}
|
||||
return (record);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert via ordinary table validation, including fingerprint uniqueness.
|
||||
*******************************************************************************/
|
||||
private QRecord insert(String table, QRecord record, QBackendTransaction tx) throws QException
|
||||
{
|
||||
return (checked(new InsertAction().execute(new InsertInput(table).withRecord(record).withTransaction(tx)).getRecords().getFirst()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist only supplied fields, with the original user's audit attribution.
|
||||
*******************************************************************************/
|
||||
private void update(String table, QRecord record, QBackendTransaction tx) throws QException
|
||||
{
|
||||
checked(new UpdateAction().execute(new UpdateInput(table).withRecords(List.of(record)).withTransaction(tx)).getRecords().getFirst());
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ package com.kingsrook.qbits.filegateway.sftp;
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer;
|
||||
@@ -115,17 +115,28 @@ public class AuditWriter
|
||||
*******************************************************************************/
|
||||
public <T> T withAccountLock(String field, Serializable value, Work<T> work) throws QException
|
||||
{
|
||||
return (withRowLock(FileGatewayTables.ACCOUNT, field, value, work));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Serialize transitions on one gateway-owned row across gateway and web pods.
|
||||
*******************************************************************************/
|
||||
public <T> T withRowLock(String table, String field, Serializable value, Work<T> work) throws QException
|
||||
{
|
||||
checkTable(table);
|
||||
return (system(() ->
|
||||
{
|
||||
try(var tx = QBackendTransaction.openFor(new QueryInput(FileGatewayTables.ACCOUNT)))
|
||||
try(var tx = QBackendTransaction.openFor(new QueryInput(table)))
|
||||
{
|
||||
transaction.set(tx);
|
||||
try
|
||||
{
|
||||
var rows = query(FileGatewayTables.ACCOUNT, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)));
|
||||
var rows = query(table, new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)));
|
||||
if(!rows.isEmpty())
|
||||
{
|
||||
update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", rows.getFirst().getValue("id")).withValue("modifyDate", Instant.now()));
|
||||
update(table, new QRecord().withValue("id", rows.getFirst().getValue("id")).withValue("modifyDate", Instant.now()));
|
||||
}
|
||||
T result = work.run();
|
||||
tx.commit();
|
||||
@@ -221,18 +232,35 @@ public class AuditWriter
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Optimistic status transition, called under withAccountLock because core checks
|
||||
** expected versions before SQL execution rather than in the SQL WHERE clause.
|
||||
** Retention deletes only gateway-owned rows under the bookkeeping session.
|
||||
*******************************************************************************/
|
||||
public void delete(String table, Serializable id) throws QException
|
||||
{
|
||||
checkTable(table);
|
||||
system(() -> new com.kingsrook.qqq.backend.core.actions.tables.DeleteAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput(table).withPrimaryKey(id).withTransaction(transaction.get())));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Guarded status transition. Callers hold withAccountLock, so the status re-read in
|
||||
** that transaction is serialized across pods; status is compared here rather than
|
||||
** through a core record-version field, because core advances version fields on
|
||||
** every update (a STRING version becomes a random UUID).
|
||||
*******************************************************************************/
|
||||
public boolean transitionUpload(QRecord changes, String expectedStatus) throws QException
|
||||
{
|
||||
QRecord result = system(() -> new UpdateAction().execute(new UpdateInput(FileGatewayTables.UPLOAD_JOB)
|
||||
.withRecords(List.of(changes)).withTransaction(transaction.get())
|
||||
.withExpectedVersions(Map.of(changes.getValue("id"), expectedStatus))).getRecords().getFirst());
|
||||
if(result.getErrors() != null && !result.getErrors().isEmpty())
|
||||
if(transaction.get() == null)
|
||||
{
|
||||
throw (new QException("Upload transitions require the account lock transaction"));
|
||||
}
|
||||
var rows = query(FileGatewayTables.UPLOAD_JOB, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, changes.getValue("id"))));
|
||||
if(rows.isEmpty() || !Objects.equals(expectedStatus, rows.getFirst().getValueString("status")))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
update(FileGatewayTables.UPLOAD_JOB, changes);
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ public class GatewaySession
|
||||
public final ServerSession ssh;
|
||||
public final String remoteIp;
|
||||
public Integer connectionId;
|
||||
public java.time.Instant connectedAt;
|
||||
public QRecord account;
|
||||
public QRecord key;
|
||||
public long operations;
|
||||
|
||||
@@ -37,6 +37,7 @@ import com.kingsrook.qbits.filegateway.storage.ObjectStorageGatewayStore;
|
||||
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
|
||||
import com.kingsrook.qbits.filegateway.vfs.GatewayFileSystem;
|
||||
import com.kingsrook.qbits.filegateway.vfs.GatewayUiApiClient;
|
||||
import com.kingsrook.qbits.filegateway.vfs.GeneratedDownloads;
|
||||
import com.kingsrook.qbits.filegateway.vfs.VirtualTree;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
@@ -69,6 +70,11 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
|
||||
private final SshKeyAuthenticator authenticator;
|
||||
private final Duration pollInterval;
|
||||
private ScheduledExecutorService poller;
|
||||
private final AuditWriter heartbeatAudit;
|
||||
private final Clock heartbeatClock;
|
||||
private final String heartbeatPod;
|
||||
private final String hostFingerprint;
|
||||
private java.time.Instant startedAt;
|
||||
|
||||
|
||||
|
||||
@@ -88,7 +94,13 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
|
||||
public SftpProtocolServer(QInstance instance, FileGatewayQBitConfig config, UiApiInvoker invoker, KeyPair hostKey, Clock clock, Duration pollInterval, String podName, GatewayObjectStore store)
|
||||
{
|
||||
this.pollInterval = pollInterval;
|
||||
heartbeatAudit = new AuditWriter(instance);
|
||||
heartbeatClock = clock;
|
||||
heartbeatPod = podName == null ? java.util.UUID.randomUUID().toString() : podName;
|
||||
hostFingerprint = org.apache.sshd.common.config.keys.KeyUtils.getFingerPrint(org.apache.sshd.common.digest.BuiltinDigests.sha256, hostKey.getPublic());
|
||||
AuditWriter audit = new AuditWriter(instance);
|
||||
var outbox = new com.kingsrook.qbits.filegateway.outbox.OutboxService(instance, store, config, clock);
|
||||
com.kingsrook.qbits.filegateway.api.FileGatewayOutbox.setProvider(outbox);
|
||||
authenticator = new SshKeyAuthenticator(instance, config, audit, clock, podName);
|
||||
server = SshServer.setUpDefaultServer();
|
||||
server.setPort(config.getSftpPort());
|
||||
@@ -108,7 +120,18 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
|
||||
server.setCommandFactory(null);
|
||||
server.setForwardingFilter(RejectAllForwardingFilter.INSTANCE);
|
||||
server.setAgentFactory(null);
|
||||
server.setSubsystemFactories(List.of(new SftpSubsystemFactory.Builder().withFileSystemAccessor(new org.apache.sshd.sftp.server.SftpFileSystemAccessor()
|
||||
server.setSubsystemFactories(List.of(new SftpSubsystemFactory.Builder().withSftpErrorStatusDataHandler(new org.apache.sshd.sftp.server.SftpErrorStatusDataHandler()
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Preserve actionable gateway and UI API errors in SFTP failure responses.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String resolveErrorMessage(org.apache.sshd.sftp.server.SftpSubsystemEnvironment subsystem, int id, Throwable error, int status, int command, Object... args)
|
||||
{
|
||||
return (error instanceof IOException && error.getMessage() != null ? error.getMessage()
|
||||
: org.apache.sshd.sftp.common.SftpHelper.resolveStatusMessage(status));
|
||||
}
|
||||
}).withFileSystemAccessor(new org.apache.sshd.sftp.server.SftpFileSystemAccessor()
|
||||
{
|
||||
/*******************************************************************************
|
||||
** MINA must use our seekable channels, not the native FileChannel default.
|
||||
@@ -140,7 +163,9 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
|
||||
public FileSystem createFileSystem(SessionContext session)
|
||||
{
|
||||
GatewaySession state = session.getAttribute(GatewaySession.ATTRIBUTE);
|
||||
return (new GatewayFileSystem(new VirtualTree(new GatewayUiApiClient(invoker, state), state, audit, clock).withUploads(new UploadJobs(audit, store, config, clock), store)));
|
||||
GatewayUiApiClient client = new GatewayUiApiClient(invoker, state);
|
||||
return (new GatewayFileSystem(new VirtualTree(client, state, audit, clock).withUploads(new UploadJobs(audit, store, config, clock), store)
|
||||
.withDownloads(new GeneratedDownloads(client, state, audit, config, clock, instance)).withOutbox(outbox)));
|
||||
}
|
||||
});
|
||||
server.addSessionListener(new SessionListener()
|
||||
@@ -211,6 +236,16 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
|
||||
public void start() throws IOException
|
||||
{
|
||||
server.start();
|
||||
startedAt = heartbeatClock.instant();
|
||||
try
|
||||
{
|
||||
heartbeat();
|
||||
}
|
||||
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
|
||||
{
|
||||
server.stop(true);
|
||||
throw (new IOException("Unable to publish gateway heartbeat", e));
|
||||
}
|
||||
poller = Executors.newSingleThreadScheduledExecutor(runnable ->
|
||||
{
|
||||
Thread thread = new Thread(runnable, "file-gateway-revocation");
|
||||
@@ -222,6 +257,7 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
|
||||
try
|
||||
{
|
||||
authenticator.poll();
|
||||
heartbeat();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
@@ -233,6 +269,28 @@ public class SftpProtocolServer implements FileGatewayProtocolServer
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publish host identity to the database for app homes in other deployments.
|
||||
*******************************************************************************/
|
||||
public void heartbeat() throws com.kingsrook.qqq.backend.core.exceptions.QException
|
||||
{
|
||||
var rows = heartbeatAudit.query(com.kingsrook.qbits.filegateway.metadata.FileGatewayTables.SERVER,
|
||||
UploadJobs.equal("podName", heartbeatPod));
|
||||
var values = new com.kingsrook.qqq.backend.core.model.data.QRecord().withValue("podName", heartbeatPod)
|
||||
.withValue("hostKeyFingerprint", hostFingerprint).withValue("port", server.getPort())
|
||||
.withValue("startedAt", startedAt).withValue("lastSeenAt", heartbeatClock.instant());
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
heartbeatAudit.insert(com.kingsrook.qbits.filegateway.metadata.FileGatewayTables.SERVER, values);
|
||||
}
|
||||
else
|
||||
{
|
||||
heartbeatAudit.update(com.kingsrook.qbits.filegateway.metadata.FileGatewayTables.SERVER, values.withValue("id", rows.getFirst().getValue("id")));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stop polling, revoke sessions and stop the listener.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -82,16 +82,15 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Audit TCP accept and enforce the pre-authentication connection rate.
|
||||
** Track TCP accept and enforce the pre-authentication connection rate. The
|
||||
** connection-log row is written at the first sign-in attempt (or a refusal),
|
||||
** so TCP health probes and port scans that never authenticate leave no row.
|
||||
*******************************************************************************/
|
||||
public synchronized void accepted(ServerSession ssh) throws QException
|
||||
{
|
||||
GatewaySession state = new GatewaySession(instance, ssh, clock);
|
||||
ssh.setAttribute(GatewaySession.ATTRIBUTE, state);
|
||||
state.connectionId = audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord()
|
||||
.withValue("sessionId", state.id).withValue("remoteIp", state.remoteIp).withValue("podName", podName)
|
||||
.withValue("connectedAt", clock.instant()).withValue("lastSeenAt", clock.instant())
|
||||
.withValue("clientChannel", "sftp").withValue("outcome", "AUTH_FAILED")).getValueInteger("id");
|
||||
state.connectedAt = clock.instant();
|
||||
sessions.put(state.id, state);
|
||||
prune(connections, clock.instant().minusSeconds(60));
|
||||
Deque<Instant> window = connections.computeIfAbsent(state.remoteIp, ignored -> new ArrayDeque<>());
|
||||
@@ -144,6 +143,7 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
|
||||
}
|
||||
QRecord account = first(FileGatewayTables.ACCOUNT, "username", username);
|
||||
String fingerprint = KeyUtils.getFingerPrint(BuiltinDigests.sha256, publicKey);
|
||||
ensureConnectionRow(state);
|
||||
audit.update(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("id", state.connectionId)
|
||||
.withValue("username", username).withValue("keyFingerprint", fingerprint).withValue("clientVersion", ssh.getClientVersion())
|
||||
.withValue("accountId", account == null ? null : account.getValue("id")));
|
||||
@@ -292,11 +292,30 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Write the connection-log row on first need (an auth attempt or a refusal).
|
||||
** Its outcome stays AUTH_FAILED until the sign-in succeeds.
|
||||
*******************************************************************************/
|
||||
private void ensureConnectionRow(GatewaySession state) throws QException
|
||||
{
|
||||
if(state.connectionId == null)
|
||||
{
|
||||
Instant at = state.connectedAt == null ? clock.instant() : state.connectedAt;
|
||||
state.connectionId = audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord()
|
||||
.withValue("sessionId", state.id).withValue("remoteIp", state.remoteIp).withValue("podName", podName)
|
||||
.withValue("connectedAt", at).withValue("lastSeenAt", clock.instant())
|
||||
.withValue("clientChannel", "sftp").withValue("outcome", "AUTH_FAILED")).getValueInteger("id");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record the authentication outcome without credentials.
|
||||
*******************************************************************************/
|
||||
private boolean outcome(GatewaySession state, String outcome, String reason) throws QException
|
||||
{
|
||||
ensureConnectionRow(state);
|
||||
audit.update(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("id", state.connectionId)
|
||||
.withValue("outcome", outcome).withValue("disconnectReason", reason).withValue("lastSeenAt", clock.instant()));
|
||||
if(reason != null)
|
||||
@@ -364,7 +383,7 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
|
||||
QRecord connection = first(FileGatewayTables.CONNECTION_LOG, "id", state.connectionId);
|
||||
if(account == null || key == null || !"ACTIVE".equals(account.getValueString("status"))
|
||||
|| expired(account, "expiresAt") || expired(key, "expiresAt") || key.getValue("revokedAt") != null
|
||||
|| connection == null || connection.getValue("disconnectedAt") != null)
|
||||
|| connection == null || connection.getValue("disconnectedAt") != null || connection.getValue("endRequestedAt") != null)
|
||||
{
|
||||
state.disconnectReason = connection != null && connection.getValueString("disconnectReason") != null
|
||||
? connection.getValueString("disconnectReason") : "Account or key revoked";
|
||||
|
||||
@@ -0,0 +1,268 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.storage;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.ClosedChannelException;
|
||||
import java.nio.channels.NonWritableChannelException;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A generated stream with a fixed-size rolling window; never regenerates or spools.
|
||||
*******************************************************************************/
|
||||
public final class GeneratedReadChannel implements SeekableByteChannel
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Charge newly consumed bytes and finalize the transfer when its handle closes.
|
||||
*******************************************************************************/
|
||||
public interface Audit
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Enforce limits before exposing the newly generated bytes to the client.
|
||||
*******************************************************************************/
|
||||
void charge(int bytes) throws IOException;
|
||||
|
||||
/*******************************************************************************
|
||||
** Record the final outcome, including stream and seek errors.
|
||||
*******************************************************************************/
|
||||
void finish(IOException failure) throws IOException;
|
||||
}
|
||||
|
||||
private final InputStream input;
|
||||
private final byte[] window;
|
||||
private final Audit audit;
|
||||
private long end;
|
||||
private long position;
|
||||
private boolean eof;
|
||||
private boolean open = true;
|
||||
private IOException failure;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The supplied stream is owned by this channel from construction through close.
|
||||
*******************************************************************************/
|
||||
public GeneratedReadChannel(InputStream input, int windowBytes, Audit audit)
|
||||
{
|
||||
if(windowBytes < 1)
|
||||
{
|
||||
throw (new IllegalArgumentException("Read-ahead window must be positive"));
|
||||
}
|
||||
this.input = input;
|
||||
this.window = new byte[windowBytes];
|
||||
this.audit = audit;
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Retain the most recent window, including skipped bytes for pipelined reads.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized int read(ByteBuffer destination) throws IOException
|
||||
{
|
||||
checkOpen();
|
||||
int total = 0;
|
||||
while(destination.hasRemaining())
|
||||
{
|
||||
int count = readChunk(destination);
|
||||
if(count < 0)
|
||||
{
|
||||
return (total == 0 ? -1 : total);
|
||||
}
|
||||
total += count;
|
||||
}
|
||||
return (total);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fill one ring segment; the public read fills its request to avoid false MINA EOF.
|
||||
*******************************************************************************/
|
||||
private int readChunk(ByteBuffer destination) throws IOException
|
||||
{
|
||||
checkOpen();
|
||||
if(failure != null)
|
||||
{
|
||||
throw (failure);
|
||||
}
|
||||
if(!destination.hasRemaining())
|
||||
{
|
||||
return (0);
|
||||
}
|
||||
try
|
||||
{
|
||||
if(eof && position >= end)
|
||||
{
|
||||
return (-1);
|
||||
}
|
||||
if(position < Math.max(0, end - window.length) || position - end >= window.length)
|
||||
{
|
||||
throw (new IOException("Read outside the generated-file window at " + position + " (stream offset " + end + "); download sequentially"));
|
||||
}
|
||||
while(position >= end && !eof)
|
||||
{
|
||||
int offset = (int) (end % window.length);
|
||||
int length = (int) Math.min(Math.min(65536, window.length - offset), position - end + Math.min(destination.remaining(), window.length));
|
||||
int count = input.readNBytes(window, offset, length);
|
||||
if(count == 0)
|
||||
{
|
||||
eof = true;
|
||||
}
|
||||
else if(count > 0)
|
||||
{
|
||||
audit.charge(count);
|
||||
end += count;
|
||||
}
|
||||
}
|
||||
if(position >= end)
|
||||
{
|
||||
return (-1);
|
||||
}
|
||||
int offset = (int) (position % window.length);
|
||||
int count = (int) Math.min(Math.min(destination.remaining(), end - position), window.length - offset);
|
||||
destination.put(window, offset, count);
|
||||
position += count;
|
||||
return (count);
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
failure = e;
|
||||
try
|
||||
{
|
||||
input.close();
|
||||
}
|
||||
catch(IOException closeError)
|
||||
{
|
||||
e.addSuppressed(closeError);
|
||||
}
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generated content is read-only.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public int write(ByteBuffer source)
|
||||
{
|
||||
throw (new NonWritableChannelException());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Current logical read offset.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized long position() throws IOException
|
||||
{
|
||||
checkOpen();
|
||||
return (position);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Validate window membership on read, when an actual request is made.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized SeekableByteChannel position(long value) throws IOException
|
||||
{
|
||||
checkOpen();
|
||||
if(value < 0)
|
||||
{
|
||||
throw (new IllegalArgumentException("Negative read offset"));
|
||||
}
|
||||
position = value;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generated size is unknown; consumers must read until EOF.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public long size() throws IOException
|
||||
{
|
||||
checkOpen();
|
||||
return (0);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generated content cannot be truncated.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public SeekableByteChannel truncate(long size)
|
||||
{
|
||||
throw (new NonWritableChannelException());
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Handle state.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean isOpen()
|
||||
{
|
||||
return (open);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Always close the HTTP stream and finalize exactly one transfer row.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public synchronized void close() throws IOException
|
||||
{
|
||||
if(open)
|
||||
{
|
||||
open = false;
|
||||
try
|
||||
{
|
||||
input.close();
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
failure = e;
|
||||
throw (e);
|
||||
}
|
||||
finally
|
||||
{
|
||||
audit.finish(failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject operations after close.
|
||||
*******************************************************************************/
|
||||
private void checkOpen() throws ClosedChannelException
|
||||
{
|
||||
if(!open)
|
||||
{
|
||||
throw (new ClosedChannelException());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,9 @@ public final class ObjectReadChannel implements SeekableByteChannel
|
||||
private final Ranges ranges;
|
||||
private final Completion completion;
|
||||
private long bytesRead;
|
||||
private Completion fullDownload;
|
||||
private long contiguousEnd;
|
||||
private final java.util.TreeMap<Long, Long> readRanges = new java.util.TreeMap<>();
|
||||
private boolean failed;
|
||||
private long position;
|
||||
private boolean open = true;
|
||||
@@ -109,6 +112,7 @@ public final class ObjectReadChannel implements SeekableByteChannel
|
||||
throw (new IOException("Stored object ended before its recorded size"));
|
||||
}
|
||||
destination.put(bytes);
|
||||
recordRange(position, position + bytes.length);
|
||||
position += bytes.length;
|
||||
bytesRead += bytes.length;
|
||||
return (bytes.length);
|
||||
@@ -191,9 +195,45 @@ public final class ObjectReadChannel implements SeekableByteChannel
|
||||
{
|
||||
open = false;
|
||||
completion.done(bytesRead, failed);
|
||||
if(!failed && contiguousEnd >= size && fullDownload != null)
|
||||
{
|
||||
fullDownload.done(bytesRead, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Optional acknowledgment, only after all bytes have been read on this handle.
|
||||
*******************************************************************************/
|
||||
public ObjectReadChannel withFullDownload(Completion callback)
|
||||
{
|
||||
fullDownload = callback;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Bounded interval tracking avoids counting repeated ranged reads as a complete file.
|
||||
*******************************************************************************/
|
||||
private void recordRange(long start, long end)
|
||||
{
|
||||
if(start <= contiguousEnd)
|
||||
{
|
||||
contiguousEnd = Math.max(contiguousEnd, end);
|
||||
}
|
||||
else if(readRanges.size() < 4096)
|
||||
{
|
||||
readRanges.merge(start, end, Math::max);
|
||||
}
|
||||
while(!readRanges.isEmpty() && readRanges.firstKey() <= contiguousEnd)
|
||||
{
|
||||
contiguousEnd = Math.max(contiguousEnd, readRanges.pollFirstEntry().getValue());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reject use after close.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -63,6 +63,7 @@ public class UploadJobWorker implements AutoCloseable
|
||||
private final AuditWriter audit;
|
||||
private final Clock clock;
|
||||
private final String pod;
|
||||
private java.util.function.Function<String, String> environment = System::getenv;
|
||||
private volatile boolean stopped;
|
||||
private ScheduledExecutorService pool;
|
||||
|
||||
@@ -80,6 +81,17 @@ public class UploadJobWorker implements AutoCloseable
|
||||
audit = new AuditWriter(instance);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Share the host-key environment seam; secret values are never persisted or logged.
|
||||
*******************************************************************************/
|
||||
public UploadJobWorker withEnvironment(java.util.function.Function<String, String> environment)
|
||||
{
|
||||
this.environment = environment;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Each scheduled lane handles one job at a time, without an unbounded work queue.
|
||||
*******************************************************************************/
|
||||
@@ -222,6 +234,19 @@ public class UploadJobWorker implements AutoCloseable
|
||||
return;
|
||||
}
|
||||
}
|
||||
if(Boolean.TRUE.equals(account.getValueBoolean("pgpDecryptUploads")) && job.getValueString("fileName").toLowerCase(java.util.Locale.ROOT).matches(".*[.](pgp|gpg)$"))
|
||||
{
|
||||
try
|
||||
{
|
||||
decrypt(job, account);
|
||||
file = inbound(job, account);
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
finish(job, new QRecord().withValue("status", "REJECTED").withValue("errorMessage", "PGP decryption failed: invalid ciphertext, key, integrity or size limit"), account);
|
||||
return;
|
||||
}
|
||||
}
|
||||
FileGatewayConsumer consumer = consumer(job, user);
|
||||
if(consumer == null)
|
||||
{
|
||||
@@ -251,6 +276,44 @@ public class UploadJobWorker implements AutoCloseable
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Keep the original for the archive and persist authenticated plaintext for retries.
|
||||
*******************************************************************************/
|
||||
private void decrypt(QRecord job, QRecord account) throws IOException, QException
|
||||
{
|
||||
if(job.getValue("plaintextStorageKey") != null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
String key = "file-gateway/plaintext/" + job.getValue("id") + "/" + java.util.UUID.randomUUID();
|
||||
String pass = config.getPgpPassphraseEnvVarName() == null ? null : environment.apply(config.getPgpPassphraseEnvVarName());
|
||||
char[] password = pass == null ? new char[0] : pass.toCharArray();
|
||||
try(var source = store.open(job.getValueString("storageKey"));
|
||||
var plaintext = com.kingsrook.qbits.filegateway.pgp.PgpStreams.decrypt(source,
|
||||
config.getPgpPrivateKeyEnvVarName() == null ? null : environment.apply(config.getPgpPrivateKeyEnvVarName()), password,
|
||||
(account.getValueInteger("maxFileSizeMb") == null ? config.getDefaultMaxFileSizeMb() : account.getValueInteger("maxFileSizeMb")) * UploadJobs.MIB))
|
||||
{
|
||||
var receipt = store.put(key, plaintext);
|
||||
QRecord changes = new QRecord().withValue("plaintextStorageKey", key).withValue("plaintextSha256", receipt.sha256()).withValue("plaintextSizeBytes", receipt.size());
|
||||
if(!persistOwned(job, changes))
|
||||
{
|
||||
throw (new IOException("Upload claim expired during PGP decryption"));
|
||||
}
|
||||
job.getValues().putAll(changes.getValues());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
store.delete(key);
|
||||
throw (new IOException("PGP decryption failed", e));
|
||||
}
|
||||
finally
|
||||
{
|
||||
java.util.Arrays.fill(password, '\0');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Registry lookup is by declared name, with the built-in bulk-load consumer reserved.
|
||||
*******************************************************************************/
|
||||
@@ -275,15 +338,24 @@ public class UploadJobWorker implements AutoCloseable
|
||||
/*******************************************************************************
|
||||
** Immutable delivery envelope; each openStream gets an independent object stream.
|
||||
*******************************************************************************/
|
||||
private InboundFile inbound(QRecord job, QRecord account)
|
||||
private InboundFile inbound(QRecord job, QRecord account) throws QException
|
||||
{
|
||||
return (new InboundFile(job.getValueInteger("id"), null, account.getValueString("username"), account.getValueString("userId"),
|
||||
job.getValueString("remoteAddress"), job.getValueString("path"), job.getValueString("fileName"), job.getValueLong("sizeBytes"),
|
||||
job.getValueString("sha256"), job.getValueInstant("createDate"), config.getFileGatewayBucketAlias(), job.getValueString("storageKey"), () ->
|
||||
String endpointName = job.getValue("aliasId") == null ? null
|
||||
: audit.query(FileGatewayTables.ALIAS, UploadJobs.equal("id", job.getValue("aliasId"))).getFirst().getValueString("name");
|
||||
boolean plaintext = job.getValue("plaintextStorageKey") != null;
|
||||
String storageKey = job.getValueString(plaintext ? "plaintextStorageKey" : "storageKey");
|
||||
String filename = job.getValueString("fileName");
|
||||
if(plaintext)
|
||||
{
|
||||
filename = filename.substring(0, filename.length() - 4);
|
||||
}
|
||||
return (new InboundFile(job.getValueInteger("id"), endpointName, account.getValueString("username"), account.getValueString("userId"),
|
||||
job.getValueString("remoteAddress"), job.getValueString("path"), filename, job.getValueLong(plaintext ? "plaintextSizeBytes" : "sizeBytes"),
|
||||
job.getValueString(plaintext ? "plaintextSha256" : "sha256"), job.getValueInstant("createDate"), config.getFileGatewayBucketAlias(), storageKey, () ->
|
||||
{
|
||||
try
|
||||
{
|
||||
return (store.open(job.getValueString("storageKey")));
|
||||
return (store.open(storageKey));
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
|
||||
@@ -65,6 +65,16 @@ public class UploadJobs
|
||||
** The target was resolved through the user's metadata; endpoint routing can reuse this seam.
|
||||
*******************************************************************************/
|
||||
public SeekableByteChannel receive(GatewaySession session, String path, String table, Integer profile) throws IOException
|
||||
{
|
||||
return (receive(session, path, table, profile, null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist the resolved endpoint with the upload before any bytes are accepted.
|
||||
*******************************************************************************/
|
||||
public SeekableByteChannel receive(GatewaySession session, String path, String table, Integer profile, QRecord alias) throws IOException
|
||||
{
|
||||
String name = path.substring(path.lastIndexOf('/') + 1);
|
||||
if(name.isBlank() || name.contains(":") || name.contains("\\") || name.chars().anyMatch(Character::isISOControl))
|
||||
@@ -82,7 +92,8 @@ public class UploadJobs
|
||||
QRecord job = audit.insert(FileGatewayTables.UPLOAD_JOB, new QRecord()
|
||||
.withValue("accountId", session.account.getValue("id")).withValue("connectionLogId", session.connectionId)
|
||||
.withValue("remoteAddress", session.remoteIp).withValue("tableName", table).withValue("savedBulkLoadProfileId", profile)
|
||||
.withValue("consumerName", "bulk-load").withValue("path", path).withValue("fileName", name).withValue("storageKey", key)
|
||||
.withValue("aliasId", alias == null ? null : alias.getValue("id"))
|
||||
.withValue("consumerName", alias != null && "INBOX".equals(alias.getValueString("kind")) ? alias.getValueString("consumerName") : "bulk-load").withValue("path", path).withValue("fileName", name).withValue("storageKey", key)
|
||||
.withValue("status", "RECEIVING").withValue("sizeBytes", 0L).withValue("attempts", 0).withValue("createDate", clock.instant()));
|
||||
long started = System.nanoTime();
|
||||
return (new UploadChannel(store, key, Math.toIntExact(config.getUploadPartSizeMb() * MIB), config.getUploadReorderWindowMb() * MIB, new UploadChannel.Lifecycle()
|
||||
|
||||
@@ -22,6 +22,7 @@ package com.kingsrook.qbits.filegateway.vfs;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.AccessDeniedException;
|
||||
import java.nio.file.NoSuchFileException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
@@ -34,6 +35,7 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInvoker;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiStreamResponse;
|
||||
import com.kingsrook.qqq.openapi.model.HttpMethod;
|
||||
|
||||
|
||||
@@ -94,7 +96,7 @@ public class GatewayUiApiClient
|
||||
/*******************************************************************************
|
||||
** Apply the fixed channel identity, credentials and contract check on every call.
|
||||
*******************************************************************************/
|
||||
protected JsonNode invoke(UiApiRequest request) throws IOException
|
||||
public JsonNode invoke(UiApiRequest request) throws IOException
|
||||
{
|
||||
if(invoker.findOperation(request.getMethod(), request.getPath()).isEmpty())
|
||||
{
|
||||
@@ -110,11 +112,11 @@ public class GatewayUiApiClient
|
||||
}
|
||||
if(response.getStatus() == 401 || response.getStatus() == 403)
|
||||
{
|
||||
throw (new AccessDeniedException(request.getPath()));
|
||||
throw (new AccessDeniedException(request.getPath(), null, response.getBody()));
|
||||
}
|
||||
if(!response.isSuccess())
|
||||
{
|
||||
throw (new IOException("UI API request failed with status " + response.getStatus()));
|
||||
throw (new IOException("UI API request failed with status " + response.getStatus() + ": " + response.getBody()));
|
||||
}
|
||||
return (response.getJson());
|
||||
}
|
||||
@@ -123,4 +125,39 @@ public class GatewayUiApiClient
|
||||
throw (new IOException("User session unavailable", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Open a binary response as this user, retaining only a bounded error body.
|
||||
*******************************************************************************/
|
||||
public UiApiStreamResponse stream(UiApiRequest request) throws IOException
|
||||
{
|
||||
if(invoker.findOperation(request.getMethod(), request.getPath()).isEmpty())
|
||||
{
|
||||
throw (new IOException("Gateway requires a spec-defined UI API operation"));
|
||||
}
|
||||
try
|
||||
{
|
||||
UiApiStreamResponse response = invoker.invokeForStream(request.withCredentials(UiApiCredentials.ofInProcessSession(session.token()))
|
||||
.withInvokedBy("file-gateway").withClientDetail(session.id));
|
||||
if(response.isSuccess())
|
||||
{
|
||||
return (response);
|
||||
}
|
||||
try(response)
|
||||
{
|
||||
String error = new String(response.getInputStream().readNBytes(65536), StandardCharsets.UTF_8);
|
||||
if(response.getStatus() == 401 || response.getStatus() == 403)
|
||||
{
|
||||
throw (new AccessDeniedException(request.getPath(), null, error));
|
||||
}
|
||||
throw (new IOException("UI API request failed with status " + response.getStatus() + ": " + error));
|
||||
}
|
||||
}
|
||||
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
|
||||
{
|
||||
throw (new IOException("User session unavailable", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.vfs;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.channels.SeekableByteChannel;
|
||||
import java.nio.file.AccessDeniedException;
|
||||
import java.nio.file.NoSuchFileException;
|
||||
import java.time.Clock;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.sftp.AuditWriter;
|
||||
import com.kingsrook.qbits.filegateway.sftp.GatewaySession;
|
||||
import com.kingsrook.qbits.filegateway.storage.GeneratedReadChannel;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiStreamResponse;
|
||||
import com.kingsrook.qqq.openapi.model.HttpMethod;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generate user-authorized exports and reports through the UI API, with durable quotas.
|
||||
*******************************************************************************/
|
||||
public final class GeneratedDownloads
|
||||
{
|
||||
private final GatewayUiApiClient client;
|
||||
private final GatewaySession session;
|
||||
private final AuditWriter audit;
|
||||
private final FileGatewayQBitConfig config;
|
||||
private final Clock clock;
|
||||
private final boolean reportsAvailable;
|
||||
private final ObjectMapper mapper = new ObjectMapper();
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Inspect only host configuration for report storage; visibility still uses user metadata.
|
||||
*******************************************************************************/
|
||||
public GeneratedDownloads(GatewayUiApiClient client, GatewaySession session, AuditWriter audit, FileGatewayQBitConfig config, Clock clock, QInstance instance)
|
||||
{
|
||||
this.client = client;
|
||||
this.session = session;
|
||||
this.audit = audit;
|
||||
this.config = config;
|
||||
this.clock = clock;
|
||||
var process = instance.getProcess("renderSavedReport");
|
||||
reportsAvailable = process != null && process.getInputFields().stream().anyMatch(field -> "storageTableName".equals(field.getName())
|
||||
&& field.getDefaultValue() != null && instance.getTable(field.getDefaultValue().toString()) != null);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A render process without its configured storage table cannot provide downloads.
|
||||
*******************************************************************************/
|
||||
public boolean reportsAvailable()
|
||||
{
|
||||
return (reportsAvailable);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Start generation at open, owning the HTTP stream until the SFTP handle closes.
|
||||
*******************************************************************************/
|
||||
public SeekableByteChannel open(VirtualTree.Node node) throws IOException
|
||||
{
|
||||
Transfer transfer = new Transfer(node);
|
||||
try
|
||||
{
|
||||
if(!session.operationAllowed())
|
||||
{
|
||||
transfer.limit = true;
|
||||
throw (new IOException("Operation limit exceeded"));
|
||||
}
|
||||
int window = Math.toIntExact(config.getDownloadReadAheadMb() * 1024L * 1024L);
|
||||
if(window < 1)
|
||||
{
|
||||
throw (new IOException("Read-ahead window must be positive"));
|
||||
}
|
||||
String prefix = node.path().substring(0, node.path().lastIndexOf('/'));
|
||||
prefix = prefix.substring(0, prefix.lastIndexOf('/'));
|
||||
String table = node.job() == null ? prefix.substring(prefix.lastIndexOf('/') + 1) : node.job().getValueString("tableName");
|
||||
JsonNode record = null;
|
||||
for(JsonNode row : client.savedObjects("VIEW".equals(node.kind()) ? "savedView" : "savedReport", table))
|
||||
{
|
||||
if(node.objectId().equals(row.path("values").path("id").asText()))
|
||||
{
|
||||
record = row.path("values");
|
||||
}
|
||||
}
|
||||
if(record == null)
|
||||
{
|
||||
throw (new NoSuchFileException(node.path()));
|
||||
}
|
||||
UiApiStreamResponse response = "VIEW".equals(node.kind()) ? export(table, record, transfer.format) : report(node.objectId(), transfer.format);
|
||||
return (new GeneratedReadChannel(response.getInputStream(), window, transfer));
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
IOException failure = e instanceof IOException io ? io : new IOException("Unable to generate download", e);
|
||||
transfer.finish(failure);
|
||||
throw (failure);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the Next/Material saved-view schema, retaining filter ordering and visible columns.
|
||||
*******************************************************************************/
|
||||
private UiApiStreamResponse export(String table, JsonNode record, String format) throws IOException
|
||||
{
|
||||
JsonNode view = mapper.readTree(record.path("viewJson").asText());
|
||||
if(view == null || !view.isObject())
|
||||
{
|
||||
throw (new IOException("Saved view has invalid viewJson"));
|
||||
}
|
||||
ObjectNode body = mapper.createObjectNode().put("format", format).put("includeHeaderRow", true);
|
||||
JsonNode filter = view.has("queryFilter") ? view.path("queryFilter") : view.path("filter");
|
||||
if(filter.isObject())
|
||||
{
|
||||
ObjectNode query = filter.deepCopy();
|
||||
query.remove(List.of("skip", "limit"));
|
||||
body.set("filter", query);
|
||||
}
|
||||
JsonNode columns = view.path("queryColumns").path("columns");
|
||||
if(columns.isArray())
|
||||
{
|
||||
var names = body.putArray("fieldNames");
|
||||
for(JsonNode column : columns)
|
||||
{
|
||||
String name = column.path("name").asText();
|
||||
if(column.path("isVisible").asBoolean(true) && !name.isEmpty() && !name.equals("__check__"))
|
||||
{
|
||||
names.add(name);
|
||||
}
|
||||
}
|
||||
if(names.isEmpty())
|
||||
{
|
||||
throw (new IOException("Saved view has no visible export columns"));
|
||||
}
|
||||
}
|
||||
else if(view.path("fieldNames").isArray())
|
||||
{
|
||||
body.set("fieldNames", view.path("fieldNames"));
|
||||
}
|
||||
return (client.stream(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/table/{tableName}/export")
|
||||
.withPathParam("tableName", table).withJsonBody(body)));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Select one record, submit the format input, then follow asynchronous render status.
|
||||
*******************************************************************************/
|
||||
private UiApiStreamResponse report(String id, String format) throws IOException
|
||||
{
|
||||
String values = mapper.writeValueAsString(Map.of("reportFormat", format));
|
||||
JsonNode response = client.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/processes/{processName}/init")
|
||||
.withPathParam("processName", "renderSavedReport").withFormParam("recordsParam", "recordIds").withFormParam("recordIds", id)
|
||||
.withFormParam("values", values));
|
||||
String uuid = response.path("processUUID").asText();
|
||||
String job = response.path("jobUUID").asText();
|
||||
for(int poll = 0; poll < 3600; poll++)
|
||||
{
|
||||
String type = response.path("type").asText();
|
||||
if("ERROR".equals(type))
|
||||
{
|
||||
throw (new IOException(response.path("userFacingError").asText("Report generation failed")));
|
||||
}
|
||||
if("JOB_STARTED".equals(type) || "RUNNING".equals(type))
|
||||
{
|
||||
if(response.hasNonNull("jobUUID"))
|
||||
{
|
||||
job = response.path("jobUUID").asText();
|
||||
}
|
||||
try
|
||||
{
|
||||
Thread.sleep(100);
|
||||
}
|
||||
catch(InterruptedException e)
|
||||
{
|
||||
Thread.currentThread().interrupt();
|
||||
throw (new IOException("Report generation interrupted", e));
|
||||
}
|
||||
response = client.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath("/processes/{processName}/{processUUID}/status/{jobUUID}")
|
||||
.withPathParam("processName", "renderSavedReport").withPathParam("processUUID", uuid).withPathParam("jobUUID", job));
|
||||
continue;
|
||||
}
|
||||
JsonNode output = response.path("values");
|
||||
if(output.hasNonNull("storageReference") && output.hasNonNull("downloadFileName") && output.hasNonNull("storageTableName"))
|
||||
{
|
||||
return (client.stream(new UiApiRequest().withMethod(HttpMethod.GET).withPath("/download/{file}")
|
||||
.withPathParam("file", output.path("downloadFileName").asText())
|
||||
.withQueryParam("storageTableName", output.path("storageTableName").asText())
|
||||
.withQueryParam("storageReference", output.path("storageReference").asText())));
|
||||
}
|
||||
if(!"input".equals(response.path("nextStep").asText()))
|
||||
{
|
||||
throw (new IOException("Unexpected report step: " + response.path("nextStep").asText()));
|
||||
}
|
||||
response = client.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/processes/{processName}/{processUUID}/step/{stepName}")
|
||||
.withPathParam("processName", "renderSavedReport").withPathParam("processUUID", uuid).withPathParam("stepName", "input")
|
||||
.withFormParam("values", values));
|
||||
}
|
||||
throw (new IOException("Report generation timed out"));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One durable row per handle, updated under the account lock as bytes flow.
|
||||
*******************************************************************************/
|
||||
private final class Transfer implements GeneratedReadChannel.Audit
|
||||
{
|
||||
private final long began = System.nanoTime();
|
||||
private final String format;
|
||||
private final QRecord row;
|
||||
private long bytes;
|
||||
private boolean limit;
|
||||
|
||||
/*******************************************************************************
|
||||
** Default unfinished transfers to ERROR, including a process terminated mid-read.
|
||||
*******************************************************************************/
|
||||
private Transfer(VirtualTree.Node node) throws IOException
|
||||
{
|
||||
format = node.job() == null ? node.path().substring(node.path().lastIndexOf('.') + 1) : node.job().getValueString("format");
|
||||
try
|
||||
{
|
||||
row = audit.insert(FileGatewayTables.TRANSFER_LOG, new QRecord().withValue("accountId", session.account.getValue("id"))
|
||||
.withValue("connectionLogId", session.connectionId).withValue("operation", "DOWNLOAD").withValue("path", node.path())
|
||||
.withValue("objectKind", node.kind()).withValue("objectId", node.objectId()).withValue("format", format)
|
||||
.withValue("bytes", 0L).withValue("outcome", "ERROR").withValue("clientChannel", "sftp/" + session.id));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
throw (new IOException("Unable to persist download audit", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Filter by account, operation and UTC day; include in-progress and failed transfers.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void charge(int count) throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
boolean allowed = audit.withAccountLock("id", session.account.getValue("id"), () ->
|
||||
{
|
||||
long maximum = config.getMaxGeneratedFileMb() * 1024L * 1024L;
|
||||
Long dailyMb = session.account.getValueLong("maxDownloadMbPerDay");
|
||||
long total = 0;
|
||||
if(dailyMb != null)
|
||||
{
|
||||
var start = clock.instant().atZone(ZoneOffset.UTC).toLocalDate().atStartOfDay(ZoneOffset.UTC).toInstant();
|
||||
for(int skip = 0; ; skip += 250)
|
||||
{
|
||||
var rows = audit.query(FileGatewayTables.TRANSFER_LOG, new QQueryFilter()
|
||||
.withCriteria(new QFilterCriteria("accountId", QCriteriaOperator.EQUALS, session.account.getValue("id")))
|
||||
.withCriteria(new QFilterCriteria("operation", QCriteriaOperator.EQUALS, "DOWNLOAD"))
|
||||
.withCriteria(new QFilterCriteria("createDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, start))
|
||||
.withOrderBy(new QFilterOrderBy("id", true)).withSkip(skip).withLimit(250));
|
||||
for(QRecord record : rows)
|
||||
{
|
||||
Long value = record.getValueLong("bytes");
|
||||
total += value == null ? 0 : value;
|
||||
}
|
||||
if(rows.size() < 250)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(bytes + count > maximum || (dailyMb != null && total + count > dailyMb * 1024L * 1024L))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
audit.update(FileGatewayTables.TRANSFER_LOG, new QRecord().withValue("id", row.getValue("id")).withValue("bytes", bytes + count));
|
||||
return (true);
|
||||
});
|
||||
if(!allowed)
|
||||
{
|
||||
limit = true;
|
||||
throw (new IOException("Generated file or daily download limit exceeded"));
|
||||
}
|
||||
bytes += count;
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
throw (new IOException("Unable to enforce download limits", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve API denial and limit outcomes instead of collapsing them to generic errors.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void finish(IOException failure) throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
audit.update(FileGatewayTables.TRANSFER_LOG, new QRecord().withValue("id", row.getValue("id"))
|
||||
.withValue("durationMs", (System.nanoTime() - began) / 1_000_000)
|
||||
.withValue("outcome", limit ? "LIMIT" : failure instanceof AccessDeniedException ? "DENIED" : failure == null ? "OK" : "ERROR")
|
||||
.withValue("message", failure == null ? null : failure.getMessage()));
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
throw (new IOException("Unable to finalize download audit", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,8 @@ public class VirtualTree
|
||||
private final Clock clock;
|
||||
private UploadJobs uploads;
|
||||
private GatewayObjectStore store;
|
||||
private GeneratedDownloads downloads;
|
||||
private com.kingsrook.qbits.filegateway.outbox.OutboxService outbox;
|
||||
private JsonNode metadata;
|
||||
private Instant metadataAt;
|
||||
private final Map<String, JsonNode> saved = new HashMap<>();
|
||||
@@ -101,6 +103,27 @@ public class VirtualTree
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Attach generated downloads and the host's report-storage availability.
|
||||
*******************************************************************************/
|
||||
public VirtualTree withDownloads(GeneratedDownloads downloads)
|
||||
{
|
||||
this.downloads = downloads;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use the same provider in the web deployment and the virtual outbox folders.
|
||||
*******************************************************************************/
|
||||
public VirtualTree withOutbox(com.kingsrook.qbits.filegateway.outbox.OutboxService outbox)
|
||||
{
|
||||
this.outbox = outbox;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Cache user metadata for sixty seconds.
|
||||
@@ -204,9 +227,42 @@ public class VirtualTree
|
||||
** Resolve permitted app and table folders from UI API metadata.
|
||||
*******************************************************************************/
|
||||
private List<Node> children(String path) throws IOException
|
||||
{
|
||||
List<Node> nodes = rawChildren(path);
|
||||
if(Boolean.TRUE.equals(session.account.getValueBoolean("pgpEncryptDownloads")))
|
||||
{
|
||||
return (nodes.stream().map(node -> node.directory() || "TEMPORARY".equals(node.kind()) ? node
|
||||
: new Node(node.path() + ".pgp", false, node.kind(), node.objectId(), 0, node.job())).toList());
|
||||
}
|
||||
return (nodes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve canonical plaintext paths before applying download encryption names.
|
||||
*******************************************************************************/
|
||||
private List<Node> rawChildren(String path) throws IOException
|
||||
{
|
||||
JsonNode meta = metadata();
|
||||
List<Node> result = new ArrayList<>();
|
||||
List<Node> endpoints = endpointChildren(path, meta);
|
||||
if(!endpoints.isEmpty() || endpointMount(path, meta) != null)
|
||||
{
|
||||
result.addAll(endpoints);
|
||||
if(!"/".equals(path))
|
||||
{
|
||||
return (result);
|
||||
}
|
||||
}
|
||||
if("ENDPOINTS_ONLY".equals(session.account.getValueString("rootMode")))
|
||||
{
|
||||
if("/".equals(path))
|
||||
{
|
||||
return (result);
|
||||
}
|
||||
throw (new NoSuchFileException(path));
|
||||
}
|
||||
if("/".equals(path))
|
||||
{
|
||||
addAppChildren(result, "", meta.path("appTree"), meta);
|
||||
@@ -259,7 +315,7 @@ public class VirtualTree
|
||||
{
|
||||
folders.add("views");
|
||||
}
|
||||
if(meta.path("tables").has("savedReport") && meta.path("processes").has("renderSavedReport"))
|
||||
if(downloads != null && downloads.reportsAvailable() && meta.path("tables").has("savedReport") && meta.path("processes").has("renderSavedReport"))
|
||||
{
|
||||
folders.add("reports");
|
||||
}
|
||||
@@ -325,6 +381,130 @@ public class VirtualTree
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Endpoint targets are resolved from the same user-scoped saved-object queries.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> visibleEndpoints(JsonNode meta) throws IOException
|
||||
{
|
||||
try
|
||||
{
|
||||
List<QRecord> visible = new ArrayList<>();
|
||||
for(QRecord alias : audit.query(FileGatewayTables.ALIAS, UploadJobs.equal("isActive", true)))
|
||||
{
|
||||
if(alias.getValue("accountId") != null && !java.util.Objects.equals(alias.getValue("accountId"), session.account.getValue("id")))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String kind = alias.getValueString("kind");
|
||||
if(List.of("INBOX", "OUTBOX").contains(kind))
|
||||
{
|
||||
visible.add(alias);
|
||||
continue;
|
||||
}
|
||||
String table = alias.getValueString("tableName");
|
||||
if(table == null || !meta.path("tables").has(table))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String source = "BULK_LOAD".equals(kind) ? "savedBulkLoadProfile" : "VIEW".equals(kind) ? "savedView" : "savedReport";
|
||||
if(!meta.path("tables").has(source)
|
||||
|| ("BULK_LOAD".equals(kind) && (!meta.path("tables").path(table).path("insertPermission").asBoolean(false) || !meta.path("processes").has(table + ".bulkInsert")))
|
||||
|| ("REPORT".equals(kind) && (downloads == null || !downloads.reportsAvailable() || !meta.path("processes").has("renderSavedReport"))))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
for(JsonNode row : saved(source, table))
|
||||
{
|
||||
if(row.path("values").path("id").asText().equals(alias.getValueString(source + "Id")))
|
||||
{
|
||||
alias.withValue("_label", row.path("values").path("label").asText());
|
||||
visible.add(alias);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return (visible);
|
||||
}
|
||||
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
|
||||
{
|
||||
throw (new IOException("Unable to resolve endpoints", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An exact visible mount, including an empty upload or outbox folder.
|
||||
*******************************************************************************/
|
||||
private QRecord endpointMount(String path, JsonNode meta) throws IOException
|
||||
{
|
||||
return (visibleEndpoints(meta).stream().filter(alias -> path.equals(alias.getValueString("path"))).findFirst().orElse(null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Mount ancestors reveal only the next segment, preserving endpoint chroots.
|
||||
*******************************************************************************/
|
||||
private List<Node> endpointChildren(String path, JsonNode meta) throws IOException
|
||||
{
|
||||
List<Node> nodes = new ArrayList<>();
|
||||
String prefix = "/".equals(path) ? "/" : path + "/";
|
||||
for(QRecord alias : visibleEndpoints(meta))
|
||||
{
|
||||
String mount = alias.getValueString("path");
|
||||
if(mount.startsWith(prefix))
|
||||
{
|
||||
String suffix = mount.substring(prefix.length()).split("/")[0];
|
||||
String child = prefix + suffix;
|
||||
if(nodes.stream().noneMatch(node -> child.equals(node.path())))
|
||||
{
|
||||
nodes.add(new Node(child, true, "ENDPOINT", alias.getValueString("id"), 0, mount.equals(child) ? alias : null));
|
||||
}
|
||||
}
|
||||
if(!mount.equals(path))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String kind = alias.getValueString("kind");
|
||||
if(List.of("BULK_LOAD", "INBOX").contains(kind))
|
||||
{
|
||||
nodes.addAll(artifacts("", alias.getValueString("tableName"), path, "upload"));
|
||||
}
|
||||
else if("OUTBOX".equals(kind) && outbox != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
for(QRecord row : outbox.list(alias.getValueInteger("id")))
|
||||
{
|
||||
nodes.add(new Node(prefix + row.getValueString("fileName"), false, "OUTBOX", row.getValueString("id"), row.getValueLong("sizeBytes"), row));
|
||||
}
|
||||
}
|
||||
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
|
||||
{
|
||||
throw (new IOException("Unable to list outbox", e));
|
||||
}
|
||||
}
|
||||
else if(List.of("VIEW", "REPORT").contains(kind))
|
||||
{
|
||||
String format = alias.getValueString("format");
|
||||
String formats = session.account.getValueString("formats");
|
||||
if(format != null && List.of((formats == null ? "csv,xlsx" : formats).split("[,\\s]+")).contains(format))
|
||||
{
|
||||
String name = alias.getValueString("fileName");
|
||||
if(name == null || name.isBlank())
|
||||
{
|
||||
name = slug(alias.getValueString("_label")) + "." + format;
|
||||
}
|
||||
nodes.add(new Node(prefix + name, false, kind, alias.getValueString("VIEW".equals(kind) ? "savedViewId" : "savedReportId"), 0, alias));
|
||||
}
|
||||
}
|
||||
}
|
||||
return (nodes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sidecars and results expire after seven days; archive objects remain until retention.
|
||||
*******************************************************************************/
|
||||
@@ -364,7 +544,7 @@ public class VirtualTree
|
||||
job.getValueLong("resultsSizeBytes") == null ? 0 : job.getValueLong("resultsSizeBytes"), job));
|
||||
}
|
||||
if("archive".equals(folder) && !active && job.getValueString("sha256") != null
|
||||
&& !job.getValueString("storageKey").startsWith("file-gateway/quarantine/"))
|
||||
&& job.getValueString("storageKey") != null && !job.getValueString("storageKey").startsWith("file-gateway/quarantine/"))
|
||||
{
|
||||
String date = job.getValueInstant("createDate").atZone(ZoneOffset.UTC).toLocalDate().toString();
|
||||
String directory = prefix + "/archive/" + date;
|
||||
@@ -412,6 +592,12 @@ public class VirtualTree
|
||||
{
|
||||
String parent = path.substring(0, path.lastIndexOf('/'));
|
||||
Node target = stat(parent);
|
||||
if(uploads != null && target.job() != null && "ENDPOINT".equals(target.kind())
|
||||
&& List.of("BULK_LOAD", "INBOX").contains(target.job().getValueString("kind")))
|
||||
{
|
||||
QRecord alias = target.job();
|
||||
return (uploads.receive(session, path, alias.getValueString("tableName"), alias.getValueInteger("savedBulkLoadProfileId"), alias));
|
||||
}
|
||||
if(uploads == null || !target.directory() || !"PROFILE".equals(target.kind()) || target.objectId() == null)
|
||||
{
|
||||
deny(path, "UPLOAD", "Uploads require a bulk load profile folder");
|
||||
@@ -424,11 +610,91 @@ public class VirtualTree
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read only stored artifacts and generated sidecars; view/report export is phase c.
|
||||
** Open generated views/reports or stored artifacts without using the filesystem.
|
||||
*******************************************************************************/
|
||||
public SeekableByteChannel read(String path) throws IOException
|
||||
{
|
||||
Node node = stat(path);
|
||||
if(Boolean.TRUE.equals(session.account.getValueBoolean("pgpEncryptDownloads")) && !node.directory() && path.endsWith(".pgp"))
|
||||
{
|
||||
String plainPath = path.substring(0, path.length() - 4);
|
||||
Node plain = rawChildren(plainPath.substring(0, plainPath.lastIndexOf('/'))).stream()
|
||||
.filter(candidate -> candidate.path().equals(plainPath)).findFirst().orElseThrow(() -> new NoSuchFileException(path));
|
||||
return (encrypted(plain));
|
||||
}
|
||||
return (readNode(node));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Encrypt lazily into the existing fixed rolling window, without a producer thread.
|
||||
*******************************************************************************/
|
||||
private SeekableByteChannel encrypted(Node node) throws IOException
|
||||
{
|
||||
SeekableByteChannel source = readNode(node);
|
||||
try
|
||||
{
|
||||
var encrypted = com.kingsrook.qbits.filegateway.pgp.PgpStreams.encrypt(java.nio.channels.Channels.newInputStream(source),
|
||||
session.account.getValueString("pgpPublicKey"), node.path().substring(node.path().lastIndexOf('/') + 1));
|
||||
boolean[] complete = {false};
|
||||
var tracked = new java.io.FilterInputStream(encrypted)
|
||||
{
|
||||
@Override
|
||||
public int read(byte[] bytes, int offset, int length) throws IOException
|
||||
{
|
||||
int count = in.read(bytes, offset, length);
|
||||
if(count < 0)
|
||||
{
|
||||
complete[0] = true;
|
||||
}
|
||||
return (count);
|
||||
}
|
||||
};
|
||||
return (new com.kingsrook.qbits.filegateway.storage.GeneratedReadChannel(tracked, 65536, new com.kingsrook.qbits.filegateway.storage.GeneratedReadChannel.Audit()
|
||||
{
|
||||
@Override
|
||||
public void charge(int bytes)
|
||||
{
|
||||
// The underlying transfer accounts for plaintext bytes and applies its limits.
|
||||
}
|
||||
|
||||
@Override
|
||||
public void finish(IOException failure) throws IOException
|
||||
{
|
||||
if(complete[0] && failure == null && "OUTBOX".equals(node.kind()))
|
||||
{
|
||||
try
|
||||
{
|
||||
outbox.downloaded(node.job());
|
||||
}
|
||||
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
|
||||
{
|
||||
throw (new IOException("Unable to acknowledge encrypted outbox", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
source.close();
|
||||
throw (e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Open a resolved plaintext node through its original authorized delivery path.
|
||||
*******************************************************************************/
|
||||
private SeekableByteChannel readNode(Node node) throws IOException
|
||||
{
|
||||
String path = node.path();
|
||||
if(downloads != null && !node.directory() && List.of("VIEW", "REPORT").contains(node.kind()))
|
||||
{
|
||||
return (downloads.open(node));
|
||||
}
|
||||
if(node.job() == null || node.directory() || "TEMPORARY".equals(node.kind()))
|
||||
{
|
||||
deny(path, "DOWNLOAD", "This file is not downloadable");
|
||||
@@ -445,7 +711,22 @@ public class VirtualTree
|
||||
return (new ObjectReadChannel(content.length, (start, end) -> new ByteArrayInputStream(content, (int) start, (int) (end - start + 1)), completion));
|
||||
}
|
||||
String key = node.job().getValueString("RESULT".equals(node.kind()) ? "resultsStorageKey" : "storageKey");
|
||||
return (new ObjectReadChannel(node.size(), (start, end) -> store.openRange(key, start, end), completion));
|
||||
ObjectReadChannel channel = new ObjectReadChannel(node.size(), (start, end) -> store.openRange(key, start, end), completion);
|
||||
if("OUTBOX".equals(node.kind()) && !Boolean.TRUE.equals(session.account.getValueBoolean("pgpEncryptDownloads")))
|
||||
{
|
||||
channel.withFullDownload((bytes, failed) ->
|
||||
{
|
||||
try
|
||||
{
|
||||
outbox.downloaded(node.job());
|
||||
}
|
||||
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
|
||||
{
|
||||
throw (new IOException("Unable to acknowledge outbox download", e));
|
||||
}
|
||||
});
|
||||
}
|
||||
return (channel);
|
||||
}
|
||||
|
||||
|
||||
@@ -460,7 +741,7 @@ public class VirtualTree
|
||||
audit.insert(FileGatewayTables.TRANSFER_LOG, new QRecord().withValue("accountId", session.account.getValue("id"))
|
||||
.withValue("connectionLogId", session.connectionId).withValue("operation", "DOWNLOAD").withValue("path", node.path())
|
||||
.withValue("objectKind", "SIDECAR".equals(node.kind()) ? "RESULT" : node.kind()).withValue("objectId", node.objectId())
|
||||
.withValue("uploadJobId", node.job().getValue("id")).withValue("bytes", bytes)
|
||||
.withValue("uploadJobId", "OUTBOX".equals(node.kind()) ? null : node.job().getValue("id")).withValue("bytes", bytes)
|
||||
.withValue("sha256", "ARCHIVE".equals(node.kind()) ? node.job().getValue("sha256") : null)
|
||||
.withValue("durationMs", (System.nanoTime() - began) / 1_000_000).withValue("outcome", failed ? "ERROR" : "OK")
|
||||
.withValue("clientChannel", "sftp/" + session.id));
|
||||
@@ -479,6 +760,19 @@ public class VirtualTree
|
||||
public void remove(String path) throws IOException
|
||||
{
|
||||
Node node = stat(path);
|
||||
if("OUTBOX".equals(node.kind()) && outbox != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
outbox.remove(node.job());
|
||||
log(path, "REMOVE", "OK", null, System.nanoTime());
|
||||
return;
|
||||
}
|
||||
catch(com.kingsrook.qqq.backend.core.exceptions.QException e)
|
||||
{
|
||||
throw (new IOException("Unable to remove outbox file", e));
|
||||
}
|
||||
}
|
||||
if(node.job() == null || !List.of("SIDECAR", "RESULT").contains(node.kind()))
|
||||
{
|
||||
deny(path, "REMOVE", "Only sidecars and results can be acknowledged");
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
|
||||
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInProcessSessions;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
|
||||
import com.kingsrook.qqq.openapi.model.HttpMethod;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
import org.apache.sshd.sftp.client.SftpClient;
|
||||
import org.apache.sshd.sftp.client.SftpClientFactory;
|
||||
import org.apache.sshd.sftp.common.SftpConstants;
|
||||
import org.apache.sshd.sftp.common.SftpException;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Real SFTP downloads against strict user-authorized UI exports and report processes.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(NoDiskExtension.class)
|
||||
class DownloadIntegrationTest
|
||||
{
|
||||
private static final String VIEW = """
|
||||
{"queryFilter":{"criteria":[{"fieldName":"quantity","operator":"GREATER_THAN","values":[1]}],
|
||||
"orderBys":[{"fieldName":"quantity","isAscending":false}],"skip":30,"limit":1},
|
||||
"queryColumns":{"columns":[{"name":"__check__","isVisible":true},{"name":"quantity","isVisible":true},
|
||||
{"name":"name","isVisible":true},{"name":"id","isVisible":false}]}}
|
||||
""";
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Store the same JSON used by Next UI, including selection and hidden columns.
|
||||
*******************************************************************************/
|
||||
private QRecord view(FileGatewayTestServer h, String json) throws Exception
|
||||
{
|
||||
return (h.audit.system(() -> new InsertAction().execute(new InsertInput("savedView").withRecord(new QRecord()
|
||||
.withValue("tableName", "alpha").withValue("label", "Selected rows").withValue("userId", "A").withValue("viewJson", json))).getRecords().getFirst()));
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read until EOF regardless of the generated file's zero stat size.
|
||||
*******************************************************************************/
|
||||
private byte[] read(SftpClient client, String path) throws Exception
|
||||
{
|
||||
try(var input = client.read(path))
|
||||
{
|
||||
return (input.readAllBytes());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Filter, column order, ordering, three formats, hidden tables and transfer identity.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void viewFormatsPermissionsAndAudit() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
h.seedRows(3);
|
||||
QRecord saved = view(h, VIEW);
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
var account = h.account("alice", "A");
|
||||
h.audit.update("file_gateway_account", new QRecord().withValue("id", account.getValue("id")).withValue("formats", "csv,xlsx,json"));
|
||||
h.key(account, key);
|
||||
var bob = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(h.account("bob", "B"), bob);
|
||||
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session);
|
||||
var deniedSession = h.login("bob", bob); var denied = SftpClientFactory.instance().createSftpClient(deniedSession))
|
||||
{
|
||||
String path = "/testapp/alpha/views/selected-rows.";
|
||||
assertThat(client.stat(path + "csv").getSize()).isZero();
|
||||
String csv = new String(read(client, path + "csv"), StandardCharsets.UTF_8);
|
||||
assertThat(csv.replace("\r\n", "\n")).isEqualTo("\"Quantity\",\"Name\"\n\"3\",\"Row 3 " + "x".repeat(50) + "\"\n\"2\",\"Row 2 " + "x".repeat(50) + "\"\n");
|
||||
try(var workbook = new XSSFWorkbook(new java.io.ByteArrayInputStream(read(client, path + "xlsx"))))
|
||||
{
|
||||
assertThat(workbook.getSheetAt(0).getRow(0).getCell(0).getStringCellValue()).isEqualTo("Quantity");
|
||||
assertThat(workbook.getSheetAt(0).getRow(1).getCell(0).getNumericCellValue()).isEqualTo(3);
|
||||
}
|
||||
var json = new ObjectMapper().readTree(read(client, path + "json"));
|
||||
assertThat(json.toString()).contains("Row 3", "Row 2").doesNotContain("Row 1");
|
||||
assertThatThrownBy(() -> denied.stat(path + "csv")).isInstanceOfSatisfying(SftpException.class,
|
||||
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_NO_SUCH_FILE));
|
||||
assertThatThrownBy(() -> read(denied, path + "csv")).isInstanceOfSatisfying(SftpException.class,
|
||||
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_NO_SUCH_FILE));
|
||||
}
|
||||
var downloads = h.audit.query("file_gateway_transfer_log", new QQueryFilter()).stream()
|
||||
.filter(r -> "DOWNLOAD".equals(r.getValueString("operation"))).toList();
|
||||
assertThat(downloads).hasSize(3).allSatisfy(row ->
|
||||
{
|
||||
assertThat(row.getValueString("outcome")).isEqualTo("OK");
|
||||
assertThat(row.getValueString("objectKind")).isEqualTo("VIEW");
|
||||
assertThat(row.getValueString("objectId")).isEqualTo(saved.getValueString("id"));
|
||||
assertThat(row.getValueLong("bytes")).isPositive();
|
||||
assertThat(row.getValueLong("durationMs")).isNotNegative();
|
||||
assertThat(row.getValueString("format")).isIn("csv", "xlsx", "json");
|
||||
});
|
||||
assertThat(h.calls).allSatisfy(call ->
|
||||
{
|
||||
assertThat(call.channel()).startsWith("sftp/");
|
||||
assertThat(call.invokedBy()).isEqualTo("file-gateway");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** FieldSecurityLock DENY removes the protected value exactly as a direct user export.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void fieldSecurityLockMatchesDirectUserExport() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance ->
|
||||
{
|
||||
instance.setEnforcePermissionsInCoreActions(true);
|
||||
instance.addSecurityKeyType(new QSecurityKeyType().withName("privateField").withAllAccessKeyName("privateFieldAll"));
|
||||
instance.getTable("alpha").getField("name").setFieldSecurityLock(new FieldSecurityLock()
|
||||
.withSecurityKeyType("privateField").withDefaultBehavior(FieldSecurityLock.Behavior.DENY).withOverrideValues(List.of("reader")));
|
||||
}))
|
||||
{
|
||||
h.seedRows(3);
|
||||
view(h, "{\"queryFilter\":{}}");
|
||||
h.audit.system(() -> new InsertAction().execute(new InsertInput("savedView").withRecord(new QRecord()
|
||||
.withValue("tableName", "alpha").withValue("label", "Denied field").withValue("userId", "A").withValue("viewJson", VIEW))));
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(h.account("alice", "A"), key);
|
||||
var user = h.audit.system(() -> new QAuthenticationModuleDispatcher().getQModule(h.instance.getAuthentication()).createAutomatedSessionForUser(h.instance, "A"));
|
||||
String token = UiApiInProcessSessions.register(user, Duration.ofMinutes(5));
|
||||
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
|
||||
{
|
||||
var direct = h.runtime.getInvoker().invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/table/{tableName}/export")
|
||||
.withPathParam("tableName", "alpha").withCredentials(UiApiCredentials.ofInProcessSession(token)).withClientDetail("direct-comparison")
|
||||
.withJsonBody(Map.of("format", "csv", "filter", Map.of(), "includeHeaderRow", true)));
|
||||
assertThat(direct.isSuccess()).as(direct.getBody()).isTrue();
|
||||
byte[] actual = read(client, "/testapp/alpha/views/selected-rows.csv");
|
||||
assertThat(new String(actual, StandardCharsets.UTF_8)).doesNotContain("Row ");
|
||||
assertThat(actual).isEqualTo(direct.getBody().getBytes(StandardCharsets.UTF_8));
|
||||
assertThatThrownBy(() -> read(client, "/testapp/alpha/views/denied-field.csv")).isInstanceOf(SftpException.class);
|
||||
}
|
||||
finally
|
||||
{
|
||||
UiApiInProcessSessions.revoke(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The daily quota excludes prior UTC days, other accounts and non-download operations.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dailyQuotaUsesFilteredUtcHistory() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
h.seedRows(3);
|
||||
view(h, "{}");
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
var account = h.account("alice", "A");
|
||||
h.audit.update("file_gateway_account", new QRecord().withValue("id", account.getValue("id")).withValue("maxDownloadMbPerDay", 1));
|
||||
h.seedTransfer(account, "DOWNLOAD", 5_000_000L, Instant.now().minusSeconds(86400));
|
||||
h.seedTransfer(account, "UPLOAD", 5_000_000L, Instant.now());
|
||||
h.seedTransfer(h.account("other", "A"), "DOWNLOAD", 5_000_000L, Instant.now());
|
||||
h.key(account, key);
|
||||
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
|
||||
{
|
||||
assertThat(read(client, "/testapp/alpha/views/selected-rows.csv")).isNotEmpty();
|
||||
assertThatThrownBy(() -> read(client, "/testapp/alpha/views/selected-rows.json")).isInstanceOfSatisfying(SftpException.class,
|
||||
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_NO_SUCH_FILE));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Non-success stream responses preserve their text and distinguish denied transfers.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(ints = {403, 500})
|
||||
void apiErrorsAreAudited(int status) throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
view(h, "{}");
|
||||
h.exportFailureStatus = status;
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(h.account("alice", "A"), key);
|
||||
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
|
||||
{
|
||||
assertThatThrownBy(() -> read(client, "/testapp/alpha/views/selected-rows.csv"))
|
||||
.isInstanceOf(SftpException.class).hasMessageContaining("Export failure test");
|
||||
}
|
||||
assertThat(h.audit.query("file_gateway_transfer_log", new QQueryFilter())).anySatisfy(row ->
|
||||
{
|
||||
assertThat(row.getValueString("operation")).isEqualTo("DOWNLOAD");
|
||||
assertThat(row.getValueString("outcome")).isEqualTo(status == 403 ? "DENIED" : "ERROR");
|
||||
assertThat(row.getValueString("message")).contains("Export failure test");
|
||||
assertThat(row.getValueLong("bytes")).isZero();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Real renderSavedReport input/execute/output and storage download as the SSH user.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void savedReportDownload() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
h.seedRows(3);
|
||||
QRecord report = h.audit.system(() -> new InsertAction().execute(new InsertInput("savedReport").withRecord(new QRecord()
|
||||
.withValue("tableName", "alpha").withValue("label", "Daily").withValue("userId", "A")
|
||||
.withValue("columnsJson", "{\"columns\":[{\"name\":\"quantity\"}]}")
|
||||
.withValue("queryFilterJson", "{\"criteria\":[{\"fieldName\":\"quantity\",\"operator\":\"EQUALS\",\"values\":[2]}]}"))).getRecords().getFirst());
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(h.account("alice", "A"), key);
|
||||
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
|
||||
{
|
||||
String csv = new String(read(client, "/testapp/alpha/reports/daily.csv"), StandardCharsets.UTF_8);
|
||||
assertThat(csv.replace("\r\n", "\n")).isEqualTo("\"Quantity\"\n\"2\"\n");
|
||||
}
|
||||
assertThat(h.calls).anySatisfy(call -> assertThat(call.path()).contains("/processes/renderSavedReport/init"))
|
||||
.anySatisfy(call -> assertThat(call.path()).contains("/download/"))
|
||||
.allSatisfy(call -> assertThat(call.channel()).startsWith("sftp/"));
|
||||
assertThat(h.audit.query("file_gateway_transfer_log", new QQueryFilter())).anySatisfy(row ->
|
||||
{
|
||||
assertThat(row.getValueString("objectKind")).isEqualTo("REPORT");
|
||||
assertThat(row.getValueString("objectId")).isEqualTo(report.getValueString("id"));
|
||||
assertThat(row.getValueString("outcome")).isEqualTo("OK");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Missing configured storage hides the report folder, including guessed paths.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void missingReportStorageHidesFolder() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance -> instance.getTables().remove("reportStorage")))
|
||||
{
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(h.account("alice", "A"), key);
|
||||
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
|
||||
{
|
||||
List<String> names = new ArrayList<>();
|
||||
client.readDir("/testapp/alpha").forEach(entry -> names.add(entry.getFilename()));
|
||||
assertThat(names).doesNotContain("reports");
|
||||
assertThatThrownBy(() -> client.stat("/testapp/alpha/reports")).isInstanceOfSatisfying(SftpException.class,
|
||||
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_NO_SUCH_FILE));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Large exports are consumed with fixed buffers; caps fail during read and audit LIMIT.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"large", "generated", "daily"})
|
||||
void streamingAndLimits(String scenario) throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
h.seedRows(50000);
|
||||
view(h, "{\"fieldNames\":[\"name\"],\"queryFilter\":{\"orderBys\":[{\"fieldName\":\"id\",\"isAscending\":true}]}}");
|
||||
h.config.withDownloadReadAheadMb(1).withMaxGeneratedFileMb(scenario.equals("generated") ? 1 : 20);
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
var account = h.account("alice", "A");
|
||||
if(scenario.equals("daily"))
|
||||
{
|
||||
h.audit.update("file_gateway_account", new QRecord().withValue("id", account.getValue("id")).withValue("maxDownloadMbPerDay", 1));
|
||||
h.audit.insert("file_gateway_transfer_log", new QRecord().withValue("accountId", account.getValue("id"))
|
||||
.withValue("operation", "DOWNLOAD").withValue("bytes", 900000L).withValue("outcome", "OK")
|
||||
.withValue("path", "/earlier.csv").withValue("clientChannel", "sftp/earlier"));
|
||||
}
|
||||
h.key(account, key);
|
||||
try(var session = h.login("alice", key); var client = SftpClientFactory.instance().createSftpClient(session))
|
||||
{
|
||||
if(scenario.equals("large"))
|
||||
{
|
||||
try(var input = client.read("/testapp/alpha/views/selected-rows.csv"))
|
||||
{
|
||||
byte[] buffer = new byte[32768];
|
||||
long bytes = 0;
|
||||
int lines = 0;
|
||||
for(int count; (count = input.read(buffer)) >= 0; )
|
||||
{
|
||||
bytes += count;
|
||||
for(int i = 0; i < count; i++)
|
||||
{
|
||||
if(buffer[i] == '\n')
|
||||
{
|
||||
lines++;
|
||||
}
|
||||
}
|
||||
}
|
||||
assertThat(bytes).isGreaterThan(3_000_000);
|
||||
assertThat(lines).isEqualTo(50001);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
assertThatThrownBy(() -> read(client, "/testapp/alpha/views/selected-rows.csv")).isInstanceOf(java.io.IOException.class)
|
||||
.hasMessageContaining("limit");
|
||||
}
|
||||
}
|
||||
assertThat(h.audit.query("file_gateway_transfer_log", new QQueryFilter())).anySatisfy(row ->
|
||||
{
|
||||
assertThat(row.getValueString("objectKind")).isEqualTo("VIEW");
|
||||
assertThat(row.getValueString("outcome")).isEqualTo(scenario.equals("large") ? "OK" : "LIMIT");
|
||||
assertThat(row.getValueLong("bytes")).isPositive().isLessThanOrEqualTo(scenario.equals("large") ? 4_000_000L : 1_048_576L);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -40,7 +40,7 @@ class EntityContractTest
|
||||
** Verify annotations, metadata completeness and fluent accessors against QRecord conversion.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"Account", "SshKey", "Alias", "UploadJob", "OutboxFile", "TransferLog", "ConnectionLog"})
|
||||
@ValueSource(strings = {"Account", "SshKey", "Alias", "UploadJob", "OutboxFile", "TransferLog", "ConnectionLog", "Server"})
|
||||
void entityRoundTrip(String suffix) throws Exception
|
||||
{
|
||||
Class<?> type = Class.forName("com.kingsrook.qbits.filegateway.model.FileGateway" + suffix);
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2026. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Demo server for screenshots: the full test instance with the Next UI on
|
||||
** port 8099 (or the first argument), MOCK auth granting every permission to
|
||||
** browser sessions, and realistic gateway data. Test scope only.
|
||||
*******************************************************************************/
|
||||
public final class FileGatewayDemoServer
|
||||
{
|
||||
private static final List<String> FILES = List.of("orders-2026-10-13.csv", "inventory.xlsx", "returns.csv", "price-update.csv", "partners.csv", "shipments-week41.csv");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private FileGatewayDemoServer()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Start, seed and keep running until killed.
|
||||
*******************************************************************************/
|
||||
public static void main(String[] args) throws Exception
|
||||
{
|
||||
int port = args.length > 0 ? Integer.parseInt(args[0]) : 8099;
|
||||
FileGatewayTestServer.TestAuthentication.grantAllToUiSessions = true;
|
||||
FileGatewayTestServer server = start(port);
|
||||
System.out.println("File gateway demo: http://localhost:" + port + "/app/file_gateway");
|
||||
Thread.currentThread().join();
|
||||
server.close();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Start the server and seed demo data.
|
||||
*******************************************************************************/
|
||||
static FileGatewayTestServer start(int port) throws Exception
|
||||
{
|
||||
FileGatewayTestServer server = new FileGatewayTestServer(Clock.systemUTC(), instance ->
|
||||
{
|
||||
}, port, true, true);
|
||||
seed(server);
|
||||
return (server);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seed accounts, keys, endpoints, jobs, transfers, connections and outbox files.
|
||||
*******************************************************************************/
|
||||
static void seed(FileGatewayTestServer h) throws Exception
|
||||
{
|
||||
Random random = new Random(42);
|
||||
Instant now = Instant.now();
|
||||
|
||||
QRecord acme = h.audit.insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", "acme-orders").withValue("label", "Acme Corp (orders feed)")
|
||||
.withValue("accountType", "SERVICE").withValue("userId", "acme-svc").withValue("ownerUserId", "jamie").withValue("allowedCidrs", "203.0.113.0/24"));
|
||||
QRecord globex = h.audit.insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", "globex").withValue("label", "Globex Logistics")
|
||||
.withValue("accountType", "SERVICE").withValue("userId", "globex-svc").withValue("ownerUserId", "jamie"));
|
||||
QRecord jamie = h.audit.insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", "jamie").withValue("label", "Jamie Rivera")
|
||||
.withValue("accountType", "USER").withValue("userId", "jamie"));
|
||||
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", globex.getValue("id")).withValue("failedLoginCount", 5)
|
||||
.withValue("lockedUntil", now.plus(25, ChronoUnit.MINUTES)));
|
||||
List<QRecord> accounts = List.of(acme, globex, jamie);
|
||||
|
||||
for(QRecord account : accounts)
|
||||
{
|
||||
h.key(account, KeyUtils.generateKeyPair("ssh-ed25519", 256));
|
||||
}
|
||||
|
||||
QRecord inbox = h.audit.insert(FileGatewayTables.ALIAS, new QRecord().withValue("name", "acme-orders-in").withValue("path", "/acme/orders-in")
|
||||
.withValue("kind", "INBOX").withValue("consumerName", "bulk-load").withValue("accountId", acme.getValue("id")).withValue("description", "Acme drops nightly order files here"));
|
||||
QRecord outbox = h.audit.insert(FileGatewayTables.ALIAS, new QRecord().withValue("name", "globex-acks").withValue("path", "/globex/acks")
|
||||
.withValue("kind", "OUTBOX").withValue("accountId", globex.getValue("id")).withValue("description", "Acknowledgements Globex picks up"));
|
||||
h.audit.insert(FileGatewayTables.ALIAS, new QRecord().withValue("name", "alpha-daily").withValue("path", "/reports/alpha-daily")
|
||||
.withValue("kind", "VIEW").withValue("tableName", "alpha").withValue("format", "csv").withValue("fileName", "alpha-daily.csv"));
|
||||
|
||||
List<String> statuses = List.of("SUCCEEDED", "SUCCEEDED", "SUCCEEDED", "SUCCEEDED", "PARTIAL", "FAILED", "REJECTED", "DUPLICATE", "QUEUED", "RUNNING");
|
||||
for(int i = 0; i < 40; i++)
|
||||
{
|
||||
QRecord account = accounts.get(i % accounts.size());
|
||||
String status = statuses.get(random.nextInt(statuses.size()));
|
||||
String file = FILES.get(random.nextInt(FILES.size()));
|
||||
Instant finished = now.minus(random.nextInt(7 * 24 * 60), ChronoUnit.MINUTES);
|
||||
int inserted = "FAILED".equals(status) || "REJECTED".equals(status) ? 0 : 50 + random.nextInt(900);
|
||||
int errors = "PARTIAL".equals(status) ? 1 + random.nextInt(12) : 0;
|
||||
h.audit.insert(FileGatewayTables.UPLOAD_JOB, new QRecord().withValue("accountId", account.getValue("id"))
|
||||
.withValue("aliasId", account == acme ? inbox.getValue("id") : null).withValue("tableName", "alpha").withValue("consumerName", "bulk-load")
|
||||
.withValue("path", "/testapp/alpha/upload/orders/" + file).withValue("fileName", file).withValue("sizeBytes", 20_000L + random.nextInt(4_000_000))
|
||||
.withValue("sha256", Long.toHexString(random.nextLong()) + Long.toHexString(random.nextLong())).withValue("status", status)
|
||||
.withValue("insertedCount", inserted).withValue("errorCount", errors).withValue("remoteAddress", "203.0.113." + (10 + random.nextInt(40)))
|
||||
.withValue("errorMessage", "FAILED".equals(status) ? "this profile needs value mappings saved in it" : "REJECTED".equals(status) ? "Virus scan: infected" : null)
|
||||
.withValue("queuedAt", finished.minusSeconds(40)).withValue("startedAt", finished.minusSeconds(30))
|
||||
.withValue("finishedAt", List.of("QUEUED", "RUNNING").contains(status) ? null : finished).withValue("attempts", 1));
|
||||
}
|
||||
|
||||
for(int i = 0; i < 60; i++)
|
||||
{
|
||||
QRecord account = accounts.get(random.nextInt(accounts.size()));
|
||||
String op = List.of("LIST", "UPLOAD", "DOWNLOAD", "LIST", "DOWNLOAD").get(random.nextInt(5));
|
||||
h.seedTransfer(account, op, "LIST".equals(op) ? 0 : 10_000L + random.nextInt(8_000_000), now.minus(random.nextInt(6 * 24 * 60), ChronoUnit.MINUTES));
|
||||
}
|
||||
|
||||
for(int i = 0; i < 12; i++)
|
||||
{
|
||||
QRecord account = accounts.get(i % accounts.size());
|
||||
boolean live = i < 2;
|
||||
Instant start = now.minus(live ? 5 + i * 7 : 60 + random.nextInt(5000), ChronoUnit.MINUTES);
|
||||
h.audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("accountId", account.getValue("id")).withValue("username", account.getValueString("username"))
|
||||
.withValue("remoteIp", "203.0.113." + (10 + i)).withValue("clientVersion", "SSH-2.0-OpenSSH_9.8").withValue("outcome", i == 5 ? "AUTH_FAILED" : "CONNECTED")
|
||||
.withValue("sessionId", "demo-" + i).withValue("podName", "file-gateway-" + (i % 2 == 0 ? "7f9c4-abcde" : "7f9c4-fghij")).withValue("connectedAt", start)
|
||||
.withValue("lastSeenAt", live ? now : start.plus(4, ChronoUnit.MINUTES)).withValue("disconnectedAt", live ? null : start.plus(4, ChronoUnit.MINUTES))
|
||||
.withValue("bytesIn", (long) random.nextInt(9_000_000)).withValue("bytesOut", (long) random.nextInt(2_000_000)).withValue("operationCount", (long) (3 + random.nextInt(40)))
|
||||
.withValue("clientChannel", "sftp"));
|
||||
}
|
||||
|
||||
for(String name : List.of("ack-1041.csv", "ack-1042.csv", "ack-1043.csv"))
|
||||
{
|
||||
h.store.put("file-gateway/outbox/" + outbox.getValue("id") + "/" + name, new java.io.ByteArrayInputStream("order,status\n1041,received\n".getBytes(java.nio.charset.StandardCharsets.UTF_8)));
|
||||
h.audit.insert(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("aliasId", outbox.getValue("id")).withValue("fileName", name).withValue("sizeBytes", 26L)
|
||||
.withValue("storageKey", "file-gateway/outbox/" + outbox.getValue("id") + "/" + name).withValue("status", name.endsWith("3.csv") ? "READY" : "DOWNLOADED"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2026. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The demo server seeds and serves (smoke test; the demo itself is manual).
|
||||
*******************************************************************************/
|
||||
class FileGatewayDemoServerTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Reset the demo-only grant.
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void reset()
|
||||
{
|
||||
FileGatewayTestServer.TestAuthentication.grantAllToUiSessions = false;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Start on a free port, check seeded data, stop.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void startsAndSeeds() throws Exception
|
||||
{
|
||||
try(FileGatewayTestServer server = FileGatewayDemoServer.start(0))
|
||||
{
|
||||
assertThat(server.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).hasSize(3);
|
||||
assertThat(server.audit.query(FileGatewayTables.UPLOAD_JOB, new QQueryFilter())).hasSize(40);
|
||||
assertThat(server.audit.query(FileGatewayTables.OUTBOX_FILE, new QQueryFilter())).hasSize(3);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,8 @@ package com.kingsrook.qbits.filegateway;
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -52,8 +54,8 @@ class FileGatewayQBitProducerTest
|
||||
assertThat(instance.getApp("file_gateway").getSections().stream().flatMap(s -> s.getTables().stream()).toList())
|
||||
.containsExactlyElementsOf(FileGatewayTables.MAIN);
|
||||
assertThat(instance.getJoins()).hasSize(6);
|
||||
assertThat(instance.getWidgets()).hasSize(6);
|
||||
assertThat(instance.getPossibleValueSources()).hasSize(19);
|
||||
assertThat(instance.getWidgets()).hasSize(16);
|
||||
assertThat(instance.getPossibleValueSources()).hasSize(20);
|
||||
instance.getTables().values().forEach(t ->
|
||||
{
|
||||
assertThat(t.getBackendName()).isEqualTo("host-db");
|
||||
@@ -73,6 +75,21 @@ class FileGatewayQBitProducerTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The three gateway permission names reach QQQ's catalog, so identity
|
||||
** policies can grant them.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void permissionsAreInTheAvailableCatalog() throws Exception
|
||||
{
|
||||
var instance = new QInstance();
|
||||
var config = new FileGatewayQBitConfig().withBackendName("host-db");
|
||||
new FileGatewayQBitProducer().withQBitConfig(config).withNamespace("test").produce(instance).addSelfToInstance(instance);
|
||||
assertThat(PermissionsHelper.getAllAvailablePermissionNames(instance)).containsAll(FileGatewayPermissions.ALL);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Package scanning never enables an unconfigured producer.
|
||||
*******************************************************************************/
|
||||
@@ -100,6 +117,8 @@ class FileGatewayQBitProducerTest
|
||||
void contingentPickerMetadata(String source) throws Exception
|
||||
{
|
||||
var instance = new QInstance();
|
||||
instance.addPossibleValueSource(new com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource().withName(source)
|
||||
.withType(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType.TABLE).withTableName(source));
|
||||
new FileGatewayQBitProducer().withQBitConfig(new FileGatewayQBitConfig()).produce(instance);
|
||||
var field = instance.getTable(FileGatewayTables.ALIAS).getField(source + "Id");
|
||||
assertThat(field.getPossibleValueSourceName()).isEqualTo(source);
|
||||
@@ -112,4 +131,27 @@ class FileGatewayQBitProducerTest
|
||||
});
|
||||
assertThat(filter.getSubFilters()).isNullOrEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A host without saved views, reports or bulk-load profiles still validates:
|
||||
** those pickers become plain id fields; present sources keep their pickers.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dropsPickersForMissingHostSources() throws Exception
|
||||
{
|
||||
QInstance instance = new QInstance();
|
||||
com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData alias = FileGatewayTables.tables(new FileGatewayQBitConfig()).stream()
|
||||
.filter(t -> t.getName().equals(FileGatewayTables.ALIAS)).findFirst().orElseThrow();
|
||||
instance.addPossibleValueSource(new com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource().withName("savedView")
|
||||
.withType(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType.TABLE).withTableName("savedView"));
|
||||
FileGatewayQBitProducer.dropMissingPickers(instance, alias);
|
||||
assertThat(alias.getField("savedViewId").getPossibleValueSourceName()).isEqualTo("savedView");
|
||||
assertThat(alias.getField("savedViewId").getPossibleValueSourceFilter()).isNotNull();
|
||||
assertThat(alias.getField("savedReportId").getPossibleValueSourceName()).isNull();
|
||||
assertThat(alias.getField("savedReportId").getPossibleValueSourceFilter()).isNull();
|
||||
assertThat(alias.getField("savedBulkLoadProfileId").getPossibleValueSourceName()).isNull();
|
||||
assertThat(alias.getField("tableName").getPossibleValueSourceName()).isEqualTo("tables");
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,7 @@ public class FileGatewayTestServer implements AutoCloseable
|
||||
public final FileGatewayRuntime runtime;
|
||||
public final SshClient client;
|
||||
public final java.util.List<ApiCall> calls = new java.util.concurrent.CopyOnWriteArrayList<>();
|
||||
public volatile int exportFailureStatus;
|
||||
|
||||
/*******************************************************************************
|
||||
** Non-secret request evidence for the invoker contract assertions.
|
||||
@@ -112,6 +113,29 @@ public class FileGatewayTestServer implements AutoCloseable
|
||||
*******************************************************************************/
|
||||
public FileGatewayTestServer(Clock clock) throws Exception
|
||||
{
|
||||
this(clock, instance ->
|
||||
{
|
||||
// Most acceptance tests use the default application metadata.
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Customize application metadata before validation and server startup.
|
||||
*******************************************************************************/
|
||||
public FileGatewayTestServer(Clock clock, java.util.function.Consumer<QInstance> customize) throws Exception
|
||||
{
|
||||
this(clock, customize, 0, false, true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Demo and web-only tests can serve Next without starting an SFTP listener.
|
||||
*******************************************************************************/
|
||||
public FileGatewayTestServer(Clock clock, java.util.function.Consumer<QInstance> customize, int port, boolean frontend, boolean sftp) throws Exception
|
||||
{
|
||||
config.withSftpEnabled(sftp);
|
||||
ConnectionManager.resetConnectionProviders();
|
||||
GRANTS.clear();
|
||||
GRANTS.put("A", Set.of("file_gateway.connect", "alpha.read", "alpha.insert", "beta.read", "beta.insert"));
|
||||
@@ -167,9 +191,11 @@ public class FileGatewayTestServer implements AutoCloseable
|
||||
}
|
||||
instance.addBackend(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData()
|
||||
.withName("memory").withBackendType(com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule.class));
|
||||
instance.getTable("reportStorage").setBackendName("memory");
|
||||
instance.addTable(new QTableMetaData().withName("testUploadArchive").withBackendName("memory").withPrimaryKeyField("reference")
|
||||
.withField(new QFieldMetaData("reference", QFieldType.STRING)).withField(new QFieldMetaData("contents", QFieldType.BLOB)));
|
||||
instance.withSupplementalMetaData(new com.kingsrook.qqq.middleware.javalin.QJavalinMetaData().withUploadedFileArchiveTableName("testUploadArchive"));
|
||||
customize.accept(instance);
|
||||
new QInstanceEnricher(instance).enrich();
|
||||
new QInstanceValidator().validate(instance);
|
||||
QContext.init(instance, new QSystemUserSession(instance));
|
||||
@@ -183,17 +209,68 @@ public class FileGatewayTestServer implements AutoCloseable
|
||||
{
|
||||
return (instance);
|
||||
}
|
||||
}).withPort(0).withServeFrontendMaterialDashboard(false).withServeFrontendNext(false)
|
||||
.withJavalinConfigCustomizer(j -> j.routes.before(ctx -> calls.add(new ApiCall(ctx.method().name(), ctx.path(), ctx.header("X-QQQ-Client"), ctx.header("X-QQQ-Invoked-By")))))
|
||||
}).withPort(port).withServeFrontendMaterialDashboard(false).withServeFrontendNext(frontend)
|
||||
.withJavalinConfigCustomizer(j -> j.routes.before(ctx ->
|
||||
{
|
||||
calls.add(new ApiCall(ctx.method().name(), ctx.path(), ctx.header("X-QQQ-Client"), ctx.header("X-QQQ-Invoked-By")));
|
||||
if(exportFailureStatus != 0 && ctx.path().endsWith("/export"))
|
||||
{
|
||||
throw (new io.javalin.http.HttpResponseException(exportFailureStatus, "Export failure test"));
|
||||
}
|
||||
}))
|
||||
.withStrictUiApiRoutes(true).withMiddlewareVersionList(List.of(new MiddlewareVersionV1()));
|
||||
http.start();
|
||||
runtime = FileGatewayRuntime.startForTest(instance, http, config, KeyUtils.generateKeyPair("ecdsa-sha2-nistp256", 256), clock, Duration.ofHours(1), store);
|
||||
runtime = sftp ? FileGatewayRuntime.startForTest(instance, http, config, KeyUtils.generateKeyPair("ecdsa-sha2-nistp256", 256), clock, Duration.ofHours(1), store) : null;
|
||||
if(!sftp)
|
||||
{
|
||||
com.kingsrook.qbits.filegateway.api.FileGatewayOutbox.setProvider(new com.kingsrook.qbits.filegateway.outbox.OutboxService(instance, store, config, clock));
|
||||
}
|
||||
client = SshClient.setUpDefaultClient();
|
||||
client.setServerKeyVerifier(AcceptAllServerKeyVerifier.INSTANCE);
|
||||
client.start();
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seed many export rows without generating unrelated insert audits.
|
||||
*******************************************************************************/
|
||||
public void seedRows(int count) throws Exception
|
||||
{
|
||||
try(var statement = keeper.prepareStatement("INSERT INTO alpha (id, name, quantity) VALUES (?, ?, ?)"))
|
||||
{
|
||||
for(int i = 1; i <= count; i++)
|
||||
{
|
||||
statement.setInt(1, i);
|
||||
statement.setString(2, "Row " + i + " " + "x".repeat(50));
|
||||
statement.setInt(3, i);
|
||||
statement.addBatch();
|
||||
if(i % 1000 == 0)
|
||||
{
|
||||
statement.executeBatch();
|
||||
}
|
||||
}
|
||||
statement.executeBatch();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Seed quota history with an exact UTC timestamp rather than the insertion clock.
|
||||
*******************************************************************************/
|
||||
public void seedTransfer(QRecord account, String operation, long bytes, java.time.Instant when) throws Exception
|
||||
{
|
||||
QRecord row = audit.insert("file_gateway_transfer_log", new QRecord().withValue("accountId", account.getValue("id"))
|
||||
.withValue("operation", operation).withValue("bytes", bytes).withValue("outcome", "OK")
|
||||
.withValue("path", "/history.csv").withValue("clientChannel", "sftp/history"));
|
||||
try(var statement = keeper.prepareStatement("UPDATE file_gateway_transfer_log SET create_date = ? WHERE id = ?"))
|
||||
{
|
||||
statement.setObject(1, when.atOffset(java.time.ZoneOffset.UTC).toLocalDateTime());
|
||||
statement.setInt(2, row.getValueInteger("id"));
|
||||
statement.executeUpdate();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create core test tables using their actual metadata.
|
||||
@@ -293,7 +370,10 @@ public class FileGatewayTestServer implements AutoCloseable
|
||||
public void close() throws Exception
|
||||
{
|
||||
client.stop();
|
||||
runtime.stop();
|
||||
if(runtime != null)
|
||||
{
|
||||
runtime.stop();
|
||||
}
|
||||
http.stop();
|
||||
keeper.close();
|
||||
QContext.clear();
|
||||
@@ -306,6 +386,27 @@ public class FileGatewayTestServer implements AutoCloseable
|
||||
*******************************************************************************/
|
||||
public static class TestAuthentication implements QAuthenticationModuleCustomizerInterface
|
||||
{
|
||||
/** Demo only: browser (MOCK) sessions get every permission in the instance. */
|
||||
public static volatile boolean grantAllToUiSessions = false;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Browser sessions: nothing, unless the demo server switches it on.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void customizeSession(QInstance instance, QSession session, Map<String, Object> context)
|
||||
{
|
||||
if(grantAllToUiSessions)
|
||||
{
|
||||
java.util.Set<String> all = new java.util.HashSet<>(com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper.getAllAvailablePermissionNames(instance));
|
||||
all.addAll(com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions.ALL);
|
||||
session.setPermissions(all);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Customize automated session for user.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
|
||||
import java.security.KeyPair;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.preferences.FileTransferPreferencesProvider;
|
||||
import com.kingsrook.qqq.backend.core.actions.preferences.UserPreferencesAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferencesMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInProcessSessions;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInvoker;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
|
||||
import com.kingsrook.qqq.openapi.model.HttpMethod;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
import org.apache.sshd.common.config.keys.PublicKeyEntry;
|
||||
import org.apache.sshd.common.digest.BuiltinDigests;
|
||||
import org.apache.sshd.sftp.client.SftpClientFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Self-service acceptance through core, strict HTTP routes and real SSH clients.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(NoDiskExtension.class)
|
||||
class FileTransferPreferencesTest
|
||||
{
|
||||
private static final String SECTION = FileTransferPreferencesProvider.SECTION_NAME;
|
||||
private final UserPreferencesAction preferences = new UserPreferencesAction();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Connect permission alone reveals the section, independently of the admin app.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void visibilityAndEmptyAccount() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, "A");
|
||||
assertThat(preferences.listSections()).hasSize(1);
|
||||
var section = preferences.getSection(SECTION);
|
||||
assertThat(section.getSection().getPermissionName()).isEqualTo("file_gateway.connect");
|
||||
assertThat(section.getSection().getAppName()).isNull();
|
||||
assertThat(section.getSection().getOrder()).isEqualTo(60);
|
||||
assertThat(section.getValues().getValues().get("username")).isEqualTo("");
|
||||
assertThat(section.getValues().getLists().get("keys")).isEmpty();
|
||||
assertThat(section.getSection().getLists().getFirst().getDescription()).contains("first key creates");
|
||||
assertThat(section.getSection().getFields()).noneMatch(field -> "host".equals(field.getName()));
|
||||
assertThat(section.getValues().getValues().get("port")).isEqualTo(h.runtime.getProtocol().getPort());
|
||||
assertThat(section.getValues().getValues().get("hostKeyFingerprint")).isNotNull();
|
||||
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).isEmpty();
|
||||
user(h, "NONE");
|
||||
assertThat(preferences.listSections()).isEmpty();
|
||||
assertThatThrownBy(() -> preferences.getSection(SECTION)).isInstanceOf(QNotFoundException.class);
|
||||
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of())).isInstanceOf(QNotFoundException.class);
|
||||
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of())).isInstanceOf(QNotFoundException.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Host is optional; heartbeat data comes from the most recent server.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void publicHostAndLatestHeartbeat() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance -> { }, 0, false, false))
|
||||
{
|
||||
assertThat(h.config.getPublicHostName()).isNull();
|
||||
UserPreferencesMetaData.of(h.instance).getSections().clear();
|
||||
FileTransferPreferencesProvider.register(h.instance, h.config.withPublicHostName("sftp.example.test"));
|
||||
user(h, "A");
|
||||
assertThat(preferences.getSection(SECTION).getValues().getValues()).containsEntry("host", "sftp.example.test").containsEntry("port", null);
|
||||
h.audit.insert(FileGatewayTables.SERVER, new QRecord().withValue("podName", "new").withValue("port", 2223)
|
||||
.withValue("lastSeenAt", Instant.now()).withValue("hostKeyFingerprint", "SHA256:new"));
|
||||
h.audit.insert(FileGatewayTables.SERVER, new QRecord().withValue("podName", "old").withValue("port", 2222)
|
||||
.withValue("lastSeenAt", Instant.now().minusSeconds(60)).withValue("hostKeyFingerprint", "SHA256:old"));
|
||||
assertThat(preferences.getSection(SECTION).getValues().getValues()).containsEntry("port", 2223).containsEntry("hostKeyFingerprint", "SHA256:new");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Creation uses schema defaults, reuses the account and preserves audit identity.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void createsAndReusesAccountWithAudits() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance ->
|
||||
{
|
||||
for(String table : List.of(FileGatewayTables.ACCOUNT, FileGatewayTables.SSH_KEY))
|
||||
{
|
||||
instance.getTable(table).withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD));
|
||||
}
|
||||
instance.getTable(FileGatewayTables.ACCOUNT).getField("maxFileSizeMb").setDefaultValue(321);
|
||||
instance.setEnforcePermissionsInCoreActions(true);
|
||||
}))
|
||||
{
|
||||
user(h, "A");
|
||||
var key = pair();
|
||||
var first = add(key, "Laptop");
|
||||
assertThat(first.getMessage()).contains(KeyUtils.getFingerPrint(BuiltinDigests.sha256, key.getPublic()));
|
||||
assertThat(first.getSection().getValues().getValues().get("username")).isEqualTo("user-a");
|
||||
assertThat(first.getSection().getValues().getLists().get("keys").getFirst().getValues()).containsEntry("keyType", "ssh-ed25519").doesNotContainKey("publicKey");
|
||||
add(pair(), "Backup");
|
||||
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).singleElement().satisfies(account ->
|
||||
{
|
||||
assertThat(account.getValueString("userId")).isEqualTo("A");
|
||||
assertThat(account.getValueString("accountType")).isEqualTo("USER");
|
||||
assertThat(account.getValueString("rootMode")).isEqualTo("FULL_TREE");
|
||||
assertThat(account.getValueString("status")).isEqualTo("ACTIVE");
|
||||
assertThat(account.getValueInteger("maxFileSizeMb")).isEqualTo(321);
|
||||
assertThat(account.getValueInteger("maxConcurrentSessions")).isEqualTo(4);
|
||||
});
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).hasSize(2);
|
||||
var audits = h.audit.system(() -> new QueryAction().execute(new QueryInput("audit")).getRecords());
|
||||
var auditTables = h.audit.system(() -> new QueryAction().execute(new QueryInput("auditTable")).getRecords());
|
||||
var auditUsers = h.audit.system(() -> new QueryAction().execute(new QueryInput("auditUser")).getRecords());
|
||||
Object tableId = auditTables.stream().filter(row -> FileGatewayTables.SSH_KEY.equals(row.getValueString("name"))).findFirst().orElseThrow().getValue("id");
|
||||
Object userId = auditUsers.stream().filter(row -> "Test user A".equals(row.getValueString("name"))).findFirst().orElseThrow().getValue("id");
|
||||
assertThat(audits).anySatisfy(audit ->
|
||||
{
|
||||
assertThat(audit.getValue("auditTableId")).isEqualTo(tableId);
|
||||
assertThat(audit.getValue("auditUserId")).isEqualTo(userId);
|
||||
assertThat(audit.getValueString("clientChannel")).isEqualTo("next-ui/preferences-test");
|
||||
});
|
||||
assertThat(QContext.getQSession().getUser().getIdReference()).isEqualTo("A");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalid first keys roll back the account; duplicates retain table messages.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void rejectsWeakDuplicateAndMissingKeys() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, "A");
|
||||
assertThatThrownBy(() -> add(KeyUtils.generateKeyPair("ssh-rsa", 2048), "Weak"))
|
||||
.isInstanceOf(QBadRequestException.class).hasMessageContaining("at least 3072 bits");
|
||||
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).isEmpty();
|
||||
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of()))
|
||||
.isInstanceOf(QBadRequestException.class).hasMessageContaining("Public key: is required").hasMessageContaining("Label: is required");
|
||||
var key = pair();
|
||||
add(key, "First");
|
||||
assertThatThrownBy(() -> add(key, "Duplicate")).isInstanceOf(QBadRequestException.class).hasMessageContaining("Fingerprint");
|
||||
user(h, "B");
|
||||
assertThatThrownBy(() -> add(key, "Other user duplicate")).isInstanceOf(QBadRequestException.class).hasMessageContaining("Fingerprint");
|
||||
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).hasSize(1);
|
||||
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter())).hasSize(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Normalization, truncation and suffixes obey the account validator.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"Alice@EXAMPLE.COM", "A", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"})
|
||||
void uniquifiesUsername(String id) throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, id);
|
||||
String base = id.toLowerCase(java.util.Locale.ROOT).replaceAll("[^a-z0-9._-]", "-");
|
||||
base = base.length() < 3 ? "user-" + base : base.substring(0, Math.min(64, base.length()));
|
||||
h.account(base, "B");
|
||||
String second = base.substring(0, Math.min(62, base.length())) + "-2";
|
||||
h.account(second, "B");
|
||||
var added = add(pair(), "Key");
|
||||
assertThat(added.getSection().getValues().getValues().get("username")).isEqualTo(base.substring(0, Math.min(62, base.length())) + "-3");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Lists and mutations exclude other users, revoked keys and service accounts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void isolatesKeysAndRejectsIdentityInjection() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
var other = h.key(h.account("bob", "B"), pair());
|
||||
var account = h.account("alice", "A");
|
||||
var revoked = h.key(account, pair());
|
||||
h.audit.update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", revoked.getValue("id")).withValue("revokedAt", Instant.now()));
|
||||
var service = h.account("service", "A");
|
||||
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", service.getValue("id")).withValue("accountType", "SERVICE"));
|
||||
var serviceKey = h.key(service, pair());
|
||||
user(h, "A");
|
||||
var own = add(pair(), "Own");
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).extracting(row -> row.getKey()).containsExactly(own.getFocusRowKey());
|
||||
for(QRecord excluded : List.of(other, revoked, serviceKey))
|
||||
{
|
||||
assertThatThrownBy(() -> preferences.runAction(SECTION, "revokeKey", Map.of("rowKey", excluded.getValueString("id"))))
|
||||
.isInstanceOf(QBadRequestException.class);
|
||||
assertThatThrownBy(() -> patch(excluded.getValueString("id"), Map.of("label", "Stolen")))
|
||||
.isInstanceOf(QBadRequestException.class).hasMessageContaining("isn't a setting here");
|
||||
}
|
||||
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of("userId", "B"))).isInstanceOf(QBadRequestException.class);
|
||||
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of("userId", "B"))).isInstanceOf(QBadRequestException.class);
|
||||
assertThatThrownBy(() -> patch(own.getFocusRowKey(), Map.of("accountId", other.getValue("accountId"))))
|
||||
.isInstanceOf(QBadRequestException.class);
|
||||
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter())).noneMatch(row -> "Stolen".equals(row.getValueString("label")));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PATCH changes only label/expiry, supports clearing expiry and rejects readonly fields.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void editsOnlyMutableFields() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, "A");
|
||||
String key = add(pair(), "Laptop").getFocusRowKey();
|
||||
Instant expires = Instant.parse("2030-01-01T00:00:00Z");
|
||||
patch(key, Map.of("label", "Work laptop", "expiresAt", expires.toString()));
|
||||
var values = preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues();
|
||||
assertThat(values).containsEntry("label", "Work laptop").containsEntry("expiresAt", expires);
|
||||
Map<String, Object> clear = new HashMap<>();
|
||||
clear.put("expiresAt", null);
|
||||
patch(key, clear);
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues()).containsEntry("expiresAt", null);
|
||||
for(String field : List.of("fingerprint", "keyType", "lastUsedAt"))
|
||||
{
|
||||
assertThatThrownBy(() -> patch(key, Map.of(field, "2030-01-01T00:00:00Z"))).isInstanceOf(QBadRequestException.class).hasMessageContaining("can't be changed");
|
||||
}
|
||||
for(String field : List.of("username", "port", "hostKeyFingerprint"))
|
||||
{
|
||||
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of("values", Map.of(field, "22"))))
|
||||
.isInstanceOf(QBadRequestException.class).hasMessageContaining("can't be changed");
|
||||
}
|
||||
assertThatThrownBy(() -> patch(key, Map.of("label", " "))).isInstanceOf(QBadRequestException.class).hasMessageContaining("Label: is required");
|
||||
assertThatThrownBy(() -> patch(key, Map.of("publicKey", "replacement"))).isInstanceOf(QBadRequestException.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A self-service key signs in and lists SFTP; revocation closes and refuses it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void addedKeyLogsInAndRevokedKeyCannot() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, "A");
|
||||
var pair = pair();
|
||||
var added = add(pair, "Laptop");
|
||||
String username = added.getSection().getValues().getValues().get("username").toString();
|
||||
try(var session = h.login(username, pair); var client = SftpClientFactory.instance().createSftpClient(session))
|
||||
{
|
||||
assertThat(client.readDir("/")).anyMatch(entry -> "testapp".equals(entry.getFilename()));
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues().get("lastUsedAt")).isNotNull();
|
||||
assertThat(preferences.runAction(SECTION, "revokeKey", Map.of("rowKey", added.getFocusRowKey())).getMessage()).isEqualTo("Key revoked.");
|
||||
h.runtime.getProtocol().getAuthenticator().poll();
|
||||
session.waitFor(Set.of(org.apache.sshd.client.session.ClientSession.ClientSessionEvent.CLOSED), Duration.ofSeconds(5));
|
||||
assertThat(session.isClosed()).isTrue();
|
||||
}
|
||||
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter()).getFirst().getValue("revokedAt")).isNotNull();
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).isEmpty();
|
||||
assertThatThrownBy(() -> h.login(username, pair)).isInstanceOf(Exception.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The strict UI route exposes actions and translates domain failures to 400.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void preferencesThroughStrictUiApi() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
QSession session = user(h, "A");
|
||||
String token = UiApiInProcessSessions.register(session, Duration.ofMinutes(1));
|
||||
try
|
||||
{
|
||||
var invoker = UiApiInvoker.forServer(h.http, "next-ui");
|
||||
var get = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath("/preferences")
|
||||
.withCredentials(UiApiCredentials.ofInProcessSession(token)));
|
||||
assertThat(get.getStatus()).isEqualTo(200);
|
||||
assertThat(get.getJson().toString()).contains("file-transfer", "addKey", "revokeKey");
|
||||
var added = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/preferences/{sectionName}/actions/{actionName}")
|
||||
.withPathParam("sectionName", SECTION).withPathParam("actionName", "addKey").withCredentials(UiApiCredentials.ofInProcessSession(token))
|
||||
.withJsonBody(Map.of("values", Map.of("publicKey", PublicKeyEntry.toString(pair().getPublic()), "label", "HTTP key"))));
|
||||
assertThat(added.getStatus()).as(added.getBody()).isEqualTo(200);
|
||||
assertThat(added.getJson().toString()).contains("SHA256:", "HTTP key");
|
||||
var invalid = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/preferences/{sectionName}/actions/{actionName}")
|
||||
.withPathParam("sectionName", SECTION).withPathParam("actionName", "addKey").withCredentials(UiApiCredentials.ofInProcessSession(token))
|
||||
.withJsonBody(Map.of("values", Map.of("publicKey", "invalid", "label", "Invalid"))));
|
||||
assertThat(invalid.getStatus()).isEqualTo(400);
|
||||
assertThat(invalid.getBody()).contains("at least 3072 bits");
|
||||
}
|
||||
finally
|
||||
{
|
||||
UiApiInProcessSessions.revoke(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set a real non-system session with only the user's normal permissions.
|
||||
*******************************************************************************/
|
||||
private QSession user(FileGatewayTestServer h, String id) throws Exception
|
||||
{
|
||||
QSession session = h.audit.system(() -> new QAuthenticationModuleDispatcher().getQModule(h.instance.getAuthentication()).createAutomatedSessionForUser(h.instance, id));
|
||||
session.setPermissions(FileGatewayTestServer.GRANTS.getOrDefault(id, Set.of("file_gateway.connect")));
|
||||
session.setClientChannel("next-ui/preferences-test");
|
||||
QContext.init(h.instance, session);
|
||||
return (session);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generate a supported public key entirely in memory.
|
||||
*******************************************************************************/
|
||||
private KeyPair pair() throws Exception
|
||||
{
|
||||
return (KeyUtils.generateKeyPair("ssh-ed25519", 256));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise registered action input validation rather than calling the provider.
|
||||
*******************************************************************************/
|
||||
private UserPreferenceActionResult add(KeyPair key, String label) throws Exception
|
||||
{
|
||||
return (preferences.runAction(SECTION, "addKey", Map.of("values", Map.of("publicKey", PublicKeyEntry.toString(key.getPublic()), "label", label))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Send the same partial row body as a preferences client.
|
||||
*******************************************************************************/
|
||||
private void patch(String key, Map<String, ?> values) throws Exception
|
||||
{
|
||||
preferences.patchSection(SECTION, Map.of("lists", Map.of("keys", List.of(Map.of("key", key, "values", values)))));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.channels.ClosedChannelException;
|
||||
import java.nio.channels.NonWritableChannelException;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import com.kingsrook.qbits.filegateway.storage.GeneratedReadChannel;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fixed-window stream semantics independently of network packet boundaries and file size.
|
||||
*******************************************************************************/
|
||||
class GeneratedReadChannelTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Capture charges and finalization without involving a database.
|
||||
*******************************************************************************/
|
||||
private GeneratedReadChannel.Audit audit(AtomicInteger bytes, AtomicReference<IOException> failure, AtomicInteger closes)
|
||||
{
|
||||
return (new GeneratedReadChannel.Audit()
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Track each newly generated byte once, including forward read-ahead.
|
||||
*******************************************************************************/
|
||||
public void charge(int count)
|
||||
{
|
||||
bytes.addAndGet(count);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Capture final outcome.
|
||||
*******************************************************************************/
|
||||
public void finish(IOException error)
|
||||
{
|
||||
failure.set(error);
|
||||
closes.incrementAndGet();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Forward gaps and backward reads inside the window reuse bytes without regeneration.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reorderedReadsAndEviction() throws Exception
|
||||
{
|
||||
byte[] source = new byte[80];
|
||||
for(int i = 0; i < source.length; i++)
|
||||
{
|
||||
source[i] = (byte) i;
|
||||
}
|
||||
AtomicInteger bytes = new AtomicInteger();
|
||||
AtomicInteger closes = new AtomicInteger();
|
||||
AtomicReference<IOException> failure = new AtomicReference<>();
|
||||
try(var channel = new GeneratedReadChannel(new ByteArrayInputStream(source), 16, audit(bytes, failure, closes)))
|
||||
{
|
||||
assertThat(channel.size()).isZero();
|
||||
channel.position(8);
|
||||
ByteBuffer data = ByteBuffer.allocate(4);
|
||||
assertThat(channel.read(data)).isEqualTo(4);
|
||||
assertThat(data.array()).containsExactly(8, 9, 10, 11);
|
||||
channel.position(0);
|
||||
data = ByteBuffer.allocate(8);
|
||||
assertThat(channel.read(data)).isEqualTo(8);
|
||||
assertThat(data.array()).containsExactly(0, 1, 2, 3, 4, 5, 6, 7);
|
||||
assertThat(bytes.get()).isEqualTo(12);
|
||||
channel.position(12);
|
||||
assertThat(channel.read(ByteBuffer.allocate(20))).isEqualTo(20);
|
||||
assertThat(channel.position()).isEqualTo(32);
|
||||
channel.position(16);
|
||||
assertThat(channel.read(ByteBuffer.allocate(4))).isEqualTo(4);
|
||||
channel.position(15);
|
||||
assertThatThrownBy(() -> channel.read(ByteBuffer.allocate(1))).isInstanceOf(IOException.class).hasMessageContaining("download sequentially");
|
||||
assertThatThrownBy(() -> channel.read(ByteBuffer.allocate(1))).isInstanceOf(IOException.class);
|
||||
}
|
||||
assertThat(closes.get()).isEqualTo(1);
|
||||
assertThat(failure.get()).hasMessageContaining("download sequentially");
|
||||
assertThat(bytes.get()).isEqualTo(32);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Tiny upstream packets cannot falsely signal EOF; only one fixed window is retained.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void fullReadsEofAndLifecycle() throws Exception
|
||||
{
|
||||
AtomicInteger closes = new AtomicInteger();
|
||||
AtomicInteger streamCloses = new AtomicInteger();
|
||||
AtomicInteger bytes = new AtomicInteger();
|
||||
AtomicReference<IOException> failure = new AtomicReference<>();
|
||||
InputStream input = new InputStream()
|
||||
{
|
||||
private int remaining = 100003;
|
||||
|
||||
/*******************************************************************************
|
||||
** Supply bytes one at a time to simulate fragmented HTTP reads.
|
||||
*******************************************************************************/
|
||||
public int read()
|
||||
{
|
||||
return (remaining-- > 0 ? 7 : -1);
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Verify stream ownership.
|
||||
*******************************************************************************/
|
||||
public void close()
|
||||
{
|
||||
streamCloses.incrementAndGet();
|
||||
}
|
||||
};
|
||||
var channel = new GeneratedReadChannel(input, 17, audit(bytes, failure, closes));
|
||||
ByteBuffer buffer = ByteBuffer.allocate(1024);
|
||||
long total = 0;
|
||||
int count;
|
||||
while((count = channel.read(buffer.clear())) >= 0)
|
||||
{
|
||||
total += count;
|
||||
assertThat(count).isEqualTo(total == 100003 ? 675 : 1024);
|
||||
}
|
||||
assertThat(total).isEqualTo(100003);
|
||||
assertThat(bytes.get()).isEqualTo(100003);
|
||||
assertThat(channel.read(ByteBuffer.allocate(0))).isZero();
|
||||
assertThatThrownBy(() -> channel.position(-1)).isInstanceOf(IllegalArgumentException.class);
|
||||
assertThatThrownBy(() -> channel.write(buffer)).isInstanceOf(NonWritableChannelException.class);
|
||||
assertThatThrownBy(() -> channel.truncate(0)).isInstanceOf(NonWritableChannelException.class);
|
||||
channel.close();
|
||||
channel.close();
|
||||
assertThat(channel.isOpen()).isFalse();
|
||||
assertThat(closes.get()).isEqualTo(1);
|
||||
assertThat(streamCloses.get()).isEqualTo(1);
|
||||
assertThat(failure.get()).isNull();
|
||||
assertThatThrownBy(channel::position).isInstanceOf(ClosedChannelException.class);
|
||||
assertThatThrownBy(channel::size).isInstanceOf(ClosedChannelException.class);
|
||||
assertThatThrownBy(() -> channel.read(buffer)).isInstanceOf(ClosedChannelException.class);
|
||||
}
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalid windows, excessive forward seeks and underlying read failures are explicit.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void failuresCloseAndAudit() throws Exception
|
||||
{
|
||||
AtomicInteger closes = new AtomicInteger();
|
||||
AtomicReference<IOException> failure = new AtomicReference<>();
|
||||
var audit = audit(new AtomicInteger(), failure, closes);
|
||||
assertThatThrownBy(() -> new GeneratedReadChannel(InputStream.nullInputStream(), 0, audit)).isInstanceOf(IllegalArgumentException.class);
|
||||
try(var channel = new GeneratedReadChannel(InputStream.nullInputStream(), 8, audit))
|
||||
{
|
||||
channel.position(8);
|
||||
assertThatThrownBy(() -> channel.read(ByteBuffer.allocate(1))).hasMessageContaining("sequentially");
|
||||
}
|
||||
try(var channel = new GeneratedReadChannel(new InputStream()
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Simulate an HTTP transport failure.
|
||||
*******************************************************************************/
|
||||
public int read() throws IOException
|
||||
{
|
||||
throw (new IOException("HTTP connection interrupted"));
|
||||
}
|
||||
}, 8, audit))
|
||||
{
|
||||
assertThatThrownBy(() -> channel.read(ByteBuffer.allocate(1))).hasMessageContaining("HTTP connection interrupted");
|
||||
}
|
||||
assertThat(failure.get()).hasMessageContaining("HTTP connection interrupted");
|
||||
assertThat(closes.get()).isEqualTo(2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.filegateway.maintenance.GatewayMaintenance;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayProcesses;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Retention and administrative effects operate on durable shared database state.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(NoDiskExtension.class)
|
||||
class MaintenanceIntegrationTest
|
||||
{
|
||||
@Test
|
||||
void everyRetentionRuleAndFreshRowsSurvive() throws Exception
|
||||
{
|
||||
Instant now = Instant.parse("2026-10-13T12:00:00Z");
|
||||
try(var h = new FileGatewayTestServer(Clock.fixed(now, ZoneOffset.UTC)))
|
||||
{
|
||||
var account = h.account("alice", "A");
|
||||
for(String status : List.of("SUCCEEDED", "RECEIVING", "RUNNING", "QUEUED"))
|
||||
{
|
||||
String key = "retention/" + status;
|
||||
h.store.put(key, new java.io.ByteArrayInputStream(new byte[]{1}));
|
||||
h.store.put(key + "-results", new java.io.ByteArrayInputStream(new byte[]{2}));
|
||||
h.store.put(key + "-plain", new java.io.ByteArrayInputStream(new byte[]{3}));
|
||||
var job = h.audit.insert(FileGatewayTables.UPLOAD_JOB, new QRecord().withValue("accountId", account.getValue("id"))
|
||||
.withValue("status", status).withValue("fileName", status)
|
||||
.withValue("storageKey", key).withValue("resultsStorageKey", key + "-results").withValue("plaintextStorageKey", key + "-plain"));
|
||||
backdate(h, FileGatewayTables.UPLOAD_JOB, job, now.minusSeconds(100 * 86400L));
|
||||
}
|
||||
for(String table : List.of(FileGatewayTables.TRANSFER_LOG, FileGatewayTables.CONNECTION_LOG))
|
||||
{
|
||||
backdate(h, table, h.audit.insert(table, new QRecord().withValue("lastSeenAt", now.minusSeconds(181 * 86400L))), now.minusSeconds(181 * 86400L));
|
||||
backdate(h, table, h.audit.insert(table, new QRecord().withValue("lastSeenAt", now)), now);
|
||||
}
|
||||
h.audit.insert(FileGatewayTables.SERVER, new QRecord().withValue("podName", "dead-pod").withValue("lastSeenAt", now.minusSeconds(86401)));
|
||||
var alias = PhaseEIntegrationTest.alias(h, "out", "/out", "OUTBOX", account);
|
||||
h.store.put("outbox/old", new java.io.ByteArrayInputStream(new byte[]{4}));
|
||||
var out = h.audit.insert(FileGatewayTables.OUTBOX_FILE, new QRecord().withValue("aliasId", alias.getValue("id")).withValue("status", "READY")
|
||||
.withValue("storageKey", "outbox/old"));
|
||||
backdate(h, FileGatewayTables.OUTBOX_FILE, out, now.minusSeconds(31 * 86400L));
|
||||
GatewayMaintenance.maintain(h.instance, h.config, h.store, Clock.fixed(now, ZoneOffset.UTC));
|
||||
var jobs = UploadIntegrationTest.jobs(h);
|
||||
assertThat(jobs).anySatisfy(row ->
|
||||
{
|
||||
assertThat(row.getValueString("status")).isEqualTo("ABORTED");
|
||||
assertThat(row.getValue("storageKey")).isNull();
|
||||
});
|
||||
assertThat(h.store.objects).doesNotContainKeys("retention/SUCCEEDED", "retention/SUCCEEDED-results", "retention/SUCCEEDED-plain", "retention/RECEIVING", "outbox/old")
|
||||
.containsKeys("retention/RUNNING", "retention/QUEUED");
|
||||
assertThat(h.audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", out.getValue("id"))).getFirst().getValueString("status")).isEqualTo("EXPIRED");
|
||||
assertThat(h.audit.query(FileGatewayTables.TRANSFER_LOG, new QQueryFilter())).hasSize(1);
|
||||
assertThat(h.audit.query(FileGatewayTables.CONNECTION_LOG, new QQueryFilter())).hasSize(1);
|
||||
assertThat(h.audit.query(FileGatewayTables.SERVER, new QQueryFilter())).hasSize(1);
|
||||
GatewayMaintenance.maintain(h.instance, h.config, h.store, Clock.fixed(now, ZoneOffset.UTC));
|
||||
assertThat(GatewayMaintenance.metadata(h.config.withSchedulerName("scheduler")).getSchedule().getRepeatSeconds()).isEqualTo(86400);
|
||||
}
|
||||
}
|
||||
|
||||
/*******************************************************************************
|
||||
** Inserts stamp createDate with the real time; move a row's createDate after.
|
||||
*******************************************************************************/
|
||||
private static void backdate(FileGatewayTestServer h, String table, QRecord row, Instant createDate) throws Exception
|
||||
{
|
||||
h.audit.update(table, new QRecord().withValue("id", row.getValue("id")).withValue("createDate", createDate));
|
||||
assertThat(h.audit.query(table, UploadJobs.equal("id", row.getValue("id"))).getFirst().getValueInstant("createDate")).isEqualTo(createDate);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Test
|
||||
void reviewResultAndLiveRevocation() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
var account = h.account("alice", "A");
|
||||
var keyPair = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
var key = h.key(account, keyPair);
|
||||
QContext.init(h.instance, new QSession().withPermissions(Set.of("file_gateway.admin")));
|
||||
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("lockedUntil", Instant.now().plusSeconds(600)).withValue("failedLoginCount", 5));
|
||||
apply(h, FileGatewayProcesses.UNLOCK, account);
|
||||
var unlocked = h.audit.query(FileGatewayTables.ACCOUNT, UploadJobs.equal("id", account.getValue("id"))).getFirst();
|
||||
assertThat(unlocked.getValue("lockedUntil")).isNull();
|
||||
assertThat(unlocked.getValueInteger("failedLoginCount")).isZero();
|
||||
try(var ssh = h.login("alice", keyPair))
|
||||
{
|
||||
var row = h.audit.query(FileGatewayTables.CONNECTION_LOG, UploadJobs.equal("accountId", account.getValue("id"))).getFirst();
|
||||
apply(h, FileGatewayProcesses.END_SESSION, row);
|
||||
assertThat(h.audit.query(FileGatewayTables.CONNECTION_LOG, UploadJobs.equal("id", row.getValue("id"))).getFirst().getValue("disconnectedAt")).isNull();
|
||||
h.runtime.getProtocol().getAuthenticator().poll();
|
||||
ssh.waitFor(java.util.EnumSet.of(org.apache.sshd.client.session.ClientSession.ClientSessionEvent.CLOSED), java.time.Duration.ofSeconds(5));
|
||||
assertThat(ssh.isClosed()).isTrue();
|
||||
}
|
||||
try(var ssh = h.login("alice", keyPair))
|
||||
{
|
||||
apply(h, FileGatewayProcesses.DISABLE, account);
|
||||
h.runtime.getProtocol().getAuthenticator().poll();
|
||||
ssh.waitFor(java.util.EnumSet.of(org.apache.sshd.client.session.ClientSession.ClientSessionEvent.CLOSED), java.time.Duration.ofSeconds(5));
|
||||
assertThat(ssh.isClosed()).isTrue();
|
||||
}
|
||||
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("status", "ACTIVE"));
|
||||
apply(h, FileGatewayProcesses.REVOKE, key);
|
||||
assertThatThrownBy(() -> h.login("alice", keyPair)).isInstanceOf(Exception.class);
|
||||
}
|
||||
}
|
||||
|
||||
private void apply(FileGatewayTestServer h, String action, QRecord row) throws Exception
|
||||
{
|
||||
QContext.init(h.instance, new QSession().withPermissions(Set.of("file_gateway.admin")));
|
||||
var input = new RunBackendStepInput().withProcessName(action).withRecords(List.of(row));
|
||||
var review = new FileGatewayProcesses.Review();
|
||||
var preview = new RunBackendStepOutput();
|
||||
review.runOnePage(input, preview);
|
||||
assertThat(review.getProcessSummary(preview, false)).hasSize(1);
|
||||
assertThat(review.getProcessSummary(preview, true)).hasSize(1);
|
||||
var result = new RunBackendStepOutput();
|
||||
new FileGatewayProcesses.Apply().runOnePage(input.withRecords(preview.getRecords()), result);
|
||||
assertThat(result.getRecords()).hasSize(1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.util.Date;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.pgp.PgpStreams;
|
||||
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
import org.apache.sshd.sftp.client.SftpClientFactory;
|
||||
import org.bouncycastle.bcpg.ArmoredOutputStream;
|
||||
import org.bouncycastle.bcpg.HashAlgorithmTags;
|
||||
import org.bouncycastle.openpgp.PGPKeyRingGenerator;
|
||||
import org.bouncycastle.openpgp.PGPPublicKey;
|
||||
import org.bouncycastle.openpgp.PGPSignature;
|
||||
import org.bouncycastle.openpgp.operator.bc.BcPGPContentSignerBuilder;
|
||||
import org.bouncycastle.openpgp.operator.bc.BcPGPDigestCalculatorProvider;
|
||||
import org.bouncycastle.openpgp.operator.jcajce.JcaPGPKeyPair;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Real transfer encryption/decryption with ephemeral test-only OpenPGP keys.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(NoDiskExtension.class)
|
||||
class PgpIntegrationTest
|
||||
{
|
||||
record Keys(String publicKey, String privateKey)
|
||||
{
|
||||
}
|
||||
|
||||
static Keys keys() throws Exception
|
||||
{
|
||||
var generator = KeyPairGenerator.getInstance("RSA");
|
||||
generator.initialize(2048);
|
||||
var signing = new JcaPGPKeyPair(PGPPublicKey.RSA_SIGN, generator.generateKeyPair(), new Date());
|
||||
var encryption = new JcaPGPKeyPair(PGPPublicKey.RSA_ENCRYPT, generator.generateKeyPair(), new Date());
|
||||
var rings = new PGPKeyRingGenerator(PGPSignature.POSITIVE_CERTIFICATION, signing, "ephemeral-test",
|
||||
new BcPGPDigestCalculatorProvider().get(HashAlgorithmTags.SHA1), null, null,
|
||||
new BcPGPContentSignerBuilder(PGPPublicKey.RSA_SIGN, HashAlgorithmTags.SHA256), null);
|
||||
rings.addSubKey(encryption);
|
||||
ByteArrayOutputStream publicBytes = new ByteArrayOutputStream();
|
||||
try(var armor = new ArmoredOutputStream(publicBytes))
|
||||
{
|
||||
rings.generatePublicKeyRing().encode(armor);
|
||||
}
|
||||
ByteArrayOutputStream privateBytes = new ByteArrayOutputStream();
|
||||
try(var armor = new ArmoredOutputStream(privateBytes))
|
||||
{
|
||||
rings.generateSecretKeyRing().encode(armor);
|
||||
}
|
||||
return (new Keys(publicBytes.toString(StandardCharsets.UTF_8), privateBytes.toString(StandardCharsets.UTF_8)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void encryptedBulkUploadInvalidCiphertextAndAccountValidation() throws Exception
|
||||
{
|
||||
Keys keys = keys();
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
UploadIntegrationTest.profile(h);
|
||||
var account = h.account("alice", "A");
|
||||
assertThatThrownBy(() -> h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id"))
|
||||
.withValue("pgpPublicKey", "invalid"))).isInstanceOf(Exception.class);
|
||||
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("pgpDecryptUploads", true));
|
||||
h.config.withPgpPrivateKeyEnvVarName("TEST_PGP").withPgpPassphraseEnvVarName("TEST_PASS");
|
||||
h.runtime.getWorker().withEnvironment(name -> "TEST_PGP".equals(name) ? keys.privateKey() : "");
|
||||
var sshKey = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(account, sshKey);
|
||||
try(var ssh = h.login("alice", sshKey); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
|
||||
{
|
||||
byte[] ciphertext;
|
||||
try(var input = PgpStreams.encrypt(new ByteArrayInputStream(UploadIntegrationTest.CSV), keys.publicKey(), "orders.csv"))
|
||||
{
|
||||
ciphertext = input.readAllBytes();
|
||||
}
|
||||
try(var output = sftp.write(UploadIntegrationTest.FOLDER + "orders.csv.pgp"))
|
||||
{
|
||||
output.write(ciphertext);
|
||||
}
|
||||
h.runtime.getWorker().runOnce();
|
||||
var job = UploadIntegrationTest.jobs(h).getFirst();
|
||||
assertThat(job.getValueString("status")).isEqualTo("SUCCEEDED");
|
||||
assertThat(job.getValueInteger("insertedCount")).isEqualTo(2);
|
||||
assertThat(job.getValueString("plaintextSha256")).isNotBlank().isNotEqualTo(job.getValueString("sha256"));
|
||||
assertThat(h.store.objects.get(job.getValueString("plaintextStorageKey"))).isEqualTo(UploadIntegrationTest.CSV);
|
||||
assertThat(h.store.objects.get(job.getValueString("storageKey"))).isEqualTo(ciphertext);
|
||||
try(var output = sftp.write(UploadIntegrationTest.FOLDER + "bad.csv.gpg"))
|
||||
{
|
||||
output.write(new byte[]{1, 2, 3});
|
||||
}
|
||||
h.runtime.getWorker().runOnce();
|
||||
assertThat(UploadIntegrationTest.jobs(h)).anySatisfy(row ->
|
||||
{
|
||||
assertThat(row.getValueString("status")).isEqualTo("REJECTED");
|
||||
assertThat(row.getValueString("errorMessage")).contains("PGP decryption failed");
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void encryptedGeneratedAndOutboxDownloadsMatchPlaintext() throws Exception
|
||||
{
|
||||
Keys keys = keys();
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
h.seedRows(2);
|
||||
h.saved("savedView", "alpha", "Daily");
|
||||
var account = h.account("alice", "A");
|
||||
var sshKey = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(account, sshKey);
|
||||
byte[] plain;
|
||||
try(var ssh = h.login("alice", sshKey); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
|
||||
{
|
||||
plain = PhaseEIntegrationTest.read(sftp, "/testapp/alpha/views/daily.csv");
|
||||
}
|
||||
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id"))
|
||||
.withValue("pgpPublicKey", keys.publicKey()).withValue("pgpEncryptDownloads", true));
|
||||
var alias = PhaseEIntegrationTest.alias(h, "out", "/out", "OUTBOX", account);
|
||||
h.audit.update(FileGatewayTables.ALIAS, new QRecord().withValue("id", alias.getValue("id")).withValue("consumerName", null));
|
||||
com.kingsrook.qbits.filegateway.api.FileGatewayOutbox.put("out", "reply.csv", new ByteArrayInputStream(plain), plain.length);
|
||||
try(var ssh = h.login("alice", sshKey); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
|
||||
{
|
||||
assertThat(PhaseEIntegrationTest.names(sftp, "/testapp/alpha/views")).contains("daily.csv.pgp");
|
||||
for(String path : java.util.List.of("/testapp/alpha/views/daily.csv.pgp", "/out/reply.csv.pgp"))
|
||||
{
|
||||
byte[] encrypted = PhaseEIntegrationTest.read(sftp, path);
|
||||
try(var decrypted = PgpStreams.decrypt(new ByteArrayInputStream(encrypted), keys.privateKey(), new char[0], 1_000_000))
|
||||
{
|
||||
assertThat(decrypted.readAllBytes()).isEqualTo(plain);
|
||||
}
|
||||
}
|
||||
assertThat(h.audit.query(FileGatewayTables.OUTBOX_FILE, new com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter()).getFirst().getValueString("status"))
|
||||
.isEqualTo("DOWNLOADED");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void integrityAndPlaintextSizeAreEnforced() throws Exception
|
||||
{
|
||||
Keys keys = keys();
|
||||
byte[] bytes = new byte[160000];
|
||||
new java.util.Random(42).nextBytes(bytes);
|
||||
byte[] encrypted;
|
||||
try(var input = PgpStreams.encrypt(new ByteArrayInputStream(bytes), keys.publicKey(), "file.bin"))
|
||||
{
|
||||
encrypted = input.readAllBytes();
|
||||
}
|
||||
try(var input = PgpStreams.decrypt(new ByteArrayInputStream(encrypted), keys.privateKey(), new char[0], bytes.length))
|
||||
{
|
||||
assertThat(input.readAllBytes()).isEqualTo(bytes);
|
||||
}
|
||||
assertThatThrownBy(() ->
|
||||
{
|
||||
try(var input = PgpStreams.decrypt(new ByteArrayInputStream(encrypted), keys.privateKey(), new char[0], 10))
|
||||
{
|
||||
input.readAllBytes();
|
||||
}
|
||||
}).isInstanceOf(java.io.IOException.class);
|
||||
encrypted[encrypted.length - 1] ^= 1;
|
||||
assertThatThrownBy(() ->
|
||||
{
|
||||
try(var input = PgpStreams.decrypt(new ByteArrayInputStream(encrypted), keys.privateKey(), new char[0], bytes.length))
|
||||
{
|
||||
input.readAllBytes();
|
||||
}
|
||||
}).isInstanceOf(java.io.IOException.class);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,318 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Clock;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.filegateway.api.FileAcceptance;
|
||||
import com.kingsrook.qbits.filegateway.api.FileGatewayConsumer;
|
||||
import com.kingsrook.qbits.filegateway.api.FileGatewayConsumersMetaData;
|
||||
import com.kingsrook.qbits.filegateway.api.FileGatewayOutbox;
|
||||
import com.kingsrook.qbits.filegateway.api.InboundFile;
|
||||
import com.kingsrook.qbits.filegateway.api.OutboxFileRef;
|
||||
import com.kingsrook.qbits.filegateway.maintenance.GatewayMaintenance;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayHome;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.outbox.OutboxService;
|
||||
import com.kingsrook.qbits.filegateway.upload.UploadJobs;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData;
|
||||
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
import org.apache.sshd.sftp.client.SftpClient;
|
||||
import org.apache.sshd.sftp.client.SftpClientFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
/*******************************************************************************
|
||||
** Phase E acceptance uses real MINA sessions, strict UI API and the adopter DDL.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(NoDiskExtension.class)
|
||||
class PhaseEIntegrationTest
|
||||
{
|
||||
static InboundFile accepted;
|
||||
static byte[] acceptedBytes;
|
||||
static int downloaded;
|
||||
static boolean failCallback;
|
||||
|
||||
public static class Consumer implements FileGatewayConsumer
|
||||
{
|
||||
@Override
|
||||
public String getName()
|
||||
{
|
||||
return ("partner");
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileAcceptance accept(InboundFile file) throws Exception
|
||||
{
|
||||
accepted = file;
|
||||
try(var stream = file.openStream())
|
||||
{
|
||||
acceptedBytes = stream.readAllBytes();
|
||||
}
|
||||
return (FileAcceptance.accepted("receipt"));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onDownloaded(OutboxFileRef ref) throws Exception
|
||||
{
|
||||
if(failCallback)
|
||||
{
|
||||
throw (new Exception("retry test"));
|
||||
}
|
||||
downloaded++;
|
||||
}
|
||||
}
|
||||
|
||||
static QRecord alias(FileGatewayTestServer h, String name, String path, String kind, QRecord account) throws Exception
|
||||
{
|
||||
return (h.audit.insert(FileGatewayTables.ALIAS, new QRecord().withValue("name", name).withValue("path", path).withValue("kind", kind)
|
||||
.withValue("accountId", account == null ? null : account.getValue("id")).withValue("consumerName", "partner")));
|
||||
}
|
||||
|
||||
static List<String> names(SftpClient sftp, String path) throws Exception
|
||||
{
|
||||
List<String> names = new ArrayList<>();
|
||||
for(var item : sftp.readDir(path))
|
||||
{
|
||||
if(!List.of(".", "..").contains(item.getFilename()))
|
||||
{
|
||||
names.add(item.getFilename());
|
||||
}
|
||||
}
|
||||
return (names);
|
||||
}
|
||||
|
||||
static byte[] read(SftpClient sftp, String path) throws Exception
|
||||
{
|
||||
try(var input = sftp.read(path))
|
||||
{
|
||||
return (input.readAllBytes());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void endpointsChrootVisibilityBulkAndInbox() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
FileGatewayConsumersMetaData.ofOrWithNew(h.instance).withConsumer(new QCodeReference(Consumer.class));
|
||||
var profile = UploadIntegrationTest.profile(h);
|
||||
var service = h.audit.insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", "service").withValue("userId", "A").withValue("accountType", "SERVICE"));
|
||||
assertThat(service.getValueString("rootMode")).isEqualTo("ENDPOINTS_ONLY");
|
||||
var user = h.account("alice", "A");
|
||||
assertThat(user.getValueString("rootMode")).isEqualTo("FULL_TREE");
|
||||
var denied = h.account("bob", "B");
|
||||
var bulk = alias(h, "orders", "/acme/orders", "BULK_LOAD", null);
|
||||
h.audit.update(FileGatewayTables.ALIAS, new QRecord().withValue("id", bulk.getValue("id")).withValue("tableName", "alpha")
|
||||
.withValue("savedBulkLoadProfileId", profile.getValue("id")));
|
||||
alias(h, "inbox", "/acme/inbox", "INBOX", service);
|
||||
alias(h, "private", "/private", "INBOX", user);
|
||||
alias(h, "shared", "/shared", "OUTBOX", null);
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(service, key);
|
||||
var userKey = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(user, userKey);
|
||||
var deniedKey = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(denied, deniedKey);
|
||||
try(var ssh = h.login("service", key); var sftp = SftpClientFactory.instance().createSftpClient(ssh);
|
||||
var aliceSsh = h.login("alice", userKey); var alice = SftpClientFactory.instance().createSftpClient(aliceSsh);
|
||||
var bobSsh = h.login("bob", deniedKey); var bob = SftpClientFactory.instance().createSftpClient(bobSsh))
|
||||
{
|
||||
assertThat(names(sftp, "/")).containsExactlyInAnyOrder("acme", "shared");
|
||||
assertThat(names(alice, "/")).contains("testapp", "acme", "private", "shared");
|
||||
assertThat(names(bob, "/")).doesNotContain("acme", "private");
|
||||
assertThatThrownBy(() -> sftp.stat("/testapp")).isInstanceOf(Exception.class);
|
||||
try(var output = sftp.write("/acme/orders/order.csv"))
|
||||
{
|
||||
output.write(UploadIntegrationTest.CSV);
|
||||
}
|
||||
h.runtime.getWorker().runOnce();
|
||||
assertThat(UploadIntegrationTest.jobs(h).getFirst().getValueString("status")).isEqualTo("SUCCEEDED");
|
||||
try(var output = sftp.write("/acme/inbox/message.txt"))
|
||||
{
|
||||
output.write("hello partner".getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
h.runtime.getWorker().runOnce();
|
||||
assertThat(accepted.endpointName()).isEqualTo("inbox");
|
||||
assertThat(accepted.remoteAddress()).isEqualTo("127.0.0.1");
|
||||
assertThat(acceptedBytes).isEqualTo("hello partner".getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void endpointValidationAndView() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
var user = h.account("alice", "A");
|
||||
for(String path : List.of("relative", "/../escape", "/UPPER", "/a//b", "/a/./b", "/a/", "/testapp/in"))
|
||||
{
|
||||
assertThatThrownBy(() -> alias(h, "invalid", path, "INBOX", user)).isInstanceOf(Exception.class);
|
||||
}
|
||||
var mount = alias(h, "good", "/good", "INBOX", user);
|
||||
assertThatThrownBy(() -> h.audit.update(FileGatewayTables.ALIAS, new QRecord().withValue("id", mount.getValue("id")).withValue("path", "/testapp"))).isInstanceOf(Exception.class);
|
||||
assertThatThrownBy(() -> alias(h, "duplicate", "/good", "INBOX", user)).isInstanceOf(Exception.class);
|
||||
h.seedRows(2);
|
||||
var saved = h.saved("savedView", "alpha", "Daily Rows");
|
||||
var view = alias(h, "view", "/download", "VIEW", user);
|
||||
h.audit.update(FileGatewayTables.ALIAS, new QRecord().withValue("id", view.getValue("id")).withValue("tableName", "alpha")
|
||||
.withValue("savedViewId", saved.getValue("id")).withValue("format", "csv"));
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(user, key);
|
||||
try(var ssh = h.login("alice", key); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
|
||||
{
|
||||
assertThat(names(sftp, "/download")).containsExactly("daily-rows.csv");
|
||||
assertThat(read(sftp, "/download/daily-rows.csv")).isEqualTo(read(sftp, "/testapp/alpha/views/daily-rows.csv"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void outboxLifecycleAndRetry() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
downloaded = 0;
|
||||
failCallback = false;
|
||||
FileGatewayConsumersMetaData.ofOrWithNew(h.instance).withConsumer(new QCodeReference(Consumer.class));
|
||||
var account = h.account("alice", "A");
|
||||
alias(h, "out", "/out", "OUTBOX", account);
|
||||
alias(h, "in", "/in", "INBOX", account);
|
||||
assertThatThrownBy(() -> FileGatewayOutbox.put("missing", "x", new ByteArrayInputStream(new byte[0]), 0)).isInstanceOf(Exception.class);
|
||||
assertThatThrownBy(() -> FileGatewayOutbox.put("in", "x", new ByteArrayInputStream(new byte[0]), 0)).isInstanceOf(Exception.class);
|
||||
byte[] bytes = new byte[200000];
|
||||
new java.util.Random(7).nextBytes(bytes);
|
||||
var ref = FileGatewayOutbox.put("out", "reply.bin", new ByteArrayInputStream(bytes), bytes.length);
|
||||
var key = KeyUtils.generateKeyPair("ssh-ed25519", 256);
|
||||
h.key(account, key);
|
||||
try(var ssh = h.login("alice", key); var sftp = SftpClientFactory.instance().createSftpClient(ssh))
|
||||
{
|
||||
assertThat(names(sftp, "/out")).containsExactly("reply.bin");
|
||||
try(var handle = sftp.open("/out/reply.bin"))
|
||||
{
|
||||
byte[] partial = new byte[100];
|
||||
for(int i = 0; i < 5; i++)
|
||||
{
|
||||
sftp.read(handle, 0, partial);
|
||||
}
|
||||
}
|
||||
assertThat(downloaded).isZero();
|
||||
assertThat(read(sftp, "/out/reply.bin")).isEqualTo(bytes);
|
||||
assertThat(downloaded).isEqualTo(1);
|
||||
var row = h.audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", ref.outboxFileId())).getFirst();
|
||||
assertThat(row.getValueString("status")).isEqualTo("DOWNLOADED");
|
||||
assertThat(row.getValue("firstDownloadedAt")).isNotNull();
|
||||
assertThat(row.getValueInteger("downloadCount")).isEqualTo(1);
|
||||
read(sftp, "/out/reply.bin");
|
||||
assertThat(downloaded).isEqualTo(1);
|
||||
sftp.remove("/out/reply.bin");
|
||||
assertThat(names(sftp, "/out")).isEmpty();
|
||||
assertThat(h.audit.query(FileGatewayTables.OUTBOX_FILE, UploadJobs.equal("id", ref.outboxFileId())).getFirst().getValueString("status")).isEqualTo("REMOVED");
|
||||
failCallback = true;
|
||||
FileGatewayOutbox.put("out", "retry.bin", new ByteArrayInputStream(bytes), bytes.length);
|
||||
read(sftp, "/out/retry.bin");
|
||||
assertThat(downloaded).isEqualTo(1);
|
||||
failCallback = false;
|
||||
GatewayMaintenance.maintain(h.instance, h.config, h.store, Clock.systemUTC());
|
||||
assertThat(downloaded).isEqualTo(2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void outboxWithoutRunningSftpAndHomeAndIntegrations() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance -> { }, 0, false, false))
|
||||
{
|
||||
assertThat(FileGatewayOutbox.isAvailable()).isTrue();
|
||||
FileGatewayOutbox.setProvider(new OutboxService(h.instance, h.store, h.config, Clock.systemUTC()));
|
||||
var account = h.account("partner", "A");
|
||||
alias(h, "out", "/out", "OUTBOX", account);
|
||||
FileGatewayOutbox.put("out", "file.txt", new ByteArrayInputStream(new byte[]{1}), 1);
|
||||
Instant now = Instant.now();
|
||||
h.seedTransfer(account, "UPLOAD", 2097152L, now);
|
||||
h.seedTransfer(account, "DOWNLOAD", 1048576L, now);
|
||||
h.audit.insert(FileGatewayTables.UPLOAD_JOB, new QRecord().withValue("accountId", account.getValue("id")).withValue("fileName", "orders.csv")
|
||||
.withValue("status", "FAILED").withValue("finishedAt", now).withValue("errorMessage", "Invalid quantity"));
|
||||
QContext.init(h.instance, new QSession().withPermissions(Set.of("file_gateway.view")));
|
||||
FileGatewayHome home = new FileGatewayHome();
|
||||
assertThat(((Number) ((StatisticsData) home.data("transfers", now)).getCount()).intValue()).isEqualTo(2);
|
||||
assertThat(((Number) ((StatisticsData) home.data("in", now)).getCount()).doubleValue()).isEqualTo(2);
|
||||
assertThat(((Number) ((StatisticsData) home.data("out", now)).getCount()).doubleValue()).isEqualTo(1);
|
||||
assertThat(((Number) ((StatisticsData) home.data("failures", now)).getCount()).intValue()).isEqualTo(1);
|
||||
assertThat(((TableData) home.data("recent", now)).getRows()).hasSize(1);
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// direction: failures and locks are bad when they grow; volume is not bad //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
assertThat(((StatisticsData) home.data("failures", now)).getIncreaseIsGood()).as("failures").isFalse();
|
||||
assertThat(((StatisticsData) home.data("locked", now)).getIncreaseIsGood()).as("locked").isFalse();
|
||||
for(String key : List.of("transfers", "in", "out", "sessions"))
|
||||
{
|
||||
assertThat(((StatisticsData) home.data(key, now)).getIncreaseIsGood()).as(key).isTrue();
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// every home table says why it is empty //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
for(String key : List.of("recent", "partners", "active", "servers"))
|
||||
{
|
||||
assertThat(((TableData) home.data(key, now)).getNoRowsFoundHTML()).as(key).isNotBlank();
|
||||
}
|
||||
assertThat(((TableData) home.data("active", now)).getRows()).isEmpty();
|
||||
assertThat(((TableData) home.data("active", now)).getNoRowsFoundHTML()).contains("No one is connected");
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// readable at all widths: home tables declare no column widths //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
assertThat(((TableData) home.data("recent", now)).getColumns()).allSatisfy(column -> assertThat(column.getWidth()).isNull());
|
||||
for(String key : List.of("recent", "partners", "active", "servers"))
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// at most four columns, so tables stay readable at phone widths //
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
assertThat(((TableData) home.data(key, now)).getColumns()).as(key).hasSizeLessThanOrEqualTo(4)
|
||||
.allSatisfy(column -> assertThat(column.getWidth()).isNull());
|
||||
}
|
||||
assertThat(((TableData) home.data("partners", now)).getRows()).hasSize(1);
|
||||
for(String widget : FileGatewayHome.KEYS)
|
||||
{
|
||||
assertThat(home.data(widget, now).getType()).isIn("statistics", "table");
|
||||
}
|
||||
assertThat(h.instance.getWidgets().values()).allSatisfy(widget -> assertThat(widget.getType()).isNotEqualTo("html"));
|
||||
assertThat(FileGatewayIntegrations.esbTableNames()).containsExactly(FileGatewayTables.ACCOUNT, FileGatewayTables.ALIAS, FileGatewayTables.UPLOAD_JOB);
|
||||
assertThat(FileGatewayIntegrations.apiTableNames()).containsExactlyElementsOf(FileGatewayTables.ALL);
|
||||
assertThat(FileGatewayIntegrations.quickSearchTables()).hasSize(3);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -78,6 +78,35 @@ class SftpIntegrationTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A TCP connection that never signs in (a health probe or port scan) leaves
|
||||
** no connection-log row and no failure; a real sign-in attempt does.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void tcpProbeLeavesNoConnectionRow() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
for(int i = 0; i < 3; i++)
|
||||
{
|
||||
try(java.net.Socket socket = new java.net.Socket("127.0.0.1", h.runtime.getProtocol().getPort()))
|
||||
{
|
||||
socket.setSoTimeout(2000);
|
||||
socket.getInputStream().read();
|
||||
}
|
||||
}
|
||||
Thread.sleep(300);
|
||||
assertThat(h.audit.query("file_gateway_connection_log", new QQueryFilter())).isEmpty();
|
||||
|
||||
KeyPair unknown = pair();
|
||||
org.assertj.core.api.Assertions.assertThatThrownBy(() -> h.login("nobody", unknown)).isInstanceOf(Exception.class);
|
||||
assertThat(h.audit.query("file_gateway_connection_log", new QQueryFilter())).singleElement()
|
||||
.satisfies(row -> assertThat(row.getValueString("outcome")).isEqualTo("AUTH_FAILED"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Permission filtered listings and audit.
|
||||
*******************************************************************************/
|
||||
@@ -109,7 +138,10 @@ class SftpIntegrationTest
|
||||
assertThat(names(ca, "/testapp/alpha/views")).containsExactlyInAnyOrder("all-orders.csv", "all-orders.xlsx");
|
||||
assertThat(names(ca, "/testapp/alpha/reports")).contains("daily-orders.csv", "daily-orders.xlsx");
|
||||
assertThat(ca.stat("/testapp/alpha/views/all-orders.csv").getSize()).isZero();
|
||||
assertThatThrownBy(() -> ca.read("/testapp/alpha/views/all-orders.csv")).isInstanceOf(SftpException.class);
|
||||
try(var download = ca.read("/testapp/alpha/views/all-orders.csv"))
|
||||
{
|
||||
assertThat(download.readAllBytes()).isNotEmpty();
|
||||
}
|
||||
assertThatThrownBy(() -> ca.write("/testapp/alpha/test.csv")).isInstanceOf(SftpException.class);
|
||||
assertThatThrownBy(() -> ca.setStat("/testapp/alpha/views/all-orders.csv", new SftpClient.Attributes().modifyTime(0))).isInstanceOfSatisfying(SftpException.class,
|
||||
e -> assertThat(e.getStatus()).isEqualTo(SftpConstants.SSH_FX_PERMISSION_DENIED));
|
||||
@@ -123,7 +155,7 @@ class SftpIntegrationTest
|
||||
}
|
||||
assertThat(harness.calls).isNotEmpty().allSatisfy(call ->
|
||||
{
|
||||
String template = call.path().substring("/qqq/v1".length()).replaceAll("/table/[^/]+/query", "/table/{tableName}/query");
|
||||
String template = call.path().substring("/qqq/v1".length()).replaceAll("/table/[^/]+/(query|export)", "/table/{tableName}/$1");
|
||||
assertThat(harness.runtime.getInvoker().findOperation(HttpMethod.valueOf(call.method()), template)).isPresent();
|
||||
assertThat(call.channel()).startsWith("sftp/");
|
||||
assertThat(call.invokedBy()).isEqualTo("file-gateway");
|
||||
|
||||
Reference in new issue
Block a user