Compare commits
51
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
79d241e1f6
|
||
|
|
c47d5a1c18 | ||
|
|
677aa54801
|
||
|
|
44a463e4d9
|
||
|
|
32f91b75c2
|
||
|
|
32239cd57d
|
||
|
|
950a8018c7
|
||
|
|
43bfe48feb
|
||
|
|
ead0c904ce
|
||
|
|
579946d62c
|
||
|
|
4f53c29318
|
||
|
|
cd3c0393a2
|
||
|
|
d9f74cb576
|
||
|
|
d12a7777af | ||
|
|
5f6a5022ce | ||
|
|
3c9a4d229b | ||
|
|
c5e1dd7e14 | ||
|
|
d6ebb60de6 | ||
|
|
e41bf1147b | ||
|
|
d4ca427445 | ||
|
|
2542dc0f89
|
||
|
|
c6954d86f1 | ||
|
|
05bfcd8d4b
|
||
|
|
d3cc594b0e | ||
|
|
ff182b6d7f
|
||
|
|
7482050234 | ||
|
|
27324a4159
|
||
|
|
37ee9028f2 | ||
|
|
c8a231f5f8
|
||
|
|
0a8c95e0ad | ||
|
|
f0e9836a89
|
||
|
|
dd4643e60d | ||
|
|
483c856594
|
||
|
|
275cc26b6b | ||
|
|
3e5cec20d2
|
||
|
|
1713d18fd7 | ||
|
|
6073091434 | ||
|
|
db241b03bc
|
||
|
|
10eb8807ba
|
||
|
|
ac55e437bd | ||
|
|
3ba219d668 | ||
|
|
d0572402a9
|
||
|
|
e6ff62181b
|
||
|
|
dc8ac5f089
|
||
|
|
7a5e42495d
|
||
|
|
5098152460 | ||
|
|
ab64cbf040 | ||
|
|
6261e197d0 | ||
|
|
3672bb513d | ||
|
|
852d474722 | ||
|
|
ec8092e468 |
No files matched your search
+10
-61
@@ -1,65 +1,14 @@
|
||||
version: 2.1
|
||||
|
||||
setup: true
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
workflows:
|
||||
|
||||
test_only:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release/.*|integration.*)/
|
||||
tags:
|
||||
only: []
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- develop
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- /release\/.*/
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [ qqq-maven-registry-credentials ]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [ /^v\d+\.\d+\.\d+$/ ]
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
@@ -0,0 +1,7 @@
|
||||
name: build
|
||||
on:
|
||||
push:
|
||||
jobs:
|
||||
maven:
|
||||
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
|
||||
secrets: inherit
|
||||
@@ -0,0 +1,23 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -1,25 +1,74 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
description: Plan a cut without changing release state
|
||||
type: boolean
|
||||
default: false
|
||||
finalize:
|
||||
description: Retry finalization of an existing cut; do not cut another RC
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Recover the same published RC after a deployment timeout; never
|
||||
reupload
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
description: Existing cut merge_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
bump_sha:
|
||||
description: Existing cut bump_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
branch:
|
||||
description: Existing cut branch (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
version:
|
||||
description: Existing cut version (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
types:
|
||||
- closed
|
||||
branches:
|
||||
- develop
|
||||
repository_dispatch:
|
||||
types:
|
||||
- qbit-rc-published
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
actions: read
|
||||
checks: read
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|
||||
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
|
||||
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
|
||||
'release:rc'))
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
|
||||
}}
|
||||
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
|
||||
== 'workflow_dispatch' && inputs.finalize) }}
|
||||
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
|
||||
}}
|
||||
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|
||||
|| '' }}
|
||||
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|
||||
|| inputs.merge_sha || '' }}
|
||||
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|
||||
|| inputs.bump_sha || '' }}
|
||||
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|
||||
|| inputs.branch || '' }}
|
||||
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|
||||
|| inputs.version || '' }}
|
||||
secrets: inherit
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release-no-integration
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
asciidoc: false
|
||||
release_completion: true
|
||||
@@ -14,13 +14,12 @@ code and QInstance. Design: `docs/DESIGN.md`.
|
||||
## Build commands
|
||||
|
||||
```bash
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT compile
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT test
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
```
|
||||
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd` and quote the `-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT` argument in PowerShell.
|
||||
Keep the POM default at `4.1.0-SNAPSHOT` for CI; the private version prevents local artifact replacement.
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -69,8 +68,12 @@ Phase (e) adds endpoints and chroot (`AliasCustomizer` validates paths), `outbox
|
||||
decrypt/encrypt, keys in memory), `maintenance/` (daily retention process and record-action
|
||||
effects), `FileGatewayHome` (typed STATISTICS and TABLE widgets only; guardrail G5),
|
||||
`FileGatewayIntegrations` (ESB, Quick Search, API lists) and `file_gateway_server` heartbeats.
|
||||
The Preferences "File transfer" section waits for QRun-IO/qqq#1162. The k8s example is in
|
||||
`docs/k8s/`.
|
||||
The `file-transfer` Preferences section requires only `file_gateway.connect`. Its provider uses
|
||||
core's trusted scope with the original session, derives identity only from the request context,
|
||||
and manages the caller's unrevoked USER-account keys through normal table actions. Add key
|
||||
creates an account on first use; Revoke sets `revoked_at`; PATCH edits labels and expiry.
|
||||
Optional `publicHostName` advertises the hostname; port/fingerprint use the latest heartbeat.
|
||||
The k8s example is in `docs/k8s/`.
|
||||
|
||||
Tests run a strict UI API server, H2 from the adopter DDL and real MINA clients. NoDiskExtension
|
||||
snapshots the working tree (excluding target) and isolated test home, and asserts the test temp
|
||||
@@ -92,7 +95,7 @@ directory is empty. The host supplies RDBMS; `qqq-backend-module-rdbms` must rem
|
||||
|
||||
## Dependencies
|
||||
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-SNAPSHOT` until 4.1.0-RC.6.
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.7` (first release with QRun-IO/qqq#1170); never a SNAPSHOT.
|
||||
Only allow-listed qbit API artifacts (`qbit-object-storage` `api` package) may be depended on.
|
||||
Notifications go through core `QAlerts.notify`, never qbit-alerts classes.
|
||||
|
||||
|
||||
+19
-4
@@ -4,8 +4,24 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- Upload-job status transitions no longer use `status` as the table's record-version field.
|
||||
QQQ 4.1.0-RC.26 advances a configured version field on every update (a STRING version
|
||||
becomes a random UUID), which overwrote upload status. The guarded transition now re-reads
|
||||
status inside the existing cross-pod account-lock transaction. The upload-job table no longer
|
||||
declares a record-version field; no schema or public API change.
|
||||
|
||||
- Endpoint pickers for saved views, saved reports and saved bulk-load profiles are dropped when
|
||||
the host has not registered those possible-value sources, so hosts without them validate.
|
||||
|
||||
### Added
|
||||
|
||||
- File transfer Preferences with connect-permission visibility, read-only connection details,
|
||||
optional public hostname, self-service Add key/Revoke actions and editable key labels/expiry.
|
||||
- First-key USER account creation with configured defaults and unique usernames; transactional
|
||||
ownership checks, friendly duplicate-key validation, core/HTTP tests and real SFTP acceptance.
|
||||
|
||||
- Phase (c) generated saved-view and saved-report downloads through the user-scoped UI API,
|
||||
CSV/XLSX/JSON account formats, a bounded rolling read window and HTTP stream cleanup.
|
||||
- Generated-file and UTC daily-download quotas, persisted byte charges under account locks,
|
||||
@@ -45,12 +61,11 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
|
||||
|
||||
### Compatibility
|
||||
|
||||
- Local invoker integration builds use `4.1.0-FG-SNAPSHOT`; CI retains the POM's
|
||||
`4.1.0-SNAPSHOT` default. RDBMS remains a host-provided runtime dependency.
|
||||
- Builds on QQQ `4.1.0-RC.7` (the first release with QRun-IO/qqq#1170). RDBMS remains a
|
||||
host-provided runtime dependency.
|
||||
- Alert type `filegateway.lockout` uses the dotted syntax required by core validation.
|
||||
- Object-storage RC.2 controls S3 part size through its connection configuration.
|
||||
- The Preferences "File transfer" section waits for QRun-IO/qqq#1162.
|
||||
- Private core #1170 sends multipart process values without re-uploading the original file.
|
||||
- Core #1170 sends multipart process values without re-uploading the original file.
|
||||
- FieldSecurityLock requires core permission enforcement enabled. Saved-report rendering still
|
||||
uses a temporary file inside core; fully disk-free report generation needs a core change.
|
||||
The report folder is hidden without the render process's configured storage table.
|
||||
@@ -1,4 +1,4 @@
|
||||
# CLAUDE.md
|
||||
# AGENTS.md
|
||||
|
||||
Guidance for coding agents working in this repository.
|
||||
|
||||
@@ -14,13 +14,12 @@ code and QInstance. Design: `docs/DESIGN.md`.
|
||||
## Build commands
|
||||
|
||||
```bash
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT compile
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT test
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
./mvnw compile
|
||||
./mvnw test
|
||||
./mvnw verify # Checkstyle; JaCoCo: 70% instructions, 90% classes
|
||||
```
|
||||
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd` and quote the `-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT` argument in PowerShell.
|
||||
Keep the POM default at `4.1.0-SNAPSHOT` for CI; the private version prevents local artifact replacement.
|
||||
Use the Maven 3.9.11 wrapper (Java 21); Maven 3.10 cannot read the parent POM. On Windows use `./mvnw.cmd`.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -69,8 +68,12 @@ Phase (e) adds endpoints and chroot (`AliasCustomizer` validates paths), `outbox
|
||||
decrypt/encrypt, keys in memory), `maintenance/` (daily retention process and record-action
|
||||
effects), `FileGatewayHome` (typed STATISTICS and TABLE widgets only; guardrail G5),
|
||||
`FileGatewayIntegrations` (ESB, Quick Search, API lists) and `file_gateway_server` heartbeats.
|
||||
The Preferences "File transfer" section waits for QRun-IO/qqq#1162. The k8s example is in
|
||||
`docs/k8s/`.
|
||||
The `file-transfer` Preferences section requires only `file_gateway.connect`. Its provider uses
|
||||
core's trusted scope with the original session, derives identity only from the request context,
|
||||
and manages the caller's unrevoked USER-account keys through normal table actions. Add key
|
||||
creates an account on first use; Revoke sets `revoked_at`; PATCH edits labels and expiry.
|
||||
Optional `publicHostName` advertises the hostname; port/fingerprint use the latest heartbeat.
|
||||
The k8s example is in `docs/k8s/`.
|
||||
|
||||
Tests run a strict UI API server, H2 from the adopter DDL and real MINA clients. NoDiskExtension
|
||||
snapshots the working tree (excluding target) and isolated test home, and asserts the test temp
|
||||
@@ -92,7 +95,7 @@ directory is empty. The host supplies RDBMS; `qqq-backend-module-rdbms` must rem
|
||||
|
||||
## Dependencies
|
||||
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-SNAPSHOT` until 4.1.0-RC.6.
|
||||
Parent: `com.kingsrook:qbit-build-parent:2.0.0`. QQQ BOM: `4.1.0-RC.7` (first release with QRun-IO/qqq#1170); never a SNAPSHOT.
|
||||
Only allow-listed qbit API artifacts (`qbit-object-storage` `api` package) may be depended on.
|
||||
Notifications go through core `QAlerts.notify`, never qbit-alerts classes.
|
||||
|
||||
|
||||
@@ -6,9 +6,9 @@ virtual folder tree. Application metadata and saved objects are read through the
|
||||
|
||||
Phases (a) through (c) implement authentication, listing/stat, account administration, uploads into
|
||||
saved bulk-load profiles, background consumer delivery, readable sidecars/results/archive and
|
||||
saved-view/report downloads. Endpoint routing/chroot, outbox delivery, preferences and the app
|
||||
home follow in later phases. No native filesystem, host-key file, home directory or spool is
|
||||
used by the gateway. See [the approved design](docs/DESIGN.md).
|
||||
saved-view/report downloads. Phase (e) adds endpoint routing/chroot, outbox delivery, PGP,
|
||||
Preferences, retention and the app home. No native filesystem, host-key file, home directory
|
||||
or spool is used by the gateway. See [the approved design](docs/DESIGN.md).
|
||||
|
||||
## Requirements
|
||||
|
||||
@@ -17,8 +17,8 @@ backend, qbit-object-storage 0.1.0-RC.2, and core saved bulk-load profile, saved
|
||||
report metadata. The gateway's
|
||||
RDBMS module dependency is test-scoped. Maven 3.9.11 is pinned by the wrapper.
|
||||
|
||||
Until the invoker core PR merges, local builds use the private `4.1.0-FG-SNAPSHOT` artifacts.
|
||||
The POM default remains `4.1.0-SNAPSHOT` for CI, to be replaced by 4.1.0-RC.6 when released.
|
||||
QQQ is pinned to `4.1.0-RC.7`, the first release with the UiApiInvoker in-process sessions,
|
||||
streams and multipart (QRun-IO/qqq#1170).
|
||||
|
||||
## Install
|
||||
|
||||
@@ -36,6 +36,10 @@ Apply [docs/ddl/file_gateway.sql](docs/ddl/file_gateway.sql) to the host's datab
|
||||
H2 and PostgreSQL and stores UTC timestamps without a zone, matching QQQ's RDBMS binding.
|
||||
The gateway does not create or migrate tables at runtime.
|
||||
|
||||
Register core's saved views, saved reports and saved bulk-load profiles (if the app uses them)
|
||||
before the gateway producer: endpoint pickers for a source the app hasn't registered become plain
|
||||
id fields, so the instance still validates.
|
||||
|
||||
Register metadata while building the QInstance:
|
||||
|
||||
```java
|
||||
@@ -68,6 +72,25 @@ least 3072 bits. Public-key validation runs on insert and update and computes it
|
||||
fingerprint, type and size. The host authentication module must assign the user's permissions
|
||||
when `createAutomatedSessionForUser` is called, including `file_gateway.connect`.
|
||||
|
||||
## Preferences
|
||||
|
||||
Users with `file_gateway.connect` manage their own SSH keys in **Preferences → File transfer**.
|
||||
They need no File Gateway admin or table permissions. **Add key** accepts an OpenSSH public key,
|
||||
a required label and optional expiry. The first successful key creates a USER account with
|
||||
FULL_TREE access, the configured account defaults and a unique login derived from the user ID.
|
||||
The section displays that login, plus the port and host key fingerprint from the latest gateway
|
||||
heartbeat. Set `.withPublicHostName("sftp.example.com")` on the shared config to display the
|
||||
public hostname; it defaults to null and is omitted when unset. The heartbeat port is the gateway
|
||||
listener port; deployments publishing a different external port must communicate that mapping.
|
||||
|
||||
The keys list contains only the user's unrevoked keys. Users can edit labels and expiry dates,
|
||||
or choose **Revoke**. Revoked keys cannot sign in; existing sessions close at the next revocation
|
||||
poll (normally within 60 seconds). Fingerprints, key types and last-used times are read-only.
|
||||
Ed25519, ECDSA and RSA of at least 3072 bits are supported. Duplicate keys are rejected with a
|
||||
validation message. Normal table customizers and configured audits apply, attributed to the
|
||||
signed-in user and client channel. Administrators manage service accounts and other users' keys
|
||||
through the standard account and SSH-key tables.
|
||||
|
||||
## Tables and permissions
|
||||
|
||||
All seven tables are registered, with camelCase QQQ fields mapped to snake_case database columns.
|
||||
@@ -243,6 +266,7 @@ present without enabling those features.
|
||||
| Options | Defaults |
|
||||
|---|---|
|
||||
| `backendName`, `sftpEnabled`, `sftpPort` | `rdbms`, `false`, `2222` |
|
||||
| `publicHostName` | null (omit host from Preferences) |
|
||||
| `hostKeyEnvVarName`, `allowGeneratedHostKey` | `FILE_GATEWAY_SSH_HOST_KEY`, `false` |
|
||||
| `lockoutThreshold`, `lockoutWindow`, `lockoutDuration` | 5, 15 minutes, 30 minutes |
|
||||
| `maxConnectionsPerIpPerMinute` | 30 |
|
||||
@@ -325,13 +349,13 @@ port 22 only (no passive range: v1 is SFTP only) with `externalTrafficPolicy: Lo
|
||||
## Build and acceptance tests
|
||||
|
||||
```bash
|
||||
./mvnw -o -Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT verify
|
||||
./mvnw verify
|
||||
```
|
||||
|
||||
PowerShell:
|
||||
|
||||
```powershell
|
||||
./mvnw.cmd -o '-Dqqq.snapshot.version=4.1.0-FG-SNAPSHOT' verify
|
||||
./mvnw.cmd verify
|
||||
```
|
||||
|
||||
The tests use H2 from the adopter DDL, a strict UI API server, generated in-memory SSH keys and
|
||||
|
||||
+34
-14
@@ -250,8 +250,8 @@ in-process session token; there is never a service session.
|
||||
(`ProcessSpecUtilsV1.addUploadedFiles`).
|
||||
|
||||
### Shipping
|
||||
§3.1–3.3 go in one core PR to `QRun-IO/qqq` `develop`, for 4.1.0-RC.6. Until then the qbit builds
|
||||
on `4.1.0-SNAPSHOT`, with the pom marked `STAND-IN(removeBy=4.1.0-RC.6)`.
|
||||
§3.1–3.3 shipped in QRun-IO/qqq#1170 (merged 2026-10-14), first released in QQQ 4.1.0-RC.7, which
|
||||
the qbit pins.
|
||||
|
||||
### 3.4 Permissions for automated sessions (qbit-authentik)
|
||||
`AuthentikGroupPermissionMapper` builds permissions from token claims, and an automated session
|
||||
@@ -265,13 +265,30 @@ has no token.
|
||||
- **Who needs it.** It is a qbit-authentik PR. The gateway works without it under MOCK auth and in
|
||||
tests, but kof22 needs it before going live.
|
||||
|
||||
### 3.5 Preferences (QRun-IO/qqq#1162)
|
||||
The gateway registers a "File transfer" section with the provider `FileTransferPreferencesProvider`.
|
||||
### 3.5 Preferences
|
||||
The gateway registers `file-transfer` ("File transfer", order 60) with
|
||||
`FileTransferPreferencesProvider`, using core's user-preferences API. Only users with
|
||||
`file_gateway.connect` see it; no admin-app or backing-table permissions are needed.
|
||||
|
||||
- **Scalar fields:** the SFTP username (read-only), "Add a public key" (TEXT) and "Key label".
|
||||
- **The `keys` list:** label, fingerprint (read-only), last used (read-only), expires and revoked.
|
||||
- **Behaviour.** The first key creates the user's `USER` account. Revoking a key sets `revoked_at`.
|
||||
- **Until #1162 is in the baseline,** admins add keys on the account record.
|
||||
- **Read-only connection details:** the account username, plus port and host key fingerprint from
|
||||
the most recent `file_gateway_server` heartbeat. The optional `publicHostName` config supplies the host; when
|
||||
unset, the host field is omitted. Before the first key, username is blank and the keys list
|
||||
explains that adding a key creates the account.
|
||||
- **The `keys` list:** only the caller's unrevoked USER-account keys. Label and expiry are editable
|
||||
through PATCH; fingerprint, key type and last-used time are read-only. Public-key text is never
|
||||
returned in preference values.
|
||||
- **Add key (`addKey`, LIST action):** requires OpenSSH public-key text and a label, with optional
|
||||
expiry. The first successful addition creates a USER account with FULL_TREE and configured
|
||||
account defaults. Its login is derived from the session user ID, normalized to lowercase
|
||||
`[a-z0-9._-]`, at most 64 characters; short IDs receive a `user-` prefix. Collisions receive
|
||||
`-2`, `-3`, etc. Subsequent keys reuse that account. The result names the computed fingerprint.
|
||||
- **Revoke (`revokeKey`, ROW action):** confirms revocation, sets `revoked_at`, and removes the
|
||||
key from the list. New logins fail immediately; existing sessions end on the next gateway poll.
|
||||
- **Persistence:** ordinary table actions retain `SshKeyCustomizer` validation and configured
|
||||
audits under the original user/client channel. Weak and duplicate keys return actionable 400
|
||||
messages. Failed first-key inserts roll back account creation. Mutations lock and recheck
|
||||
ownership; another user's key is not a setting here. Identity comes only from core's request
|
||||
context, never from the request body. Service-account keys remain administrator-managed.
|
||||
|
||||
## 4. Upload
|
||||
1. **Receive.** The SFTP write streams straight to object storage:
|
||||
@@ -441,9 +458,11 @@ sized for the largest report. The no-disk test asserts the temp area is empty af
|
||||
(§15) would add only a server and an authenticator.
|
||||
|
||||
## 7. Audit
|
||||
- **Connections.** Every connection writes `file_gateway_connection_log`. The row is opened at TCP
|
||||
accept and updated at auth and at disconnect. It records the outcome, IP, key fingerprint, SSH
|
||||
client version, byte and operation counts, and channel `sftp`.
|
||||
- **Connections.** Every sign-in attempt writes `file_gateway_connection_log`. The row is opened
|
||||
at the first authentication attempt (or a pre-auth refusal such as the connection rate limit)
|
||||
and updated at auth and at disconnect. It records the outcome, IP, key fingerprint, SSH client
|
||||
version, byte and operation counts, and channel `sftp`. A TCP connection that never tries to
|
||||
sign in, such as a kubelet health probe or a port scan, leaves no row.
|
||||
- **Operations.** Every list, upload, download, rename and remove writes
|
||||
`file_gateway_transfer_log`. It records the operation, path, object kind and id, bytes, sha256,
|
||||
duration, outcome and message, with channel `sftp/<sessionId>`.
|
||||
@@ -569,6 +588,7 @@ FileGatewayConsumersMetaData.ofOrWithNew(qInstance).withConsumer(new QCodeRefere
|
||||
- `sftpEnabled` (default false; true only in the gateway deployment, from
|
||||
`FILE_GATEWAY_SFTP_ENABLED`);
|
||||
- `sftpPort` (2222);
|
||||
- `publicHostName` (null; optional public hostname shown in Preferences);
|
||||
- `hostKeyEnvVarName` (`FILE_GATEWAY_SSH_HOST_KEY`);
|
||||
- `allowGeneratedHostKey` (false).
|
||||
- **Storage and buffers:**
|
||||
@@ -678,10 +698,10 @@ All tests run in the qbit's build.
|
||||
`Class.forName` of another qbit. The gateway's own `api` package is the published contract.
|
||||
- **G2.** Server tests run in strict mode.
|
||||
- **G3.** No extension is registered: the gateway is a UI API client.
|
||||
- **G4.** Stand-ins are marked `STAND-IN(core=..., removeBy=...)`: the 4.1.0-SNAPSHOT baseline
|
||||
until RC.6, and the preferences section until #1162.
|
||||
- **G4.** No stand-ins remain: the core baseline is a release, and the preferences section uses
|
||||
core's API from QRun-IO/qqq#1162.
|
||||
- **G5.** No HTML widgets.
|
||||
- **G7.** Pin 4.1.0-RC.6 as soon as it is cut. Until then, develop builds on the SNAPSHOT.
|
||||
- **G7.** Pinned to QQQ 4.1.0-RC.7; never a SNAPSHOT.
|
||||
|
||||
## 14. Phases
|
||||
- **(a)** The SFTP server, the virtual filesystem, and auth:
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
|
||||
<revision>0.1.0-SNAPSHOT</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
@@ -33,7 +34,7 @@
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>2.2.1</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
@@ -81,7 +82,7 @@
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
<artifactId>qbit-object-storage</artifactId>
|
||||
<version>0.1.0-RC.2</version>
|
||||
<version>0.1.0-RC.10</version>
|
||||
</dependency>
|
||||
<!-- MINA's Ed25519 key decoder and signer. -->
|
||||
<dependency>
|
||||
@@ -117,7 +118,7 @@
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-frontend-next</artifactId>
|
||||
<version>1.0.0-RC.14</version>
|
||||
<version>1.0.0-RC.38</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -238,8 +239,8 @@
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<!-- STAND-IN(core=QRun-IO/qqq develop after v4.1.0-RC.5, removeBy=4.1.0-RC.6): UiApiInvoker client channels and QAlerts.notify are on develop only; pin the RC when it is cut -->
|
||||
<qqq.snapshot.version>4.1.0-SNAPSHOT</qqq.snapshot.version>
|
||||
<!-- QQQ 4.1.0-RC.7: the first released core with the UiApiInvoker in-process sessions, streams and multipart (QRun-IO/qqq#1170). Never a SNAPSHOT (guardrail G7). -->
|
||||
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -39,6 +39,7 @@ public class FileGatewayQBitConfig implements QBitConfig
|
||||
private String backendName = "rdbms";
|
||||
private boolean sftpEnabled = false;
|
||||
private int sftpPort = 2222;
|
||||
private String publicHostName = null;
|
||||
private String hostKeyEnvVarName = "FILE_GATEWAY_SSH_HOST_KEY";
|
||||
private boolean allowGeneratedHostKey = false;
|
||||
private String fileGatewayBucketAlias = "file-gateway";
|
||||
@@ -123,6 +124,27 @@ public class FileGatewayQBitConfig implements QBitConfig
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Public SFTP hostname advertised in Preferences, when configured.
|
||||
*******************************************************************************/
|
||||
public String getPublicHostName()
|
||||
{
|
||||
return (publicHostName);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set the public SFTP hostname; null omits it from Preferences.
|
||||
*******************************************************************************/
|
||||
public FileGatewayQBitConfig withPublicHostName(String publicHostName)
|
||||
{
|
||||
this.publicHostName = publicHostName;
|
||||
return (this);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for sftpPort.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
@@ -43,6 +44,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
*******************************************************************************/
|
||||
public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGatewayQBitConfig>
|
||||
{
|
||||
/** Core possible-value sources a host may or may not register. */
|
||||
static final java.util.Set<String> OPTIONAL_HOST_SOURCES = java.util.Set.of("savedView", "savedReport", "savedBulkLoadProfile");
|
||||
|
||||
/*******************************************************************************
|
||||
** The sftp wall owns its list (core's qbit prefixes are transitional and the
|
||||
** assistant may now read audits): partners never reach identity, audits,
|
||||
@@ -149,6 +153,7 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
|
||||
table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
|
||||
}
|
||||
table.setSourceQBitName(qBit.getName());
|
||||
dropMissingPickers(qInstance, table);
|
||||
output.add(table);
|
||||
if(FileGatewayTables.MAIN.contains(table.getName()))
|
||||
{
|
||||
@@ -171,6 +176,7 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
|
||||
.withSourceQBitName(qBit.getName()).withSubjectTableName(FileGatewayTables.ACCOUNT)
|
||||
.withVisibilityPermission(FileGatewayPermissions.VIEW));
|
||||
output.addSelfToInstance(qInstance);
|
||||
com.kingsrook.qbits.filegateway.preferences.FileTransferPreferencesProvider.register(qInstance, qBitConfig);
|
||||
ClientChannelsMetaData channels = ClientChannelsMetaData.ofOrWithNew(qInstance)
|
||||
.withChannel("sftp", "SFTP file gateway").withDeniedPattern("sftp", "file_gateway_*")
|
||||
.withDeniedPattern("assistant", "file_gateway_*");
|
||||
@@ -186,4 +192,25 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
|
||||
new com.kingsrook.qbits.filegateway.storage.ObjectStorageGatewayStore(qInstance, qBitConfig.getFileGatewayBucketAlias()), qBitConfig, java.time.Clock.systemUTC()));
|
||||
return (new MetaDataProducerMultiOutput());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Saved views, saved reports and saved bulk-load profiles are optional in a
|
||||
** host. A picker whose possible-value source the host has not registered (yet)
|
||||
** stays a plain id field, so the instance validates; register those core
|
||||
** metadata providers before this producer to get the pickers.
|
||||
*******************************************************************************/
|
||||
static void dropMissingPickers(QInstance qInstance, QTableMetaData table)
|
||||
{
|
||||
for(QFieldMetaData field : table.getFields().values())
|
||||
{
|
||||
String source = field.getPossibleValueSourceName();
|
||||
if(source != null && OPTIONAL_HOST_SOURCES.contains(source) && qInstance.getPossibleValueSource(source) == null)
|
||||
{
|
||||
field.setPossibleValueSourceName(null);
|
||||
field.setPossibleValueSourceFilter(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -108,13 +108,13 @@ public class FileGatewayHome extends AbstractWidgetRenderer
|
||||
long value = "transfers".equals(key) ? rows.size() : rows.stream()
|
||||
.filter(row -> ("in".equals(key) ? "UPLOAD" : "DOWNLOAD").equals(row.getValueString("operation")))
|
||||
.mapToLong(row -> number(row, "bytes")).sum();
|
||||
return (new StatisticsData("transfers".equals(key) ? value : Math.round(value / 10485.76) / 100.0, null, null));
|
||||
return (new StatisticsData("transfers".equals(key) ? value : Math.round(value / 10485.76) / 100.0, null, null).withIncreaseIsGood(true));
|
||||
}
|
||||
if("locked".equals(key))
|
||||
{
|
||||
long count = query(FileGatewayTables.ACCOUNT, new QQueryFilter()).stream().filter(row -> "LOCKED".equals(row.getValueString("status"))
|
||||
|| (row.getValueInstant("lockedUntil") != null && row.getValueInstant("lockedUntil").isAfter(now))).count();
|
||||
return (new StatisticsData(count, null, null));
|
||||
return (new StatisticsData(count, null, null).withIncreaseIsGood(false));
|
||||
}
|
||||
if(List.of("failures", "recent").contains(key))
|
||||
{
|
||||
@@ -122,24 +122,26 @@ public class FileGatewayHome extends AbstractWidgetRenderer
|
||||
.withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, "FAILED", "PARTIAL", "REJECTED")));
|
||||
if("failures".equals(key))
|
||||
{
|
||||
return (new StatisticsData(rows.size(), null, null));
|
||||
return (new StatisticsData(rows.size(), null, null).withIncreaseIsGood(false));
|
||||
}
|
||||
rows.sort(java.util.Comparator.comparing((QRecord row) -> row.getValueInstant("finishedAt")).reversed());
|
||||
return (table("Recent failures", FileGatewayTables.UPLOAD_JOB, rows.stream().limit(20).toList(),
|
||||
List.of("fileName", "status", "errorMessage", "finishedAt"), List.of("File", "Status", "Message", "Finished")));
|
||||
List.of("fileName", "status", "errorMessage", "finishedAt"), List.of("File", "Status", "Message", "Finished"), "No failed transfers in the last 7 days."));
|
||||
}
|
||||
if(List.of("sessions", "active").contains(key))
|
||||
{
|
||||
var rows = query(FileGatewayTables.CONNECTION_LOG, since("lastSeenAt", now.minusSeconds(180))
|
||||
.withCriteria(new QFilterCriteria("disconnectedAt", QCriteriaOperator.IS_BLANK))
|
||||
.withCriteria(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "CONNECTED")));
|
||||
return ("sessions".equals(key) ? new StatisticsData(rows.size(), null, null) : table("Active sessions", FileGatewayTables.CONNECTION_LOG, rows,
|
||||
List.of("username", "remoteIp", "connectedAt", "operationCount", "podName"), List.of("Username", "IP", "Connected", "Operations", "Pod")));
|
||||
return ("sessions".equals(key) ? new StatisticsData(rows.size(), null, null).withIncreaseIsGood(true) : table("Active sessions", FileGatewayTables.CONNECTION_LOG, rows,
|
||||
List.of("username", "remoteIp", "connectedAt", "operationCount"), List.of("Username", "IP", "Connected", "Operations"),
|
||||
"No one is connected right now (no session has been seen in the last 3 minutes)."));
|
||||
}
|
||||
if("servers".equals(key))
|
||||
{
|
||||
return (table("Server", FileGatewayTables.SERVER, query(FileGatewayTables.SERVER, since("lastSeenAt", now.minusSeconds(180))),
|
||||
List.of("podName", "hostKeyFingerprint", "port", "lastSeenAt"), List.of("Pod", "Host key fingerprint", "Port", "Last seen")));
|
||||
List.of("podName", "hostKeyFingerprint", "port", "lastSeenAt"), List.of("Pod", "Host key fingerprint", "Port", "Last seen"),
|
||||
"No gateway server has reported in during the last 3 minutes."));
|
||||
}
|
||||
Map<Integer, QRecord> partners = new LinkedHashMap<>();
|
||||
for(QRecord account : query(FileGatewayTables.ACCOUNT, new QQueryFilter()))
|
||||
@@ -157,7 +159,8 @@ public class FileGatewayHome extends AbstractWidgetRenderer
|
||||
}
|
||||
List<QRecord> rows = partners.values().stream().filter(row -> number(row, "transfers") > 0)
|
||||
.sorted(java.util.Comparator.comparingLong((QRecord row) -> number(row, "transfers")).reversed()).limit(10).toList();
|
||||
return (table("Top partners (7 days)", FileGatewayTables.ACCOUNT, rows, List.of("username", "transfers", "mb"), List.of("Partner", "Transfers", "MB")));
|
||||
return (table("Top partners (7 days)", FileGatewayTables.ACCOUNT, rows, List.of("username", "transfers", "mb"), List.of("Partner", "Transfers", "MB"),
|
||||
"No partner has transferred a file in the last 7 days."));
|
||||
}
|
||||
|
||||
|
||||
@@ -165,12 +168,12 @@ public class FileGatewayHome extends AbstractWidgetRenderer
|
||||
/*******************************************************************************
|
||||
** Typed text blocks carry record links; cells never contain markup.
|
||||
*******************************************************************************/
|
||||
private TableData table(String label, String table, List<QRecord> records, List<String> fields, List<String> labels)
|
||||
private TableData table(String label, String table, List<QRecord> records, List<String> fields, List<String> labels, String noRowsText)
|
||||
{
|
||||
List<TableData.Column> columns = new ArrayList<>();
|
||||
for(int i = 0; i < fields.size(); i++)
|
||||
{
|
||||
columns.add(new TableData.Column(i == 0 ? "block" : "default", labels.get(i), fields.get(i), "1fr", null));
|
||||
columns.add(new TableData.Column(i == 0 ? "block" : "default", labels.get(i), fields.get(i), null, null));
|
||||
}
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
for(QRecord record : records)
|
||||
@@ -185,7 +188,7 @@ public class FileGatewayHome extends AbstractWidgetRenderer
|
||||
.withLink("/record/" + table + "/" + record.getValue("id")));
|
||||
rows.add(row);
|
||||
}
|
||||
return (new TableData(null, columns, rows).withRowsPerPage(10).withHidePaginationDropdown(true));
|
||||
return (new TableData(null, columns, rows).withRowsPerPage(10).withHidePaginationDropdown(true).withNoRowsFoundHTML(noRowsText));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -297,7 +297,7 @@ public final class FileGatewayTables
|
||||
*******************************************************************************/
|
||||
private static QTableMetaData upload_job(FileGatewayQBitConfig config)
|
||||
{
|
||||
QTableMetaData table = base(config, UPLOAD_JOB, "Upload Jobs").withRecordVersionField("status")
|
||||
QTableMetaData table = base(config, UPLOAD_JOB, "Upload Jobs")
|
||||
.withRecordLabelFormatAndFields("%s", "id")
|
||||
.withField(field("accountId", QFieldType.INTEGER).withPossibleValueSourceName(ACCOUNT))
|
||||
.withField(field("aliasId", QFieldType.INTEGER).withPossibleValueSourceName(ALIAS))
|
||||
|
||||
@@ -22,12 +22,21 @@ package com.kingsrook.qbits.filegateway.metadata;
|
||||
|
||||
|
||||
import java.security.PublicKey;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Optional;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
@@ -47,8 +56,9 @@ public class SshKeyCustomizer implements TableCustomizerInterface
|
||||
** Parse and validate inserts and partial updates without logging key material.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecords)
|
||||
public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecords) throws QException
|
||||
{
|
||||
Set<String> fingerprints = new HashSet<>();
|
||||
for(QRecord record : records)
|
||||
{
|
||||
QRecord old = oldRecords.orElse(List.of()).stream().filter(candidate -> Objects.equals(candidate.getValue("id"), record.getValue("id"))).findFirst().orElse(null);
|
||||
@@ -69,6 +79,19 @@ public class SshKeyCustomizer implements TableCustomizerInterface
|
||||
catch(Exception e)
|
||||
{
|
||||
record.addError(new BadInputStatusMessage("Use an OpenSSH Ed25519, ECDSA, or RSA key of at least 3072 bits"));
|
||||
continue;
|
||||
}
|
||||
// Core's unique-key check precedes this computed fingerprint.
|
||||
QQueryFilter filter = new QQueryFilter().withCriteria("fingerprint", QCriteriaOperator.EQUALS, record.getValue("fingerprint"));
|
||||
if(old != null)
|
||||
{
|
||||
filter.withCriteria("id", QCriteriaOperator.NOT_EQUALS, old.getValue("id"));
|
||||
}
|
||||
var transaction = input instanceof InsertInput insert ? insert.getTransaction() : ((UpdateInput) input).getTransaction();
|
||||
if(!fingerprints.add(record.getValueString("fingerprint")) || !new QueryAction().execute(new QueryInput(FileGatewayTables.SSH_KEY)
|
||||
.withFilter(filter.withLimit(1)).withTransaction(transaction)).getRecords().isEmpty())
|
||||
{
|
||||
record.addError(new BadInputStatusMessage("Fingerprint: this SSH key is already registered. Use a different key."));
|
||||
}
|
||||
}
|
||||
return (records);
|
||||
|
||||
+43
@@ -22,14 +22,19 @@ package com.kingsrook.qbits.filegateway.permissions;
|
||||
|
||||
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.AvailablePermission;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionSubType;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
@@ -60,4 +65,42 @@ public class FileGatewayPermissionChecker implements CustomPermissionChecker
|
||||
throw (new QPermissionDeniedException("Missing file gateway permission"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** This checker gates on the three gateway permission names, not on per-object
|
||||
** names, so it tells the catalog (getAllAvailablePermissions) those names.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public boolean handlesBuildAvailablePermission()
|
||||
{
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Collapse every gateway object onto file_gateway.view, .admin or .connect.
|
||||
** The object name and type are fixed so the catalog (a set) lists each name
|
||||
** once: reads and the app need view, writes and processes need admin, and the
|
||||
** app also stands for SFTP sign-in (connect), which the gateway grants to
|
||||
** sessions rather than checking on an object.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public AvailablePermission buildAvailablePermission(QPermissionRules rules, PermissionSubType permissionSubType, String baseName, MetaDataWithName metaDataWithName, String objectType)
|
||||
{
|
||||
AvailablePermission base = buildBaseAvailablePermission(rules, permissionSubType, metaDataWithName, objectType);
|
||||
if(base == null)
|
||||
{
|
||||
return (null);
|
||||
}
|
||||
boolean write = "WRITE".equals(base.getPermissionType()) || "Process".equals(objectType);
|
||||
String name = write ? FileGatewayPermissions.ADMIN : FileGatewayPermissions.VIEW;
|
||||
if(metaDataWithName instanceof QAppMetaData)
|
||||
{
|
||||
name = FileGatewayPermissions.CONNECT;
|
||||
}
|
||||
return (base.withName(name).withObjectName("File Gateway").withObjectType("App").withPermissionType(name.substring(name.indexOf('.') + 1)));
|
||||
}
|
||||
}
|
||||
+411
@@ -0,0 +1,411 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway.preferences;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionRequest;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionScope;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceChange;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceListMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRequestContext;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRow;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceRowChange;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceSectionMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceValues;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferencesMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Self-service SSH keys. Core supplies the trusted scope and retains the user
|
||||
** and client channel; this provider never switches to a system session.
|
||||
*******************************************************************************/
|
||||
public class FileTransferPreferencesProvider implements UserPreferenceProviderInterface
|
||||
{
|
||||
public static final String SECTION_NAME = "file-transfer";
|
||||
private static final List<String> KEY_FIELDS = List.of("label", "fingerprint", "keyType", "lastUsedAt", "expiresAt");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Register without an app scope: connect permission is sufficient.
|
||||
*******************************************************************************/
|
||||
public static void register(QInstance instance, FileGatewayQBitConfig config)
|
||||
{
|
||||
UserPreferenceSectionMetaData section = new UserPreferenceSectionMetaData()
|
||||
.withName(SECTION_NAME).withLabel("File transfer").withIcon(new QIcon("swap_vert")).withOrder(60)
|
||||
.withDescription("Sign in to the SFTP file gateway with your SSH keys.")
|
||||
.withPermissionName(FileGatewayPermissions.CONNECT).withProvider(new QCodeReference(FileTransferPreferencesProvider.class))
|
||||
.withField(new QFieldMetaData("username", QFieldType.STRING).withLabel("Username").withIsEditable(false)
|
||||
.withHelpContent(new QHelpContent("Adding your first key creates your SFTP account and username.")))
|
||||
.withField(new QFieldMetaData("port", QFieldType.INTEGER).withLabel("Port").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("hostKeyFingerprint", QFieldType.STRING).withLabel("Host key fingerprint").withIsEditable(false))
|
||||
.withList(new UserPreferenceListMetaData().withName("keys").withLabel("SSH keys").withRowLabel("Key")
|
||||
.withDescription("Adding your first key creates your SFTP account.")
|
||||
.withField(new QFieldMetaData("label", QFieldType.STRING).withLabel("Label").withIsRequired(true).withMaxLength(1024))
|
||||
.withField(new QFieldMetaData("fingerprint", QFieldType.STRING).withLabel("Fingerprint").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("keyType", QFieldType.STRING).withLabel("Key type").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("lastUsedAt", QFieldType.DATE_TIME).withLabel("Last used").withIsEditable(false))
|
||||
.withField(new QFieldMetaData("expiresAt", QFieldType.DATE_TIME).withLabel("Expires at")))
|
||||
.withAction(new UserPreferenceActionMetaData().withName("addKey").withLabel("Add key")
|
||||
.withScope(UserPreferenceActionScope.LIST).withListName("keys").withOrder(10)
|
||||
.withInputField(new QFieldMetaData("publicKey", QFieldType.TEXT).withLabel("Public key").withIsRequired(true))
|
||||
.withInputField(new QFieldMetaData("label", QFieldType.STRING).withLabel("Label").withIsRequired(true).withMaxLength(1024))
|
||||
.withInputField(new QFieldMetaData("expiresAt", QFieldType.DATE_TIME).withLabel("Expires at")))
|
||||
.withAction(new UserPreferenceActionMetaData().withName("revokeKey").withLabel("Revoke")
|
||||
.withScope(UserPreferenceActionScope.ROW).withListName("keys").withOrder(20).withIsDestructive(true)
|
||||
.withConfirmationText("Revoke this SSH key? It will no longer be able to sign in."));
|
||||
if(config.getPublicHostName() != null && !config.getPublicHostName().isBlank())
|
||||
{
|
||||
section.withField(new QFieldMetaData("host", QFieldType.STRING).withLabel("Host").withIsEditable(false).withDefaultValue(config.getPublicHostName()));
|
||||
}
|
||||
UserPreferencesMetaData.ofOrWithNew(instance).withSection(section);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Publish only the user's active key metadata, never the public-key text.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public UserPreferenceValues load(UserPreferenceRequestContext context) throws QException
|
||||
{
|
||||
QRecord account = account(context, null);
|
||||
UserPreferenceValues result = new UserPreferenceValues();
|
||||
result.getValues().put("username", account == null ? "" : account.getValue("username"));
|
||||
List<QRecord> servers = query(FileGatewayTables.SERVER, new QQueryFilter()
|
||||
.withOrderBy(new QFilterOrderBy("lastSeenAt", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(1), null);
|
||||
if(!servers.isEmpty())
|
||||
{
|
||||
result.getValues().put("port", servers.getFirst().getValue("port"));
|
||||
result.getValues().put("hostKeyFingerprint", servers.getFirst().getValue("hostKeyFingerprint"));
|
||||
}
|
||||
List<UserPreferenceRow> rows = new ArrayList<>();
|
||||
if(account != null)
|
||||
{
|
||||
for(QRecord key : query(FileGatewayTables.SSH_KEY, keysFilter(account), null))
|
||||
{
|
||||
Map<String, Serializable> values = new LinkedHashMap<>();
|
||||
KEY_FIELDS.forEach(field -> values.put(field, key.getValue(field)));
|
||||
rows.add(new UserPreferenceRow().withKey(key.getValueString("id")).withLabel(key.getValueString("label")).withValues(values));
|
||||
}
|
||||
}
|
||||
result.getLists().put("keys", rows);
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Core validates types; reject blank labels as well as missing ones.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<String> validate(UserPreferenceRequestContext context, UserPreferenceChange change)
|
||||
{
|
||||
List<String> errors = new ArrayList<>();
|
||||
for(UserPreferenceRowChange row : change.getLists().getOrDefault("keys", List.of()))
|
||||
{
|
||||
if(row.getValues().containsKey("label") && (row.getValues().get("label") == null || row.getValues().get("label").toString().isBlank()))
|
||||
{
|
||||
errors.add("Label: is required");
|
||||
}
|
||||
}
|
||||
return (errors);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply partial changes atomically, checking ownership again under row locks.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void store(UserPreferenceRequestContext context, UserPreferenceChange change) throws QException
|
||||
{
|
||||
try(var tx = QBackendTransaction.openFor(new QueryInput(FileGatewayTables.ACCOUNT)))
|
||||
{
|
||||
try
|
||||
{
|
||||
QRecord account = lockedAccount(context, tx);
|
||||
for(UserPreferenceRowChange row : change.getLists().getOrDefault("keys", List.of()))
|
||||
{
|
||||
QRecord key = lockedKey(account, row.getKey(), tx);
|
||||
QRecord update = new QRecord().withValue("id", key.getValue("id"));
|
||||
for(String field : List.of("label", "expiresAt"))
|
||||
{
|
||||
if(row.getValues().containsKey(field))
|
||||
{
|
||||
update.setValue(field, row.getValues().get(field));
|
||||
}
|
||||
}
|
||||
update(FileGatewayTables.SSH_KEY, update, tx);
|
||||
}
|
||||
tx.commit();
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
tx.rollback();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add through the normal key customizer, or revoke an owned, unrevoked row.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public UserPreferenceActionResult runAction(UserPreferenceRequestContext context, UserPreferenceActionRequest request) throws QException
|
||||
{
|
||||
if(!List.of("addKey", "revokeKey").contains(request.getActionName()))
|
||||
{
|
||||
throw (new QBadRequestException("This action isn't available."));
|
||||
}
|
||||
if("addKey".equals(request.getActionName()))
|
||||
{
|
||||
List<String> errors = new ArrayList<>();
|
||||
for(String field : List.of("publicKey", "label"))
|
||||
{
|
||||
if(request.getValues().get(field) == null || request.getValues().get(field).toString().isBlank())
|
||||
{
|
||||
errors.add(("publicKey".equals(field) ? "Public key" : "Label") + ": is required");
|
||||
}
|
||||
}
|
||||
if(!errors.isEmpty())
|
||||
{
|
||||
throw (new QBadRequestException(String.join("\n", errors)));
|
||||
}
|
||||
}
|
||||
try(var tx = QBackendTransaction.openFor(new QueryInput(FileGatewayTables.ACCOUNT)))
|
||||
{
|
||||
try
|
||||
{
|
||||
QRecord account = lockedAccount(context, tx);
|
||||
UserPreferenceActionResult result;
|
||||
if("addKey".equals(request.getActionName()))
|
||||
{
|
||||
if(account == null)
|
||||
{
|
||||
account = insert(FileGatewayTables.ACCOUNT, new QRecord().withValue("username", availableUsername(context.getUserId(), tx))
|
||||
.withValue("userId", context.getUserId()).withValue("accountType", "USER").withValue("rootMode", "FULL_TREE"), tx);
|
||||
}
|
||||
QRecord key = insert(FileGatewayTables.SSH_KEY, new QRecord().withValue("accountId", account.getValue("id"))
|
||||
.withValue("publicKey", request.getValues().get("publicKey")).withValue("label", request.getValues().get("label"))
|
||||
.withValue("expiresAt", request.getValues().get("expiresAt")), tx);
|
||||
result = new UserPreferenceActionResult().withMessage("Added key " + key.getValueString("fingerprint") + ".").withFocusRowKey(key.getValueString("id"));
|
||||
}
|
||||
else
|
||||
{
|
||||
QRecord key = lockedKey(account, request.getRowKey(), tx);
|
||||
update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", key.getValue("id")).withValue("revokedAt", Instant.now()), tx);
|
||||
result = new UserPreferenceActionResult().withMessage("Key revoked.");
|
||||
}
|
||||
tx.commit();
|
||||
return (result);
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
tx.rollback();
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Use only the session identity; service accounts stay in the admin tables.
|
||||
*******************************************************************************/
|
||||
private QRecord account(UserPreferenceRequestContext context, QBackendTransaction tx) throws QException
|
||||
{
|
||||
List<QRecord> rows = query(FileGatewayTables.ACCOUNT, new QQueryFilter()
|
||||
.withCriteria("userId", QCriteriaOperator.EQUALS, context.getUserId())
|
||||
.withCriteria("accountType", QCriteriaOperator.EQUALS, "USER").withOrderBy(new QFilterOrderBy("id", true)).withLimit(1), tx);
|
||||
return (rows.isEmpty() ? null : rows.getFirst());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A transactional timestamp update acquires the same lock used by gateway auth.
|
||||
** Re-read after the lock in case an administrator changed ownership meanwhile.
|
||||
*******************************************************************************/
|
||||
private QRecord lockedAccount(UserPreferenceRequestContext context, QBackendTransaction tx) throws QException
|
||||
{
|
||||
QRecord account = account(context, tx);
|
||||
if(account != null)
|
||||
{
|
||||
update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", account.getValue("id")).withValue("modifyDate", Instant.now()), tx);
|
||||
QRecord current = account(context, tx);
|
||||
if(current == null || !Objects.equals(current.getValue("id"), account.getValue("id")))
|
||||
{
|
||||
throw (new QBadRequestException("Account: isn't a setting here"));
|
||||
}
|
||||
return (current);
|
||||
}
|
||||
return (null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scope every key lookup to the caller's account and unrevoked rows.
|
||||
*******************************************************************************/
|
||||
private QQueryFilter keysFilter(QRecord account)
|
||||
{
|
||||
return (new QQueryFilter().withCriteria("accountId", QCriteriaOperator.EQUALS, account.getValue("id"))
|
||||
.withCriteria("revokedAt", QCriteriaOperator.IS_BLANK).withOrderBy(new QFilterOrderBy("id", true)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check ownership both before and after locking the key against reassignment.
|
||||
*******************************************************************************/
|
||||
private QRecord lockedKey(QRecord account, String rowKey, QBackendTransaction tx) throws QException
|
||||
{
|
||||
QRecord key = ownKey(account, rowKey, tx);
|
||||
update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", key.getValue("id")).withValue("modifyDate", Instant.now()), tx);
|
||||
return (ownKey(account, rowKey, tx));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Do not interpret an arbitrary caller-supplied key as an unrestricted ID.
|
||||
*******************************************************************************/
|
||||
private QRecord ownKey(QRecord account, String rowKey, QBackendTransaction tx) throws QException
|
||||
{
|
||||
if(account != null)
|
||||
{
|
||||
for(QRecord key : query(FileGatewayTables.SSH_KEY, keysFilter(account), tx))
|
||||
{
|
||||
if(key.getValueString("id").equals(rowKey))
|
||||
{
|
||||
return (key);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw (new QBadRequestException("Key: isn't a setting here"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Normalize opaque user IDs to the account validator's 3-64 character syntax.
|
||||
** Truncate before suffixing, so even a 64-character collision stays valid.
|
||||
*******************************************************************************/
|
||||
private String availableUsername(String userId, QBackendTransaction tx) throws QException
|
||||
{
|
||||
String base = userId.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]", "-");
|
||||
if(base.length() < 3)
|
||||
{
|
||||
base = "user-" + base;
|
||||
}
|
||||
base = base.substring(0, Math.min(64, base.length()));
|
||||
String username = base;
|
||||
int suffix = 2;
|
||||
while(!query(FileGatewayTables.ACCOUNT, new QQueryFilter().withCriteria("username", QCriteriaOperator.EQUALS, username).withLimit(1), tx).isEmpty())
|
||||
{
|
||||
String ending = "-" + suffix++;
|
||||
username = base.substring(0, Math.min(base.length(), 64 - ending.length())) + ending;
|
||||
}
|
||||
return (username);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Standard actions retain customizers, configured audits and the caller's session.
|
||||
*******************************************************************************/
|
||||
private List<QRecord> query(String table, QQueryFilter filter, QBackendTransaction tx) throws QException
|
||||
{
|
||||
return (new QueryAction().execute(new QueryInput(table).withFilter(filter).withTransaction(tx)).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Preserve domain validation messages while withholding internal database errors.
|
||||
*******************************************************************************/
|
||||
private QRecord checked(QRecord record) throws QException
|
||||
{
|
||||
if(record.getErrors() != null && !record.getErrors().isEmpty())
|
||||
{
|
||||
List<String> messages = record.getErrors().stream().map(error -> error instanceof BadInputStatusMessage ? error.getMessage() : "Could not save this setting.").toList();
|
||||
throw (new QBadRequestException(String.join("\n", messages)));
|
||||
}
|
||||
return (record);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert via ordinary table validation, including fingerprint uniqueness.
|
||||
*******************************************************************************/
|
||||
private QRecord insert(String table, QRecord record, QBackendTransaction tx) throws QException
|
||||
{
|
||||
return (checked(new InsertAction().execute(new InsertInput(table).withRecord(record).withTransaction(tx)).getRecords().getFirst()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Persist only supplied fields, with the original user's audit attribution.
|
||||
*******************************************************************************/
|
||||
private void update(String table, QRecord record, QBackendTransaction tx) throws QException
|
||||
{
|
||||
checked(new UpdateAction().execute(new UpdateInput(table).withRecords(List.of(record)).withTransaction(tx)).getRecords().getFirst());
|
||||
}
|
||||
}
|
||||
@@ -24,7 +24,7 @@ package com.kingsrook.qbits.filegateway.sftp;
|
||||
import java.io.Serializable;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer;
|
||||
@@ -244,18 +244,23 @@ public class AuditWriter
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Optimistic status transition, called under withAccountLock because core checks
|
||||
** expected versions before SQL execution rather than in the SQL WHERE clause.
|
||||
** Guarded status transition. Callers hold withAccountLock, so the status re-read in
|
||||
** that transaction is serialized across pods; status is compared here rather than
|
||||
** through a core record-version field, because core advances version fields on
|
||||
** every update (a STRING version becomes a random UUID).
|
||||
*******************************************************************************/
|
||||
public boolean transitionUpload(QRecord changes, String expectedStatus) throws QException
|
||||
{
|
||||
QRecord result = system(() -> new UpdateAction().execute(new UpdateInput(FileGatewayTables.UPLOAD_JOB)
|
||||
.withRecords(List.of(changes)).withTransaction(transaction.get())
|
||||
.withExpectedVersions(Map.of(changes.getValue("id"), expectedStatus))).getRecords().getFirst());
|
||||
if(result.getErrors() != null && !result.getErrors().isEmpty())
|
||||
if(transaction.get() == null)
|
||||
{
|
||||
throw (new QException("Upload transitions require the account lock transaction"));
|
||||
}
|
||||
var rows = query(FileGatewayTables.UPLOAD_JOB, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, changes.getValue("id"))));
|
||||
if(rows.isEmpty() || !Objects.equals(expectedStatus, rows.getFirst().getValueString("status")))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
update(FileGatewayTables.UPLOAD_JOB, changes);
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ public class GatewaySession
|
||||
public final ServerSession ssh;
|
||||
public final String remoteIp;
|
||||
public Integer connectionId;
|
||||
public java.time.Instant connectedAt;
|
||||
public QRecord account;
|
||||
public QRecord key;
|
||||
public long operations;
|
||||
|
||||
@@ -82,16 +82,15 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Audit TCP accept and enforce the pre-authentication connection rate.
|
||||
** Track TCP accept and enforce the pre-authentication connection rate. The
|
||||
** connection-log row is written at the first sign-in attempt (or a refusal),
|
||||
** so TCP health probes and port scans that never authenticate leave no row.
|
||||
*******************************************************************************/
|
||||
public synchronized void accepted(ServerSession ssh) throws QException
|
||||
{
|
||||
GatewaySession state = new GatewaySession(instance, ssh, clock);
|
||||
ssh.setAttribute(GatewaySession.ATTRIBUTE, state);
|
||||
state.connectionId = audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord()
|
||||
.withValue("sessionId", state.id).withValue("remoteIp", state.remoteIp).withValue("podName", podName)
|
||||
.withValue("connectedAt", clock.instant()).withValue("lastSeenAt", clock.instant())
|
||||
.withValue("clientChannel", "sftp").withValue("outcome", "AUTH_FAILED")).getValueInteger("id");
|
||||
state.connectedAt = clock.instant();
|
||||
sessions.put(state.id, state);
|
||||
prune(connections, clock.instant().minusSeconds(60));
|
||||
Deque<Instant> window = connections.computeIfAbsent(state.remoteIp, ignored -> new ArrayDeque<>());
|
||||
@@ -144,6 +143,7 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
|
||||
}
|
||||
QRecord account = first(FileGatewayTables.ACCOUNT, "username", username);
|
||||
String fingerprint = KeyUtils.getFingerPrint(BuiltinDigests.sha256, publicKey);
|
||||
ensureConnectionRow(state);
|
||||
audit.update(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("id", state.connectionId)
|
||||
.withValue("username", username).withValue("keyFingerprint", fingerprint).withValue("clientVersion", ssh.getClientVersion())
|
||||
.withValue("accountId", account == null ? null : account.getValue("id")));
|
||||
@@ -292,11 +292,30 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Write the connection-log row on first need (an auth attempt or a refusal).
|
||||
** Its outcome stays AUTH_FAILED until the sign-in succeeds.
|
||||
*******************************************************************************/
|
||||
private void ensureConnectionRow(GatewaySession state) throws QException
|
||||
{
|
||||
if(state.connectionId == null)
|
||||
{
|
||||
Instant at = state.connectedAt == null ? clock.instant() : state.connectedAt;
|
||||
state.connectionId = audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord()
|
||||
.withValue("sessionId", state.id).withValue("remoteIp", state.remoteIp).withValue("podName", podName)
|
||||
.withValue("connectedAt", at).withValue("lastSeenAt", clock.instant())
|
||||
.withValue("clientChannel", "sftp").withValue("outcome", "AUTH_FAILED")).getValueInteger("id");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record the authentication outcome without credentials.
|
||||
*******************************************************************************/
|
||||
private boolean outcome(GatewaySession state, String outcome, String reason) throws QException
|
||||
{
|
||||
ensureConnectionRow(state);
|
||||
audit.update(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("id", state.connectionId)
|
||||
.withValue("outcome", outcome).withValue("disconnectReason", reason).withValue("lastSeenAt", clock.instant()));
|
||||
if(reason != null)
|
||||
|
||||
@@ -22,6 +22,8 @@ package com.kingsrook.qbits.filegateway;
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -73,6 +75,21 @@ class FileGatewayQBitProducerTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The three gateway permission names reach QQQ's catalog, so identity
|
||||
** policies can grant them.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void permissionsAreInTheAvailableCatalog() throws Exception
|
||||
{
|
||||
var instance = new QInstance();
|
||||
var config = new FileGatewayQBitConfig().withBackendName("host-db");
|
||||
new FileGatewayQBitProducer().withQBitConfig(config).withNamespace("test").produce(instance).addSelfToInstance(instance);
|
||||
assertThat(PermissionsHelper.getAllAvailablePermissionNames(instance)).containsAll(FileGatewayPermissions.ALL);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Package scanning never enables an unconfigured producer.
|
||||
*******************************************************************************/
|
||||
@@ -100,6 +117,8 @@ class FileGatewayQBitProducerTest
|
||||
void contingentPickerMetadata(String source) throws Exception
|
||||
{
|
||||
var instance = new QInstance();
|
||||
instance.addPossibleValueSource(new com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource().withName(source)
|
||||
.withType(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType.TABLE).withTableName(source));
|
||||
new FileGatewayQBitProducer().withQBitConfig(new FileGatewayQBitConfig()).produce(instance);
|
||||
var field = instance.getTable(FileGatewayTables.ALIAS).getField(source + "Id");
|
||||
assertThat(field.getPossibleValueSourceName()).isEqualTo(source);
|
||||
@@ -112,4 +131,27 @@ class FileGatewayQBitProducerTest
|
||||
});
|
||||
assertThat(filter.getSubFilters()).isNullOrEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A host without saved views, reports or bulk-load profiles still validates:
|
||||
** those pickers become plain id fields; present sources keep their pickers.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void dropsPickersForMissingHostSources() throws Exception
|
||||
{
|
||||
QInstance instance = new QInstance();
|
||||
com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData alias = FileGatewayTables.tables(new FileGatewayQBitConfig()).stream()
|
||||
.filter(t -> t.getName().equals(FileGatewayTables.ALIAS)).findFirst().orElseThrow();
|
||||
instance.addPossibleValueSource(new com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource().withName("savedView")
|
||||
.withType(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType.TABLE).withTableName("savedView"));
|
||||
FileGatewayQBitProducer.dropMissingPickers(instance, alias);
|
||||
assertThat(alias.getField("savedViewId").getPossibleValueSourceName()).isEqualTo("savedView");
|
||||
assertThat(alias.getField("savedViewId").getPossibleValueSourceFilter()).isNotNull();
|
||||
assertThat(alias.getField("savedReportId").getPossibleValueSourceName()).isNull();
|
||||
assertThat(alias.getField("savedReportId").getPossibleValueSourceFilter()).isNull();
|
||||
assertThat(alias.getField("savedBulkLoadProfileId").getPossibleValueSourceName()).isNull();
|
||||
assertThat(alias.getField("tableName").getPossibleValueSourceName()).isEqualTo("tables");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.filegateway;
|
||||
|
||||
|
||||
import java.security.KeyPair;
|
||||
import java.time.Clock;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
|
||||
import com.kingsrook.qbits.filegateway.preferences.FileTransferPreferencesProvider;
|
||||
import com.kingsrook.qqq.backend.core.actions.preferences.UserPreferencesAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferenceActionResult;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.preferences.UserPreferencesMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiCredentials;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInProcessSessions;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiInvoker;
|
||||
import com.kingsrook.qqq.middleware.javalin.specs.invoke.UiApiRequest;
|
||||
import com.kingsrook.qqq.openapi.model.HttpMethod;
|
||||
import org.apache.sshd.common.config.keys.KeyUtils;
|
||||
import org.apache.sshd.common.config.keys.PublicKeyEntry;
|
||||
import org.apache.sshd.common.digest.BuiltinDigests;
|
||||
import org.apache.sshd.sftp.client.SftpClientFactory;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Self-service acceptance through core, strict HTTP routes and real SSH clients.
|
||||
*******************************************************************************/
|
||||
@ExtendWith(NoDiskExtension.class)
|
||||
class FileTransferPreferencesTest
|
||||
{
|
||||
private static final String SECTION = FileTransferPreferencesProvider.SECTION_NAME;
|
||||
private final UserPreferencesAction preferences = new UserPreferencesAction();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Connect permission alone reveals the section, independently of the admin app.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void visibilityAndEmptyAccount() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, "A");
|
||||
assertThat(preferences.listSections()).hasSize(1);
|
||||
var section = preferences.getSection(SECTION);
|
||||
assertThat(section.getSection().getPermissionName()).isEqualTo("file_gateway.connect");
|
||||
assertThat(section.getSection().getAppName()).isNull();
|
||||
assertThat(section.getSection().getOrder()).isEqualTo(60);
|
||||
assertThat(section.getValues().getValues().get("username")).isEqualTo("");
|
||||
assertThat(section.getValues().getLists().get("keys")).isEmpty();
|
||||
assertThat(section.getSection().getLists().getFirst().getDescription()).contains("first key creates");
|
||||
assertThat(section.getSection().getFields()).noneMatch(field -> "host".equals(field.getName()));
|
||||
assertThat(section.getValues().getValues().get("port")).isEqualTo(h.runtime.getProtocol().getPort());
|
||||
assertThat(section.getValues().getValues().get("hostKeyFingerprint")).isNotNull();
|
||||
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).isEmpty();
|
||||
user(h, "NONE");
|
||||
assertThat(preferences.listSections()).isEmpty();
|
||||
assertThatThrownBy(() -> preferences.getSection(SECTION)).isInstanceOf(QNotFoundException.class);
|
||||
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of())).isInstanceOf(QNotFoundException.class);
|
||||
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of())).isInstanceOf(QNotFoundException.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Host is optional; heartbeat data comes from the most recent server.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void publicHostAndLatestHeartbeat() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance -> { }, 0, false, false))
|
||||
{
|
||||
assertThat(h.config.getPublicHostName()).isNull();
|
||||
UserPreferencesMetaData.of(h.instance).getSections().clear();
|
||||
FileTransferPreferencesProvider.register(h.instance, h.config.withPublicHostName("sftp.example.test"));
|
||||
user(h, "A");
|
||||
assertThat(preferences.getSection(SECTION).getValues().getValues()).containsEntry("host", "sftp.example.test").containsEntry("port", null);
|
||||
h.audit.insert(FileGatewayTables.SERVER, new QRecord().withValue("podName", "new").withValue("port", 2223)
|
||||
.withValue("lastSeenAt", Instant.now()).withValue("hostKeyFingerprint", "SHA256:new"));
|
||||
h.audit.insert(FileGatewayTables.SERVER, new QRecord().withValue("podName", "old").withValue("port", 2222)
|
||||
.withValue("lastSeenAt", Instant.now().minusSeconds(60)).withValue("hostKeyFingerprint", "SHA256:old"));
|
||||
assertThat(preferences.getSection(SECTION).getValues().getValues()).containsEntry("port", 2223).containsEntry("hostKeyFingerprint", "SHA256:new");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Creation uses schema defaults, reuses the account and preserves audit identity.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void createsAndReusesAccountWithAudits() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer(Clock.systemUTC(), instance ->
|
||||
{
|
||||
for(String table : List.of(FileGatewayTables.ACCOUNT, FileGatewayTables.SSH_KEY))
|
||||
{
|
||||
instance.getTable(table).withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.RECORD));
|
||||
}
|
||||
instance.getTable(FileGatewayTables.ACCOUNT).getField("maxFileSizeMb").setDefaultValue(321);
|
||||
instance.setEnforcePermissionsInCoreActions(true);
|
||||
}))
|
||||
{
|
||||
user(h, "A");
|
||||
var key = pair();
|
||||
var first = add(key, "Laptop");
|
||||
assertThat(first.getMessage()).contains(KeyUtils.getFingerPrint(BuiltinDigests.sha256, key.getPublic()));
|
||||
assertThat(first.getSection().getValues().getValues().get("username")).isEqualTo("user-a");
|
||||
assertThat(first.getSection().getValues().getLists().get("keys").getFirst().getValues()).containsEntry("keyType", "ssh-ed25519").doesNotContainKey("publicKey");
|
||||
add(pair(), "Backup");
|
||||
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).singleElement().satisfies(account ->
|
||||
{
|
||||
assertThat(account.getValueString("userId")).isEqualTo("A");
|
||||
assertThat(account.getValueString("accountType")).isEqualTo("USER");
|
||||
assertThat(account.getValueString("rootMode")).isEqualTo("FULL_TREE");
|
||||
assertThat(account.getValueString("status")).isEqualTo("ACTIVE");
|
||||
assertThat(account.getValueInteger("maxFileSizeMb")).isEqualTo(321);
|
||||
assertThat(account.getValueInteger("maxConcurrentSessions")).isEqualTo(4);
|
||||
});
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).hasSize(2);
|
||||
var audits = h.audit.system(() -> new QueryAction().execute(new QueryInput("audit")).getRecords());
|
||||
var auditTables = h.audit.system(() -> new QueryAction().execute(new QueryInput("auditTable")).getRecords());
|
||||
var auditUsers = h.audit.system(() -> new QueryAction().execute(new QueryInput("auditUser")).getRecords());
|
||||
Object tableId = auditTables.stream().filter(row -> FileGatewayTables.SSH_KEY.equals(row.getValueString("name"))).findFirst().orElseThrow().getValue("id");
|
||||
Object userId = auditUsers.stream().filter(row -> "Test user A".equals(row.getValueString("name"))).findFirst().orElseThrow().getValue("id");
|
||||
assertThat(audits).anySatisfy(audit ->
|
||||
{
|
||||
assertThat(audit.getValue("auditTableId")).isEqualTo(tableId);
|
||||
assertThat(audit.getValue("auditUserId")).isEqualTo(userId);
|
||||
assertThat(audit.getValueString("clientChannel")).isEqualTo("next-ui/preferences-test");
|
||||
});
|
||||
assertThat(QContext.getQSession().getUser().getIdReference()).isEqualTo("A");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalid first keys roll back the account; duplicates retain table messages.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void rejectsWeakDuplicateAndMissingKeys() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, "A");
|
||||
assertThatThrownBy(() -> add(KeyUtils.generateKeyPair("ssh-rsa", 2048), "Weak"))
|
||||
.isInstanceOf(QBadRequestException.class).hasMessageContaining("at least 3072 bits");
|
||||
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).isEmpty();
|
||||
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of()))
|
||||
.isInstanceOf(QBadRequestException.class).hasMessageContaining("Public key: is required").hasMessageContaining("Label: is required");
|
||||
var key = pair();
|
||||
add(key, "First");
|
||||
assertThatThrownBy(() -> add(key, "Duplicate")).isInstanceOf(QBadRequestException.class).hasMessageContaining("Fingerprint");
|
||||
user(h, "B");
|
||||
assertThatThrownBy(() -> add(key, "Other user duplicate")).isInstanceOf(QBadRequestException.class).hasMessageContaining("Fingerprint");
|
||||
assertThat(h.audit.query(FileGatewayTables.ACCOUNT, new QQueryFilter())).hasSize(1);
|
||||
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter())).hasSize(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Normalization, truncation and suffixes obey the account validator.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"Alice@EXAMPLE.COM", "A", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"})
|
||||
void uniquifiesUsername(String id) throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, id);
|
||||
String base = id.toLowerCase(java.util.Locale.ROOT).replaceAll("[^a-z0-9._-]", "-");
|
||||
base = base.length() < 3 ? "user-" + base : base.substring(0, Math.min(64, base.length()));
|
||||
h.account(base, "B");
|
||||
String second = base.substring(0, Math.min(62, base.length())) + "-2";
|
||||
h.account(second, "B");
|
||||
var added = add(pair(), "Key");
|
||||
assertThat(added.getSection().getValues().getValues().get("username")).isEqualTo(base.substring(0, Math.min(62, base.length())) + "-3");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Lists and mutations exclude other users, revoked keys and service accounts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void isolatesKeysAndRejectsIdentityInjection() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
var other = h.key(h.account("bob", "B"), pair());
|
||||
var account = h.account("alice", "A");
|
||||
var revoked = h.key(account, pair());
|
||||
h.audit.update(FileGatewayTables.SSH_KEY, new QRecord().withValue("id", revoked.getValue("id")).withValue("revokedAt", Instant.now()));
|
||||
var service = h.account("service", "A");
|
||||
h.audit.update(FileGatewayTables.ACCOUNT, new QRecord().withValue("id", service.getValue("id")).withValue("accountType", "SERVICE"));
|
||||
var serviceKey = h.key(service, pair());
|
||||
user(h, "A");
|
||||
var own = add(pair(), "Own");
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).extracting(row -> row.getKey()).containsExactly(own.getFocusRowKey());
|
||||
for(QRecord excluded : List.of(other, revoked, serviceKey))
|
||||
{
|
||||
assertThatThrownBy(() -> preferences.runAction(SECTION, "revokeKey", Map.of("rowKey", excluded.getValueString("id"))))
|
||||
.isInstanceOf(QBadRequestException.class);
|
||||
assertThatThrownBy(() -> patch(excluded.getValueString("id"), Map.of("label", "Stolen")))
|
||||
.isInstanceOf(QBadRequestException.class).hasMessageContaining("isn't a setting here");
|
||||
}
|
||||
assertThatThrownBy(() -> preferences.runAction(SECTION, "addKey", Map.of("userId", "B"))).isInstanceOf(QBadRequestException.class);
|
||||
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of("userId", "B"))).isInstanceOf(QBadRequestException.class);
|
||||
assertThatThrownBy(() -> patch(own.getFocusRowKey(), Map.of("accountId", other.getValue("accountId"))))
|
||||
.isInstanceOf(QBadRequestException.class);
|
||||
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter())).noneMatch(row -> "Stolen".equals(row.getValueString("label")));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** PATCH changes only label/expiry, supports clearing expiry and rejects readonly fields.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void editsOnlyMutableFields() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, "A");
|
||||
String key = add(pair(), "Laptop").getFocusRowKey();
|
||||
Instant expires = Instant.parse("2030-01-01T00:00:00Z");
|
||||
patch(key, Map.of("label", "Work laptop", "expiresAt", expires.toString()));
|
||||
var values = preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues();
|
||||
assertThat(values).containsEntry("label", "Work laptop").containsEntry("expiresAt", expires);
|
||||
Map<String, Object> clear = new HashMap<>();
|
||||
clear.put("expiresAt", null);
|
||||
patch(key, clear);
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues()).containsEntry("expiresAt", null);
|
||||
for(String field : List.of("fingerprint", "keyType", "lastUsedAt"))
|
||||
{
|
||||
assertThatThrownBy(() -> patch(key, Map.of(field, "2030-01-01T00:00:00Z"))).isInstanceOf(QBadRequestException.class).hasMessageContaining("can't be changed");
|
||||
}
|
||||
for(String field : List.of("username", "port", "hostKeyFingerprint"))
|
||||
{
|
||||
assertThatThrownBy(() -> preferences.patchSection(SECTION, Map.of("values", Map.of(field, "22"))))
|
||||
.isInstanceOf(QBadRequestException.class).hasMessageContaining("can't be changed");
|
||||
}
|
||||
assertThatThrownBy(() -> patch(key, Map.of("label", " "))).isInstanceOf(QBadRequestException.class).hasMessageContaining("Label: is required");
|
||||
assertThatThrownBy(() -> patch(key, Map.of("publicKey", "replacement"))).isInstanceOf(QBadRequestException.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A self-service key signs in and lists SFTP; revocation closes and refuses it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void addedKeyLogsInAndRevokedKeyCannot() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
user(h, "A");
|
||||
var pair = pair();
|
||||
var added = add(pair, "Laptop");
|
||||
String username = added.getSection().getValues().getValues().get("username").toString();
|
||||
try(var session = h.login(username, pair); var client = SftpClientFactory.instance().createSftpClient(session))
|
||||
{
|
||||
assertThat(client.readDir("/")).anyMatch(entry -> "testapp".equals(entry.getFilename()));
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys").getFirst().getValues().get("lastUsedAt")).isNotNull();
|
||||
assertThat(preferences.runAction(SECTION, "revokeKey", Map.of("rowKey", added.getFocusRowKey())).getMessage()).isEqualTo("Key revoked.");
|
||||
h.runtime.getProtocol().getAuthenticator().poll();
|
||||
session.waitFor(Set.of(org.apache.sshd.client.session.ClientSession.ClientSessionEvent.CLOSED), Duration.ofSeconds(5));
|
||||
assertThat(session.isClosed()).isTrue();
|
||||
}
|
||||
assertThat(h.audit.query(FileGatewayTables.SSH_KEY, new QQueryFilter()).getFirst().getValue("revokedAt")).isNotNull();
|
||||
assertThat(preferences.getSection(SECTION).getValues().getLists().get("keys")).isEmpty();
|
||||
assertThatThrownBy(() -> h.login(username, pair)).isInstanceOf(Exception.class);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The strict UI route exposes actions and translates domain failures to 400.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void preferencesThroughStrictUiApi() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
QSession session = user(h, "A");
|
||||
String token = UiApiInProcessSessions.register(session, Duration.ofMinutes(1));
|
||||
try
|
||||
{
|
||||
var invoker = UiApiInvoker.forServer(h.http, "next-ui");
|
||||
var get = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.GET).withPath("/preferences")
|
||||
.withCredentials(UiApiCredentials.ofInProcessSession(token)));
|
||||
assertThat(get.getStatus()).isEqualTo(200);
|
||||
assertThat(get.getJson().toString()).contains("file-transfer", "addKey", "revokeKey");
|
||||
var added = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/preferences/{sectionName}/actions/{actionName}")
|
||||
.withPathParam("sectionName", SECTION).withPathParam("actionName", "addKey").withCredentials(UiApiCredentials.ofInProcessSession(token))
|
||||
.withJsonBody(Map.of("values", Map.of("publicKey", PublicKeyEntry.toString(pair().getPublic()), "label", "HTTP key"))));
|
||||
assertThat(added.getStatus()).as(added.getBody()).isEqualTo(200);
|
||||
assertThat(added.getJson().toString()).contains("SHA256:", "HTTP key");
|
||||
var invalid = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath("/preferences/{sectionName}/actions/{actionName}")
|
||||
.withPathParam("sectionName", SECTION).withPathParam("actionName", "addKey").withCredentials(UiApiCredentials.ofInProcessSession(token))
|
||||
.withJsonBody(Map.of("values", Map.of("publicKey", "invalid", "label", "Invalid"))));
|
||||
assertThat(invalid.getStatus()).isEqualTo(400);
|
||||
assertThat(invalid.getBody()).contains("at least 3072 bits");
|
||||
}
|
||||
finally
|
||||
{
|
||||
UiApiInProcessSessions.revoke(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Set a real non-system session with only the user's normal permissions.
|
||||
*******************************************************************************/
|
||||
private QSession user(FileGatewayTestServer h, String id) throws Exception
|
||||
{
|
||||
QSession session = h.audit.system(() -> new QAuthenticationModuleDispatcher().getQModule(h.instance.getAuthentication()).createAutomatedSessionForUser(h.instance, id));
|
||||
session.setPermissions(FileGatewayTestServer.GRANTS.getOrDefault(id, Set.of("file_gateway.connect")));
|
||||
session.setClientChannel("next-ui/preferences-test");
|
||||
QContext.init(h.instance, session);
|
||||
return (session);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Generate a supported public key entirely in memory.
|
||||
*******************************************************************************/
|
||||
private KeyPair pair() throws Exception
|
||||
{
|
||||
return (KeyUtils.generateKeyPair("ssh-ed25519", 256));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Exercise registered action input validation rather than calling the provider.
|
||||
*******************************************************************************/
|
||||
private UserPreferenceActionResult add(KeyPair key, String label) throws Exception
|
||||
{
|
||||
return (preferences.runAction(SECTION, "addKey", Map.of("values", Map.of("publicKey", PublicKeyEntry.toString(key.getPublic()), "label", label))));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Send the same partial row body as a preferences client.
|
||||
*******************************************************************************/
|
||||
private void patch(String key, Map<String, ?> values) throws Exception
|
||||
{
|
||||
preferences.patchSection(SECTION, Map.of("lists", Map.of("keys", List.of(Map.of("key", key, "values", values)))));
|
||||
}
|
||||
}
|
||||
@@ -274,6 +274,36 @@ class PhaseEIntegrationTest
|
||||
assertThat(((Number) ((StatisticsData) home.data("out", now)).getCount()).doubleValue()).isEqualTo(1);
|
||||
assertThat(((Number) ((StatisticsData) home.data("failures", now)).getCount()).intValue()).isEqualTo(1);
|
||||
assertThat(((TableData) home.data("recent", now)).getRows()).hasSize(1);
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// direction: failures and locks are bad when they grow; volume is not bad //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
assertThat(((StatisticsData) home.data("failures", now)).getIncreaseIsGood()).as("failures").isFalse();
|
||||
assertThat(((StatisticsData) home.data("locked", now)).getIncreaseIsGood()).as("locked").isFalse();
|
||||
for(String key : List.of("transfers", "in", "out", "sessions"))
|
||||
{
|
||||
assertThat(((StatisticsData) home.data(key, now)).getIncreaseIsGood()).as(key).isTrue();
|
||||
}
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
// every home table says why it is empty //
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
for(String key : List.of("recent", "partners", "active", "servers"))
|
||||
{
|
||||
assertThat(((TableData) home.data(key, now)).getNoRowsFoundHTML()).as(key).isNotBlank();
|
||||
}
|
||||
assertThat(((TableData) home.data("active", now)).getRows()).isEmpty();
|
||||
assertThat(((TableData) home.data("active", now)).getNoRowsFoundHTML()).contains("No one is connected");
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// readable at all widths: home tables declare no column widths //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
assertThat(((TableData) home.data("recent", now)).getColumns()).allSatisfy(column -> assertThat(column.getWidth()).isNull());
|
||||
for(String key : List.of("recent", "partners", "active", "servers"))
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// at most four columns, so tables stay readable at phone widths //
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
assertThat(((TableData) home.data(key, now)).getColumns()).as(key).hasSizeLessThanOrEqualTo(4)
|
||||
.allSatisfy(column -> assertThat(column.getWidth()).isNull());
|
||||
}
|
||||
assertThat(((TableData) home.data("partners", now)).getRows()).hasSize(1);
|
||||
for(String widget : FileGatewayHome.KEYS)
|
||||
{
|
||||
|
||||
@@ -78,6 +78,35 @@ class SftpIntegrationTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A TCP connection that never signs in (a health probe or port scan) leaves
|
||||
** no connection-log row and no failure; a real sign-in attempt does.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void tcpProbeLeavesNoConnectionRow() throws Exception
|
||||
{
|
||||
try(var h = new FileGatewayTestServer())
|
||||
{
|
||||
for(int i = 0; i < 3; i++)
|
||||
{
|
||||
try(java.net.Socket socket = new java.net.Socket("127.0.0.1", h.runtime.getProtocol().getPort()))
|
||||
{
|
||||
socket.setSoTimeout(2000);
|
||||
socket.getInputStream().read();
|
||||
}
|
||||
}
|
||||
Thread.sleep(300);
|
||||
assertThat(h.audit.query("file_gateway_connection_log", new QQueryFilter())).isEmpty();
|
||||
|
||||
KeyPair unknown = pair();
|
||||
org.assertj.core.api.Assertions.assertThatThrownBy(() -> h.login("nobody", unknown)).isInstanceOf(Exception.class);
|
||||
assertThat(h.audit.query("file_gateway_connection_log", new QQueryFilter())).singleElement()
|
||||
.satisfies(row -> assertThat(row.getValueString("outcome")).isEqualTo("AUTH_FAILED"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Permission filtered listings and audit.
|
||||
*******************************************************************************/
|
||||
|
||||
Reference in new issue
Block a user