Compare commits
38
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
46afa2e533
|
||
|
|
0740fab027 | ||
|
|
30820a6d4f | ||
|
|
22bb23613f | ||
|
|
0e80a5e736 | ||
|
|
2b04d936cf | ||
|
|
23b67c51f7
|
||
|
|
a710d56e8c
|
||
|
|
006dbb4600
|
||
|
|
7a39277072
|
||
|
|
778a5d904c | ||
|
|
a2200d736e | ||
|
|
13408c3cfa | ||
|
|
1c4a8cc115 | ||
|
|
39c28a4e00 | ||
|
|
8015b8b090 | ||
|
|
2c8abf6a64 | ||
|
|
fb70b5f7c2 | ||
|
|
d91d996289 | ||
|
|
74837f5db2 | ||
|
|
0252c6811b | ||
|
|
73bdcc1a6f | ||
|
|
d937b19050 | ||
|
|
2b75c6beb9 | ||
|
|
12ae927c33 | ||
|
|
6bcc28e332 | ||
|
|
cc89f73680 | ||
|
|
e35f1f6b99 | ||
|
|
cedc3023c8 | ||
|
|
281d1bdb07 | ||
|
|
9a56ec64e2 | ||
|
|
308830a06a | ||
|
|
1a8d393d03 | ||
|
|
d709359317 | ||
|
|
aafc96a839 | ||
|
|
59e606418c | ||
|
|
6d981933b9 | ||
|
|
de378a70ae |
No files matched your search
@@ -1,28 +0,0 @@
|
||||
---
|
||||
|
||||
version: 2.1
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
# Use Ubuntu's archive mirror, which also carries the signed security suite.
|
||||
sudo sed -i 's|http://security.ubuntu.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
|
||||
# Build and unit tests on every branch push. Dependencies are public
|
||||
# (Maven Central and the Central snapshot repository), so no context is needed.
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
@@ -0,0 +1,33 @@
|
||||
name: build
|
||||
on:
|
||||
push:
|
||||
jobs:
|
||||
container:
|
||||
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/container.yaml@v1
|
||||
secrets: inherit
|
||||
with:
|
||||
# Same gates as the old orb build-and-test + core smoke: script unit tests,
|
||||
# the full reactor (BOM tests, packaged integration tests), then the packaged
|
||||
# core jar smoked over HTTP. The Dockerfile copies that verified jar.
|
||||
prebuild: |
|
||||
set -euo pipefail
|
||||
# The smoke scripts need Python 3 (stdlib only); the job image has none and no root for apt.
|
||||
mkdir -p /tmp/python
|
||||
curl -fsSLo /tmp/python.tgz https://github.com/astral-sh/python-build-standalone/releases/download/20261013/cpython-3.12.15%2B20261013-x86_64-unknown-linux-gnu-install_only.tar.gz
|
||||
echo "5d546fc004fc2435fa142eb8f42ed8de5eb4725963dc22f7e6671014a80eb560 /tmp/python.tgz" | sha256sum -c -
|
||||
tar -xzf /tmp/python.tgz -C /tmp/python --strip-components=1 && rm /tmp/python.tgz
|
||||
# The release-bundle and full-smoke config tests render compose.yaml with `docker compose config`
|
||||
# (no daemon). Provide the Compose binary behind a `docker compose`-only shim.
|
||||
curl -fsSLo /tmp/python/bin/docker-compose https://github.com/docker/compose/releases/download/v5.6.1/docker-compose-linux-x86_64
|
||||
echo "47737f4236e6d3ccea7291995203f8f018058783b4c3a9469119811a0b5db8c0 /tmp/python/bin/docker-compose" | sha256sum -c -
|
||||
printf '%s\n' '#!/bin/sh' '[ "$1" = compose ] || { echo "only docker compose is installed" >&2; exit 1; }' 'shift' 'exec docker-compose "$@"' > /tmp/python/bin/docker
|
||||
chmod +x /tmp/python/bin/docker-compose /tmp/python/bin/docker
|
||||
export PATH=/tmp/python/bin:$PATH
|
||||
python3 -m unittest discover -s scripts -p 'test_*.py' -v
|
||||
mvn verify
|
||||
python3 scripts/run_core_smoke.py qqq-all-app/target/qqq-all-app.jar
|
||||
images: |
|
||||
qqq-all|Dockerfile|.
|
||||
# develop pushes git.kof22.com/qrun-io/qqq-all:ci-<sha>-run-<n> plus the moving develop tag.
|
||||
publish-branch: develop
|
||||
extra-tags: develop
|
||||
@@ -0,0 +1,23 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -1,48 +0,0 @@
|
||||
name: Publish develop image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [develop]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Check public package visibility
|
||||
id: package_visibility
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
if visibility="$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility 2>/dev/null)" && [ "$visibility" = public ]; then
|
||||
echo 'publish=true' >> "$GITHUB_OUTPUT"
|
||||
echo 'The qqq-all container package is public; develop image publication is enabled.' >> "$GITHUB_STEP_SUMMARY"
|
||||
else
|
||||
echo 'publish=false' >> "$GITHUB_OUTPUT"
|
||||
echo 'Skipped develop image publication: the qqq-all container package is not yet public or cannot be read. An owner must bootstrap the package and set it to Public before C9 enables publication.' >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: '21'
|
||||
cache: maven
|
||||
- name: Verify application
|
||||
run: mvn -B -ntp verify
|
||||
- uses: docker/login-action@v3
|
||||
if: steps.package_visibility.outputs.publish == 'true'
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Build and publish develop image
|
||||
if: steps.package_visibility.outputs.publish == 'true'
|
||||
run: |
|
||||
docker build \
|
||||
--label org.opencontainers.image.source=https://github.com/QRun-IO/qqq-all \
|
||||
--tag ghcr.io/qrun-io/qqq-all:develop .
|
||||
docker push ghcr.io/qrun-io/qqq-all:develop
|
||||
@@ -1,124 +0,0 @@
|
||||
name: Release 4.1.0
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_sha:
|
||||
description: Full develop commit SHA already tagged 4.1.0
|
||||
required: true
|
||||
type: string
|
||||
smoke_run_id:
|
||||
description: Successful Smoke workflow run ID for that exact develop SHA
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
actions: read
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
concurrency:
|
||||
group: qqq-all-release-4.1.0
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
environment: qqq-all-release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
RELEASE_SHA: ${{ inputs.release_sha }}
|
||||
SMOKE_RUN_ID: ${{ inputs.smoke_run_id }}
|
||||
RELEASE_ENABLED: ${{ vars.QQQ_ALL_RELEASE_ENABLED }}
|
||||
LICENSE_CLEARED: ${{ vars.QQQ_ALL_RELEASE_LICENSE_APPROVED }}
|
||||
VERSION: 4.1.0
|
||||
TAG: 4.1.0
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Check manual release controls and exact tag
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$GITHUB_REF" = refs/heads/develop
|
||||
test "$RELEASE_ENABLED" = true
|
||||
test "$LICENSE_CLEARED" = true
|
||||
[[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$SMOKE_RUN_ID" =~ ^[0-9]+$ ]]
|
||||
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
|
||||
test "$RELEASE_SHA" = "$GITHUB_SHA"
|
||||
test "$(git cat-file -t "refs/tags/$TAG")" = tag
|
||||
test "$(git rev-parse "refs/tags/$TAG^{commit}")" = "$RELEASE_SHA"
|
||||
- name: Verify upstream GA, exact-commit smoke, Central POMs, and public GHCR
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ga_state="$(gh api repos/QRun-IO/qqq/releases/tags/v4.1.0 \
|
||||
--jq 'if (.tag_name == "v4.1.0" and .draft == false and .prerelease == false) then "ready" else "blocked" end')"
|
||||
test "$ga_state" = ready
|
||||
gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SMOKE_RUN_ID" > "$RUNNER_TEMP/smoke-run.json"
|
||||
gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SMOKE_RUN_ID/jobs?per_page=100" > "$RUNNER_TEMP/smoke-jobs.json"
|
||||
central="https://repo.maven.apache.org/maven2/com/kingsrook/qqq"
|
||||
for artifact in qqq-all-parent qqq-all-bom; do
|
||||
curl --fail --silent --show-error \
|
||||
"$central/$artifact/$VERSION/$artifact-$VERSION.pom" \
|
||||
--output "$RUNNER_TEMP/$artifact.pom"
|
||||
done
|
||||
python3 scripts/release_gate.py --root . --version "$VERSION" --sha "$RELEASE_SHA" \
|
||||
--smoke-run "$RUNNER_TEMP/smoke-run.json" \
|
||||
--smoke-jobs "$RUNNER_TEMP/smoke-jobs.json" \
|
||||
--published-parent "$RUNNER_TEMP/qqq-all-parent.pom" \
|
||||
--published-bom "$RUNNER_TEMP/qqq-all-bom.pom"
|
||||
test "$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility)" = public
|
||||
gh api --paginate 'orgs/QRun-IO/packages/container/qqq-all/versions?per_page=100' \
|
||||
--jq '.[] | .metadata.container.tags[]?' > "$RUNNER_TEMP/ghcr-tags.txt"
|
||||
if grep -Fxq "$VERSION" "$RUNNER_TEMP/ghcr-tags.txt"; then
|
||||
echo "GHCR tag $VERSION already exists" >&2
|
||||
exit 1
|
||||
fi
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: '21'
|
||||
cache: maven
|
||||
- name: Verify release reactor
|
||||
run: mvn -B -ntp verify
|
||||
- name: Recheck develop, annotated tag, and release absence before publication
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(gh api "repos/$GITHUB_REPOSITORY/branches/develop" --jq .commit.sha)" = "$RELEASE_SHA"
|
||||
test "$(git ls-remote origin "refs/tags/$TAG^{}" | cut -f1)" = "$RELEASE_SHA"
|
||||
if release_check="$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$TAG" 2>&1)"; then
|
||||
echo "GitHub release $TAG already exists" >&2
|
||||
exit 1
|
||||
elif [[ "$release_check" != *"HTTP 404"* ]]; then
|
||||
echo "$release_check" >&2
|
||||
exit 1
|
||||
fi
|
||||
- uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Publish versioned image
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
image="ghcr.io/qrun-io/qqq-all:$VERSION"
|
||||
docker build \
|
||||
--label org.opencontainers.image.source=https://github.com/QRun-IO/qqq-all \
|
||||
--label "org.opencontainers.image.version=$VERSION" \
|
||||
--label "org.opencontainers.image.revision=$RELEASE_SHA" \
|
||||
--tag "$image" .
|
||||
docker push "$image"
|
||||
printf 'Published image: `%s`\n' "$image" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Publish GitHub release with Compose asset
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh release create "$TAG" compose.yaml --repo "$GITHUB_REPOSITORY" \
|
||||
--verify-tag --title "qqq-all $VERSION" --notes-from-tag
|
||||
@@ -1,47 +0,0 @@
|
||||
name: Smoke
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [develop]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
core:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: "21"
|
||||
cache: maven
|
||||
- name: Check smoke assertions
|
||||
run: python3 -m unittest discover -s scripts -p 'test_*.py' -v
|
||||
- name: Verify and package core app
|
||||
run: mvn -B -ntp -pl qqq-all-app -am verify
|
||||
- name: Smoke packaged core over HTTP
|
||||
run: python3 scripts/run_core_smoke.py qqq-all-app/target/qqq-all-app.jar
|
||||
|
||||
full:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: "21"
|
||||
cache: maven
|
||||
- name: Package full app for Docker image
|
||||
run: mvn -B -ntp -pl qqq-all-app -am -DskipTests package
|
||||
- name: Smoke full Compose over HTTP
|
||||
run: python3 scripts/run_full_smoke.py
|
||||
@@ -1,6 +1,7 @@
|
||||
FROM eclipse-temurin:21-jre-jammy
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get upgrade -y --no-install-recommends \
|
||||
&& apt-get install -y --no-install-recommends curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --system --gid 10001 qqq \
|
||||
@@ -10,6 +11,7 @@ RUN apt-get update \
|
||||
|
||||
WORKDIR /app
|
||||
COPY --chown=qqq:qqq qqq-all-app/target/qqq-all-app.jar /app/qqq-all.jar
|
||||
COPY --chown=qqq:qqq LICENSE NOTICE /app/
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
ENV QQQ_ALL_DATA_DIR=/app/data
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
A runnable reference app for QQQ 4.1 with two profiles: `core` uses local storage and embedded Artemis; `full` adds external storage, RabbitMQ, OpenSearch quick search, Keycloak, Mailpit, and the included qbits. It is a local demo under construction while QQQ 4.1 finishes. See the [design](docs/design.md), [plan](docs/plan.md), and [epic #1](https://github.com/QRun-IO/qqq-all/issues/1).
|
||||
|
||||
The stack pins public Next **1.0.0-RC.8** for application testing. The owner approved an RC carrying its documented Firefox/WebKit/tablet failures, report/query and input parity gaps, sub-path hosting, real Google Drive verification, and visual-approval limitations. These remain open; passing qqq-all HTTP smoke checks do not establish browser parity or GA acceptance. See the [RC8 scope and approval](docs/release.md#next-rc8-integration).
|
||||
|
||||
## Quick start
|
||||
|
||||
Install Java 21 and Maven, then build the app and its BOM from the repository root:
|
||||
@@ -39,10 +41,15 @@ Full adds sample tables on PostgreSQL, MySQL, MongoDB, MinIO S3, and SFTP. It in
|
||||
|
||||
To stop the full stack, run `docker compose --profile full down`. Add `--volumes` only when you intend to erase demo data. For a fresh Artemis volume, the configured login is required; old volumes created for anonymous access should be recreated. This reference stack uses demo credentials and is not a production deployment.
|
||||
|
||||
The [develop image workflow](.github/workflows/publish-image.yml) verifies the build and publishes to GHCR only after an organization owner bootstraps the container package and sets its visibility to public. Until then, image publication is skipped and local Compose remains available.
|
||||
CI runs [.gitea/workflows/build.yaml](.gitea/workflows/build.yaml) through the shared `KofTwentyTwo/ci-workflows` container workflow: script unit tests, `mvn verify`, and the packaged core HTTP smoke check on every push, then the image build and Trivy scan. Pushes to `develop` publish `git.kof22.com/qrun-io/qqq-all` (`ci-<sha>-run-<n>` and `develop`). The full Compose smoke check runs locally with `python3 scripts/run_full_smoke.py`.
|
||||
|
||||
The [4.1.0 release procedure](docs/release.md) describes the separate Maven parent/BOM publication path and the manually gated, exact-commit [release workflow](.github/workflows/release.yml). The workflow will not release the current snapshot build.
|
||||
The [4.1.0 release procedure](docs/release.md) describes the separate Maven parent/BOM publication path. The GitHub release workflow that called the retired qqq-orb was removed; a versioned release path on Gitea is not defined yet.
|
||||
|
||||
## Contributing and license
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md), the [Code of Conduct](CODE_OF_CONDUCT.md), and the [security policy](SECURITY.md). Licensed under Apache-2.0; see [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
||||
|
||||
The application JAR includes exact copies of these project documents under
|
||||
`META-INF/qqq-all/`, separate from dependency notices. The container also includes
|
||||
them at `/app/LICENSE` and `/app/NOTICE`. `PackagedNoticesIT` checks the shaded JAR's
|
||||
project documents against the root files during the existing `mvn verify` lifecycle.
|
||||
+4
-2
@@ -124,7 +124,7 @@ services:
|
||||
- mongo-config:/data/configdb
|
||||
- ./infra/mongo/start.sh:/start.sh:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "mongosh --quiet -u \"$$MONGO_INITDB_ROOT_USERNAME\" -p \"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --eval 'db.adminCommand({ping:1}).ok' | grep -q 1"]
|
||||
test: ["CMD-SHELL", "mongosh --quiet --username=\"$$MONGO_INITDB_ROOT_USERNAME\" --password=\"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --eval 'db.adminCommand({ping:1}).ok' | grep -q 1"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
@@ -181,7 +181,9 @@ services:
|
||||
profiles: [full]
|
||||
command: "${SFTP_USER:-qqq}:${SFTP_PASSWORD:?set SFTP_PASSWORD}:1001:1001:upload"
|
||||
volumes:
|
||||
- sftp-data:/home/${SFTP_USER:-qqq}/upload
|
||||
- type: volume
|
||||
source: sftp-data
|
||||
target: /home/${SFTP_USER:-qqq}/upload
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pgrep sshd >/dev/null"]
|
||||
interval: 5s
|
||||
|
||||
+23
-5
@@ -1,11 +1,29 @@
|
||||
# 4.1.0 release preparation
|
||||
|
||||
This procedure prepares a release; this repository currently remains on `4.1.0-SNAPSHOT`. The [manual release workflow](../.github/workflows/release.yml) only publishes the versioned GHCR image and a GitHub release containing `compose.yaml`. It does **not** deploy Maven artifacts or create a tag. Publication is authorized once the release gates pass. Keep the workflow disabled until Apache-2.0 alignment, public artifact verification and GHCR visibility are complete.
|
||||
This procedure prepares a release; this repository currently remains on `4.1.0-SNAPSHOT`. The GitHub manual release workflow (GHCR image plus Compose bundle, via the retired qqq-orb) was removed with the move to Gitea CI; `develop` images now publish to `git.kof22.com/qrun-io/qqq-all`. A versioned release path on Gitea is not defined yet.
|
||||
|
||||
## Exact coordinated RC under QQQ #921
|
||||
|
||||
The same workflow accepts only `4.1.0` (default GA path) or exact `4.1.0-RC.1`. For the [coordinated RC chain](https://github.com/QRun-IO/qqq/issues/921), first publish and verify the accepted Next candidate, QQQ `v4.1.0-RC.1`, and the corresponding build parent/eight QBits. Then review actual source pins: root `revision` and BOM `qqq.version` both `4.1.0-RC.1`, explicit accepted Next version, and immutable published QBit versions. No command-line version overrides substitute for source edits. The current RC8/snapshot pins are intentionally unchanged and cannot pass release validation. Acceptance limitations belong to the exact accepted pair; the RC8 notes below are historical evidence, not automatic approval for a newer pair.
|
||||
|
||||
Use the existing `release-central` profile and the parent/BOM-only command below to stage the exact RC POMs for **manual** Central review (`autoPublish=false`). Its enforcers accept only GA or exact RC1, require the QQQ pin to equal the source revision, and reject snapshot/range/LATEST dependency pins. After both POMs are public and the reviewed source is merged, require a successful CircleCI `verify-and-smoke` workflow with `build-and-test`, `core` and `full` on that exact develop SHA and an annotated signed tag named `4.1.0-RC.1` at it. The workflow requires GitHub's tag verification to report `verified=true`, `reason=valid`, and checks the exact local/remote tag object and commit. Public QQQ/Next release tags must match their pins and prerelease state; first-party BOM inputs (including recursive public parents) must resolve anonymously, with qbit-build-parent importing this QQQ version. The full reactor also rejects transitive snapshot dependencies. Public POM comparison includes the explicit Next pin. Existing GHCR visibility, protected environment, license/enable controls, destination-absence checks and live ref rechecks remain required.
|
||||
|
||||
When those gates and owner authorization are satisfied, dispatch **Release 4.1.0** on develop with `version=4.1.0-RC.1`, the full `release_sha`, `smoke_evidence=circle`, and its successful `circle_workflow_id`. RC publication uses `--prerelease --latest=false`; it never becomes the latest GA release. The default `version=4.1.0` still requires upstream non-prerelease GA and follows the separate GA requirements below. Do not dispatch while any pin or external prerequisite is pending.
|
||||
|
||||
## Image Compose bundle
|
||||
|
||||
The workflow attaches `qqq-all-<version>.tar.gz` and its rendered `compose.yaml`. Download/extract the **whole bundle**, retain the `infra` directory beside Compose, and follow its README. Both app profiles use the exact pushed GHCR digest and have no app build context. The bundle includes the tracked PostgreSQL/MySQL/Mongo initialization, Keycloak realm, MinIO Dockerfile, `.env.example`, LICENSE/NOTICE and `release.json` source/image identity. It excludes local `.env`, application source and Maven outputs. Full-profile MinIO still builds locally and needs network access; other service images are unchanged. The standalone Compose asset alone is insufficient for full-profile operation. Configure a unique Compose project and demo credentials, build only MinIO for full, then start with `--no-build`. Verify anonymous image pull and real core/full startup from the extracted bundle after publication; config-rendering tests are not runtime acceptance.
|
||||
|
||||
## Next RC8 integration
|
||||
|
||||
The stack BOM explicitly pins public `com.kingsrook.qqq:qqq-frontend-next:1.0.0-RC.8`, independently of the framework snapshot's dashboard pin. Its [release source](https://github.com/QRun-IO/qqq-frontend-next/commit/cd3d4731d1910e68e994da9c74a8c28a2f1b7c1c) is `cd3d4731d1910e68e994da9c74a8c28a2f1b7c1c`; the public JAR SHA-256 is `238c9ae80beea336717be759219f0d65a1d614e8e7d6a145a16374cc392a73a1`. The owner [approved a QQQ RC with these documented limitations](https://github.com/QRun-IO/qqq/issues/798#issuecomment-5930629521): Firefox/WebKit/tablet failures, incomplete report/query and input parity, sub-path hosting, real Google Drive verification, and visual approval. See the [RC8 release notes](https://github.com/QRun-IO/qqq-frontend-next/releases/tag/v1.0.0-RC.8) for the retained failures and incomplete runs.
|
||||
|
||||
This approval applies to exact RC8 and does not turn those limitations into passing acceptance. The successor contracts `core.widget.data_bag_viewer`, `core.widget.pivot_table_setup`, `core.widget.filter_and_columns_setup`, `core.widget.row_builder`, `core.widget.script_viewer`, `core.widget.blocks`, and `dashboard.acceptance` remain pending and excluded from verified counts, with Next 1.0 final / QQQ 4.1 GA as the follow-up target. qqq-all's package and HTTP smoke checks verify integration and asset delivery, not browser parity. This repository remains a snapshot integration build; the separate GA procedure below and other release/security/public-consumer gates remain in force.
|
||||
|
||||
## Prerequisites and Maven artifacts
|
||||
|
||||
1. QQQ must publish a non-draft, non-prerelease `v4.1.0` GitHub release with ESB and the Next dashboard included. Release `qbit-build-parent` against the 4.1 BOM, then publish all eight qbits against that parent. This work occurs in their own repositories.
|
||||
2. In a separate reviewed change to this repository, set root `revision` and the BOM's `qqq.version` to `4.1.0`; pin each qbit to a published non-snapshot version. Run `mvn -B -ntp verify`, the core and full smoke checks, and review the resolved dependencies and effective POMs. Do not use `-Drevision=4.1.0` against a snapshot source tree as a substitute for reviewed pins.
|
||||
2. In a separate reviewed change to this repository, set root `revision` and the BOM's `qqq.version` to `4.1.0`; replace the explicit Next RC8 pin with the accepted Next 1.0 final version and pin each qbit to a published non-snapshot version. Run `mvn -B -ntp verify`, the core and full smoke checks, and review the resolved dependencies and effective POMs. Do not use `-Drevision=4.1.0` against a snapshot source tree as a substitute for reviewed pins.
|
||||
3. Stage **both** `com.kingsrook.qqq:qqq-all-parent:4.1.0` and `com.kingsrook.qqq:qqq-all-bom:4.1.0` using the opt-in `release-central` Maven profile. The profile runs the release-version/dependency checks, signs the POMs with GPG, and uses the Central Publishing Maven plugin with `autoPublish=false`. It is only for the parent and BOM reactor; the app and BOM test JAR are excluded. POM packaging needs no source or Javadoc JARs. Configure the owner's `central` Portal user token in a private Maven `settings.xml` and a trusted GPG key outside the repository. Do not place credentials in this project or CI.
|
||||
|
||||
```bash
|
||||
@@ -19,10 +37,10 @@ The owner selected Apache-2.0 for first-party code in [QQQ #923](https://github.
|
||||
|
||||
## Exact commit and manual dispatch
|
||||
|
||||
After the release POMs are public, merge the reviewed version-pin change and the C7 smoke workflow to `develop`. Require a successful **push** run of `Smoke` on the exact intended commit; both `core` and `full` jobs must pass. Review the run's full SHA and run ID in Actions. Create an **annotated** `4.1.0` tag at that exact `develop` commit through the normal signed-tag process. The workflow checks that the local tag is annotated and that the remote tag resolves to the same SHA; it does not verify a tag signature, so the operator must do that separately. Do not move the tag or advance `develop` before dispatch.
|
||||
After the release POMs are public, merge the version-pin change to `develop`. Require a successful webhook-triggered CircleCI `verify-and-smoke` workflow on the exact intended commit; `build-and-test`, `core` and `full` must all pass. Record its full SHA and workflow UUID. The shared verifier checks public Circle records against the latest trusted GitHub CircleCI check; it needs no exported Circle token for this public repository. Create an **annotated**, signed `4.1.0` tag at that exact `develop` commit. The workflow checks GitHub's valid signature verification, the tag object and its exact commit, including live remote refs. Do not move the tag or advance `develop` before dispatch.
|
||||
|
||||
Configure a protected `qqq-all-release` GitHub environment with required reviewers and branch restriction to `develop`. After Apache alignment and the publication preconditions pass, set environment variables `QQQ_ALL_RELEASE_LICENSE_APPROVED=true` and `QQQ_ALL_RELEASE_ENABLED=true`. Unset them to close the gate. These variables and the environment approval are operational controls, not substitutes for the release review.
|
||||
Configure a protected `qqq-all-release` GitHub environment with no required human reviewers and branch restriction to `develop`. After Apache alignment and the publication preconditions pass, set environment variables `QQQ_ALL_RELEASE_LICENSE_APPROVED=true` and `QQQ_ALL_RELEASE_ENABLED=true`. Unset them to close the gate. These variables are automated release controls; enable them only after the documented artifact and license prerequisites are established.
|
||||
|
||||
Dispatch **Release 4.1.0** on `develop` with `release_sha` (the full 40-character commit SHA) and `smoke_run_id` (the successful push run ID). The job requires that checkout, `develop`, the annotated tag, and the smoke run all refer to that SHA. It checks upstream GA, both public Central POMs, public GHCR visibility, and the absence of a `4.1.0` GHCR tag; runs `mvn -B -ntp verify`; then rechecks the live `develop` and remote tag refs before publication. Only then does it build and push `ghcr.io/qrun-io/qqq-all:4.1.0` and create the GitHub `4.1.0` release with `compose.yaml` attached.
|
||||
Dispatch **Release 4.1.0** on `develop` with `version=4.1.0` (default), `release_sha` (the full 40-character commit SHA) `smoke_evidence=circle`, and `circle_workflow_id` (the successful Circle workflow UUID). The job requires that checkout, `develop`, the annotated signed tag, and the smoke run all refer to that SHA. It checks upstream GA, both public Central POMs and immutable upstream inputs, public GHCR visibility, and destination absence; verifies the reactor with a fresh public-only cache and rejects transitive snapshots; then rechecks the live `develop` and remote tag refs before publication. Only then does it build and push `ghcr.io/qrun-io/qqq-all:4.1.0` and create the GitHub `4.1.0` release with the Compose bundle and rendered `compose.yaml` attached.
|
||||
|
||||
The image push and GitHub release creation are separate API operations, not an atomic transaction. If the release step fails after the image push, inspect the image digest and workflow logs before recovery; never reuse or overwrite the version tag blindly. Verify the public image pull, release asset, and both Maven POMs from an unauthenticated environment after publication. No workflow in this preparation branch publishes or tags anything by itself.
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
set -euo pipefail
|
||||
|
||||
for attempt in {1..30}; do
|
||||
if mongosh --host mongo --quiet -u "${MONGO_USER}" -p "${MONGO_PASSWORD}" --authenticationDatabase admin \
|
||||
if mongosh --host mongo --quiet --username="${MONGO_USER}" --password="${MONGO_PASSWORD}" --authenticationDatabase admin \
|
||||
--eval 'try { rs.status().ok } catch (e) { if (e.codeName === "NotYetInitialized") { rs.initiate({_id: process.env.MONGO_REPLICA_SET, members: [{_id: 0, host: "mongo:27017"}]}).ok } else { throw e } }' | grep -q 1; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
-- Quick Search 1.x protects its admin processes (quickSearchFullReindex and friends) and its
|
||||
-- admin app with the quick_search_admin permission; the full-profile admin runs the reindex.
|
||||
INSERT INTO permission (name, object_type, object_label)
|
||||
VALUES
|
||||
('quick_search_admin.hasAccess', 'process', 'Quick Search Admin')
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
INSERT INTO role_permission_int (role_id, permission_id)
|
||||
SELECT role.id, permission.id
|
||||
FROM role CROSS JOIN permission
|
||||
WHERE permission.name = 'quick_search_admin.hasAccess'
|
||||
AND role.name = 'admin'
|
||||
ON CONFLICT (role_id, permission_id) DO NOTHING;
|
||||
@@ -62,15 +62,15 @@
|
||||
<version>3.5.0</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>require-ga-revision</id>
|
||||
<id>require-release-revision</id>
|
||||
<phase>validate</phase>
|
||||
<goals><goal>enforce</goal></goals>
|
||||
<configuration>
|
||||
<rules>
|
||||
<requireProperty>
|
||||
<property>revision</property>
|
||||
<regex>^4[.]1[.]0$</regex>
|
||||
<message>Central release requires a reviewed 4.1.0 source revision.</message>
|
||||
<regex>^4[.]1[.]0(-RC[.]1)?$</regex>
|
||||
<message>Central release requires reviewed 4.1.0 or exact 4.1.0-RC.1 source pins.</message>
|
||||
</requireProperty>
|
||||
<requireReleaseDeps/>
|
||||
</rules>
|
||||
|
||||
@@ -15,6 +15,18 @@ Core uses mock authentication. The `sessionId` cookie selects the two fixed demo
|
||||
|
||||
The core demo seeds a customer table view, a shipping address linked to the geo country/state/city tables, and an order review workflow. The `RunRecordWorkflow` process can run that workflow against an order; `syncOrder` writes a process trace after an order event. It also seeds an active `orderStored` webhook subscription to its receiver at `/demo/order-webhook`. Insert an order as Admin, run `SendWebhookEvent` as Admin, and inspect `/demo/order-webhook-receipts` to see the delivered payload. Demo User can read webhook records but cannot edit their configuration or invoke the delivery process. The receiver is available only in the core profile, which Compose publishes on host loopback.
|
||||
|
||||
## Enterprise profile: the assistant
|
||||
|
||||
`QQQ_ALL_PROFILE=enterprise` (the terminal-client test server, started by qqq-cli's
|
||||
`scripts/enterprise-server.sh`) installs qbit-assistant with two model provider rows, reconciled at every
|
||||
start: **Anthropic (Claude)** (key variable `ANTHROPIC_API_KEY`; only the name is stored) and **Local
|
||||
Ollama** (OpenAI-compatible, kind OLLAMA, `QQQ_ASSISTANT_OLLAMA_URL` default `http://127.0.0.1:11434/v1`,
|
||||
model `QQQ_ASSISTANT_OLLAMA_MODEL` default `llama3.3:70b-instruct-q4_K_M`, no key). The default is
|
||||
Anthropic when `ANTHROPIC_API_KEY` is set, else Ollama when it answers and lists the model, else neither
|
||||
(the assistant reports `NOT_CONFIGURED` with what to configure). Every enterprise session holds
|
||||
`assistant.hasAccess` and `assistant.admin`. A turn may take 900 s and one Ollama call 600 s, for a local
|
||||
70B model.
|
||||
|
||||
## Full profile
|
||||
|
||||
Set `QQQ_ALL_PROFILE=full` to use external services. The full profile requires an explicit `QQQ_ALL_BIND_HOST`; use `127.0.0.1` for local-only access or deliberately set another interface when a reverse proxy needs it. `QQQ_ALL_PORT` defaults to `8080`. The full profile replaces core's mock authentication with Keycloak OIDC. It does not start an embedded broker.
|
||||
@@ -108,6 +120,10 @@ For an existing volume, apply the idempotent grants and then sign in again:
|
||||
COMPOSE_PROFILES=full docker compose exec -T postgres sh -c 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < infra/postgres/05-runtime-demo-permissions.sql
|
||||
```
|
||||
|
||||
Quick Search 1.x protects `quickSearchFullReindex` and its other admin processes with
|
||||
`quick_search_admin.hasAccess`; fresh volumes grant it to the admin role through
|
||||
`infra/postgres/06-quick-search-permissions.sql`. Apply that file the same way on an existing volume.
|
||||
|
||||
`mvn -pl qqq-all-app -am verify` includes native local API, calculation/permission,
|
||||
memory restart, and packaged-JAR checks. `scripts/run_core_smoke.py` and
|
||||
`scripts/run_full_smoke.py` verify these routes with actual Admin and restricted
|
||||
|
||||
+40
-1
@@ -42,13 +42,37 @@
|
||||
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-esb</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-frontend-next</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-user-role-permissions</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-quick-search</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-quick-search</artifactId><version>1.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-customizable-table-views</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-standard-process-trace</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-webhooks</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-workflows</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-geo-data</artifactId></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-sftp-data-integration</artifactId></dependency>
|
||||
<!-- Enterprise profile (QQQ_ALL_PROFILE=enterprise): every locally runnable qbit, for terminal-client testing. -->
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-alerts</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-assistant</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-assistant-provider-anthropic</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-assistant-provider-openai</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-backups</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-cdn</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-cluster</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-collaboration</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-data-analysis</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-db-performance</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-email</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-file-gateway</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-help</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-jobs</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-object-storage</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-qa-capture</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-redis</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-release-notes</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-secured-data</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-session-store</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-slack</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-sql-workbench</artifactId><version>1.0.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>com.kingsrook.qbits</groupId><artifactId>qbit-worm-audit</artifactId><version>0.1.0-SNAPSHOT</version></dependency>
|
||||
<dependency><groupId>org.postgresql</groupId><artifactId>postgresql</artifactId></dependency>
|
||||
<dependency><groupId>com.mysql</groupId><artifactId>mysql-connector-j</artifactId></dependency>
|
||||
<dependency><groupId>com.h2database</groupId><artifactId>h2</artifactId></dependency>
|
||||
@@ -60,6 +84,20 @@
|
||||
|
||||
<build>
|
||||
<finalName>qqq-all-app</finalName>
|
||||
<resources>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
</resource>
|
||||
<resource>
|
||||
<directory>${project.basedir}/..</directory>
|
||||
<targetPath>META-INF/qqq-all</targetPath>
|
||||
<filtering>false</filtering>
|
||||
<includes>
|
||||
<include>LICENSE</include>
|
||||
<include>NOTICE</include>
|
||||
</includes>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
@@ -93,6 +131,7 @@
|
||||
<configuration>
|
||||
<systemPropertyVariables>
|
||||
<qqqAllJar>${project.build.directory}/${project.build.finalName}.jar</qqqAllJar>
|
||||
<qqqAllRoot>${project.basedir}/..</qqqAllRoot>
|
||||
</systemPropertyVariables>
|
||||
</configuration>
|
||||
<executions>
|
||||
|
||||
@@ -66,6 +66,10 @@ final class CoreRuntime implements AutoCloseable
|
||||
try
|
||||
{
|
||||
WebhookSubscriptionsHelper.clearMemoizations();
|
||||
if(EnterpriseFeatures.isEnabled())
|
||||
{
|
||||
EnterpriseAssistant.reconcileAtStartup(launcher.getQInstance());
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -34,6 +34,18 @@ final class DemoQBitData
|
||||
"redirect_state", "user", "role", "permission", "role_permission_int", "user_role_int",
|
||||
"user_permission_int");
|
||||
|
||||
/** H2 (MySQL mode) keywords that qbit tables use as column or table names. */
|
||||
private static final Set<String> H2_RESERVED_WORDS = Set.of("VALUE", "KEY", "VALUES", "USER", "ORDER", "GROUP",
|
||||
"LIMIT", "OFFSET", "YEAR", "MONTH", "DAY", "HOUR", "MINUTE", "SECOND", "ROW", "ROWS", "END", "FROM", "TO",
|
||||
"ALL", "AND", "OR", "NOT", "NULL", "TRUE", "FALSE", "IF", "IN", "IS", "AS", "ON", "BY", "SELECT", "WHERE",
|
||||
"TABLE", "INTERVAL", "DEFAULT", "CHECK", "CONSTRAINT", "PRIMARY", "UNIQUE", "FOREIGN", "INDEX", "JOIN",
|
||||
"LEFT", "RIGHT", "INNER", "OUTER", "CROSS", "NATURAL", "UNION", "EXCEPT", "INTERSECT", "MINUS", "DISTINCT",
|
||||
"HAVING", "QUALIFY", "WINDOW", "CASE", "WHEN", "THEN", "ELSE", "FETCH", "FOR", "WITH", "CURRENT_DATE",
|
||||
"CURRENT_TIME", "CURRENT_TIMESTAMP", "CURRENT_USER", "LOCALTIME", "LOCALTIMESTAMP", "ARRAY", "BETWEEN",
|
||||
"EXISTS", "LIKE", "ILIKE", "REGEXP", "SET", "UNKNOWN", "USING", "TOP", "SYSDATE", "SYSTIME", "TODAY",
|
||||
"_ROWID_", "ROWNUM", "LISTAGG", "INTERSECTS", "ANY", "SOME", "ASYMMETRIC", "SYMMETRIC", "AUTHORIZATION",
|
||||
"CURRENT_CATALOG", "CURRENT_SCHEMA", "CURRENT_PATH", "CURRENT_ROLE", "LATERAL", "UESCAPE");
|
||||
|
||||
private DemoQBitData()
|
||||
{
|
||||
}
|
||||
@@ -231,7 +243,7 @@ final class DemoQBitData
|
||||
{
|
||||
throw new IllegalArgumentException("Unsupported demo schema identifier");
|
||||
}
|
||||
if("value".equalsIgnoreCase(value) || "key".equalsIgnoreCase(value))
|
||||
if(H2_RESERVED_WORDS.contains(value.toUpperCase()))
|
||||
{
|
||||
return "`" + value + "`";
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@ import com.kingsrook.qbits.customizabletableviews.CustomizableTableViewsQBitConf
|
||||
import com.kingsrook.qbits.customizabletableviews.CustomizableTableViewsQBitProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchStartupMode;
|
||||
import com.kingsrook.qbits.quicksearch.SearchableFieldConfig;
|
||||
import com.kingsrook.qbits.sftpdataintegration.SFTPDataIntegrationQBitConfig;
|
||||
import com.kingsrook.qbits.sftpdataintegration.SFTPDataIntegrationQBitProducer;
|
||||
@@ -34,8 +35,11 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.ProvidedOrSuppliedTableConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
|
||||
@@ -47,6 +51,8 @@ import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataPro
|
||||
import com.kingsrook.qqq.backend.core.model.scheduledjobs.ScheduledJobsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.common.TimeZonePossibleValueSourceMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.simple.SimpleSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
/** Registers the locally usable qbits against the demo's relational backend. */
|
||||
@@ -111,6 +117,8 @@ final class DemoQBits
|
||||
.withCategory(WebhookEventCategory.STORE).withTableName("order")
|
||||
.withCustomizer(new QCodeReference(DemoOrderWebhookContent.class)));
|
||||
|
||||
moveTestEventFormAfterPreview(instance);
|
||||
|
||||
new WorkflowsQBitProducer()
|
||||
.withQBitConfig(new WorkflowsQBitConfig().withTableMetaDataCustomizer(tableCustomizer))
|
||||
.produce(instance).addSelfToInstance(instance);
|
||||
@@ -156,14 +164,23 @@ final class DemoQBits
|
||||
.withOpensearchUsername(settings.get("OPENSEARCH_USER"))
|
||||
.withOpensearchPassword(settings.get("OPENSEARCH_PASSWORD"))
|
||||
.withUseSsl(Boolean.parseBoolean(settings.getOrDefault("OPENSEARCH_SSL", "false")))
|
||||
// quick search 1.x connects at startup unless degraded; 0.2.x did not (the enterprise profile bumped it)
|
||||
.withStartupMode(QuickSearchStartupMode.DEGRADED)
|
||||
.withSearchableTable("customer", List.of(new SearchableFieldConfig("name"),
|
||||
new SearchableFieldConfig("email")));
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// quick search 1.x registers its own QBitMetaData when produced //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
new QuickSearchQBitProducer().withConfig(searchConfig).produce(instance);
|
||||
instance.addQBit(new QBitMetaData().withGroupId("com.kingsrook.qbits")
|
||||
.withArtifactId("quick-search").withVersion("0.2.1-SNAPSHOT").withConfig(searchConfig));
|
||||
|
||||
new SavedBulkLoadProfileMetaDataProvider().defineAll(instance, "postgres", null);
|
||||
new SavedReportsMetaDataProvider().defineAll(instance, "postgres", "sftp", null);
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
// rendered reports are files, one per record, on the SFTP server: core //
|
||||
// now refuses a filesystem table without backend details (qqq#1364) //
|
||||
///////////////////////////////////////////////////////////////////////////
|
||||
instance.getTable("reportStorage").setBackendDetails(new SFTPTableBackendDetails().withBasePath("reportStorage")
|
||||
.withCardinality(Cardinality.ONE).withFileNameFieldName("reference").withContentsFieldName("contents"));
|
||||
new SFTPDataIntegrationQBitProducer().withSftpDataIntegrationQBitConfig(new SFTPDataIntegrationQBitConfig()
|
||||
.withSourceFileTableConfig(ProvidedOrSuppliedTableConfig.provideTableUsingBackendNamed("sftp"))
|
||||
.withDestinationFileTableConfig(ProvidedOrSuppliedTableConfig.provideTableUsingBackendNamed("sftp"))
|
||||
@@ -181,4 +198,27 @@ final class DemoQBits
|
||||
return table;
|
||||
})).produce(instance, "demo-import");
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* qbit-webhooks 0.x puts SendTestEventToWebhook's input form before the
|
||||
* preview step, so the records chosen at init are lost and the run fails
|
||||
* with "No records were selected" (QRun-IO/qqq#1361; fixed on qbit-webhooks
|
||||
* fix/test-event-records). Until a release carries it, move the form after
|
||||
* the first backend step here.
|
||||
*/
|
||||
static void moveTestEventFormAfterPreview(QInstance instance)
|
||||
{
|
||||
QProcessMetaData process = instance.getProcess("SendTestEventToWebhook");
|
||||
if(process == null || process.getStepList() == null || process.getStepList().size() < 2)
|
||||
{
|
||||
return;
|
||||
}
|
||||
List<QStepMetaData> steps = process.getStepList();
|
||||
if(steps.get(0) instanceof QFrontendStepMetaData form && "input".equals(form.getName()) && steps.get(1) instanceof QBackendStepMetaData)
|
||||
{
|
||||
steps.remove(0);
|
||||
steps.add(1, form);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,10 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
@@ -62,13 +66,25 @@ final class DemoRuntimeFeatures
|
||||
instance.withRuntimeService(new QCodeReference(TransientNotesService.class));
|
||||
instance.addProcess(new QProcessMetaData().withName("calculateOrderTotal").withLabel("Calculate Order Total (JavaScript)")
|
||||
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION))
|
||||
// the inputs as a form, so interactive clients ask for them (QRun-IO/qqq-all#45)
|
||||
.withStep(new QFrontendStepMetaData().withName("input").withLabel("Order line")
|
||||
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
|
||||
.withFormField(new QFieldMetaData("quantity", QFieldType.INTEGER).withLabel("Quantity").withIsRequired(true)
|
||||
.withHelpContent(new QHelpContent("1 to 1000")))
|
||||
.withFormField(new QFieldMetaData("unitPriceCents", QFieldType.INTEGER).withLabel("Unit price (cents)").withIsRequired(true)
|
||||
.withHelpContent(new QHelpContent("0 to 1000000"))))
|
||||
.withStep(new QBackendStepMetaData().withName("calculate")
|
||||
.withCode(new QCodeReference(CalculateOrderTotalStep.class))
|
||||
.withInputData(new QFunctionInputMetaData().withFieldList(List.of(
|
||||
new QFieldMetaData("quantity", QFieldType.INTEGER),
|
||||
new QFieldMetaData("unitPriceCents", QFieldType.INTEGER))))
|
||||
.withOutputMetaData(new QFunctionOutputMetaData().withFieldList(List.of(
|
||||
new QFieldMetaData("totalCents", QFieldType.INTEGER))))));
|
||||
new QFieldMetaData("totalCents", QFieldType.INTEGER)))))
|
||||
.withStep(new QFrontendStepMetaData().withName("result").withLabel("Total")
|
||||
.withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM))
|
||||
.withViewField(new QFieldMetaData("quantity", QFieldType.INTEGER).withLabel("Quantity"))
|
||||
.withViewField(new QFieldMetaData("unitPriceCents", QFieldType.INTEGER).withLabel("Unit price (cents)"))
|
||||
.withViewField(new QFieldMetaData("totalCents", QFieldType.INTEGER).withLabel("Total (cents)"))));
|
||||
QJavalinMetaData.ofOrWithNew(instance).withAdditionalRouteProviderReference(new QCodeReference(DemoCatalogRoutes.class));
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,9 @@ package com.kingsrook.qqq.all;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
@@ -54,7 +56,7 @@ public class DemoUsers implements QAuthenticationModuleCustomizerInterface
|
||||
public void customizeSession(QInstance instance, QSession session, Map<String, Object> context)
|
||||
{
|
||||
session.withPermissions("apiCatalog.read", "demoNote.read");
|
||||
Boolean admin = ADMIN_SESSION.equals(session.getUuid());
|
||||
Boolean admin = ADMIN_SESSION.equals(session.getUuid()) || EnterpriseFeatures.isEnabled();
|
||||
String name = admin ? "Admin" : "Demo User";
|
||||
String userId = admin ? "core:admin" : "core:demo";
|
||||
session.getUser().setIdReference(userId);
|
||||
@@ -84,6 +86,50 @@ public class DemoUsers implements QAuthenticationModuleCustomizerInterface
|
||||
session.withPermission(processName + ".hasAccess");
|
||||
}
|
||||
}
|
||||
if(EnterpriseFeatures.isEnabled())
|
||||
{
|
||||
grantEnterpriseAccess(instance, session, userId);
|
||||
}
|
||||
session.withValueForFrontend("user", new LinkedHashMap<>(Map.of("name", name, "email", userId + "@example.test")));
|
||||
if(EnterpriseFeatures.isEnabled())
|
||||
{
|
||||
EnterpriseSessions.record(session);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* The enterprise test server: every session is an all-access admin, so the
|
||||
* terminal clients can reach every qbit surface.
|
||||
*/
|
||||
static void grantEnterpriseAccess(QInstance instance, QSession session, String userId)
|
||||
{
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// enterprise test server: every session is an all-access admin, so //
|
||||
// the terminal clients can reach every qbit surface //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
session.withPermissions(PermissionsHelper.getAllAvailablePermissionNames(instance));
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// permissions some modules check by literal name, outside the //
|
||||
// table/process permission rules (QRun-IO/qqq#1362) //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
session.withPermissions("jobs_job.read", "jobs_admin", "data_analysis.admin", "assistant.hasAccess", "assistant.admin");
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
// qbit-collaboration's custom checkers (Custom Fields, tags, the //
|
||||
// admin tables) test these names; without them the Custom Fields //
|
||||
// table answered 403 "This needs collaboration.view." (#47) //
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
session.withPermissions("collaboration.view", "collaboration.admin", "collaboration.fields.manage", "collaboration.tags.manage");
|
||||
for(QSecurityKeyType keyType : instance.getSecurityKeyTypes().values())
|
||||
{
|
||||
if(keyType.getAllAccessKeyName() != null)
|
||||
{
|
||||
session.withSecurityKeyValue(keyType.getAllAccessKeyName(), true);
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// per-user keys (owner locks of qbit tables) take the user id //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
session.withSecurityKeyValue(keyType.getName(), userId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.time.Duration;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.function.Predicate;
|
||||
import com.kingsrook.qbits.assistant.AssistantQBitConfig;
|
||||
import com.kingsrook.qbits.assistant.model.AssistantProvider;
|
||||
import com.kingsrook.qbits.assistant.provider.anthropic.AnthropicModelProvider;
|
||||
import com.kingsrook.qbits.assistant.provider.anthropic.AnthropicProviderType;
|
||||
import com.kingsrook.qbits.assistant.provider.openai.OpenAiCompatibleProviderType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The assistant on the enterprise test server: two provider rows, kept in
|
||||
** step with the environment at every start.
|
||||
**
|
||||
** - "Anthropic (Claude)": type anthropic, key variable ANTHROPIC_API_KEY
|
||||
** (only the NAME is stored; enterprise-server.sh passes the value from its
|
||||
** private 0600 secrets file). Enabled when the variable is set.
|
||||
** - "Local Ollama": type openai-compatible, kind OLLAMA, base URL
|
||||
** http://127.0.0.1:11434/v1, model QQQ_ASSISTANT_OLLAMA_MODEL (default
|
||||
** llama3.3:70b-instruct-q4_K_M), no key. Enabled when Ollama answers and
|
||||
** lists that model.
|
||||
**
|
||||
** The default is Anthropic when its key is set, else Ollama when reachable;
|
||||
** with neither, both rows are disabled and GET /assistant/config answers
|
||||
** NOT_CONFIGURED with a message that says what to configure.
|
||||
*******************************************************************************/
|
||||
final class EnterpriseAssistant
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(EnterpriseAssistant.class);
|
||||
|
||||
static final String ANTHROPIC_ROW = "Anthropic (Claude)";
|
||||
static final String OLLAMA_ROW = "Local Ollama";
|
||||
static final String ANTHROPIC_KEY_ENV = "ANTHROPIC_API_KEY";
|
||||
static final String OLLAMA_MODEL_ENV = "QQQ_ASSISTANT_OLLAMA_MODEL";
|
||||
static final String OLLAMA_URL_ENV = "QQQ_ASSISTANT_OLLAMA_URL";
|
||||
static final String OLLAMA_BASE_URL = "http://127.0.0.1:11434/v1";
|
||||
static final String OLLAMA_MODEL = "llama3.3:70b-instruct-q4_K_M";
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
// a local 70B model reads the tool catalog slowly: give a turn and one //
|
||||
// model call room (the qbit defaults are 300 s and 120 s) //
|
||||
/////////////////////////////////////////////////////////////////////////////
|
||||
static final Integer MAX_TURN_SECONDS = 900;
|
||||
static final Integer OLLAMA_REQUEST_SECONDS = 600;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Which provider answers, from the environment at startup.
|
||||
*******************************************************************************/
|
||||
enum Choice
|
||||
{
|
||||
ANTHROPIC,
|
||||
OLLAMA,
|
||||
NONE
|
||||
}
|
||||
|
||||
|
||||
|
||||
private EnterpriseAssistant()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The QBit config: both provider types, the Ollama URL allowed as a base
|
||||
** URL override, and the code-registered Anthropic fallback.
|
||||
*******************************************************************************/
|
||||
static AssistantQBitConfig config(String backendName, String uiApiBaseUrl, Function<String, String> env)
|
||||
{
|
||||
return new AssistantQBitConfig()
|
||||
.withBackendName(backendName).withUiApiBaseUrl(uiApiBaseUrl)
|
||||
.withProviderType(new AnthropicProviderType())
|
||||
.withProviderType(new OpenAiCompatibleProviderType())
|
||||
.withAllowedBaseUrls(Set.of(ollamaBaseUrl(env)))
|
||||
.withMaxTurnSeconds(MAX_TURN_SECONDS);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Anthropic when its key is set, else Ollama when reachable, else none.
|
||||
*******************************************************************************/
|
||||
static Choice choose(Function<String, String> env, Predicate<String> ollamaHasModel)
|
||||
{
|
||||
String key = env.apply(ANTHROPIC_KEY_ENV);
|
||||
if(key != null && !key.isBlank())
|
||||
{
|
||||
return Choice.ANTHROPIC;
|
||||
}
|
||||
return ollamaHasModel.test(ollamaModel(env)) ? Choice.OLLAMA : Choice.NONE;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Ollama model (QQQ_ASSISTANT_OLLAMA_MODEL, default llama3.3 70B).
|
||||
*******************************************************************************/
|
||||
static String ollamaModel(Function<String, String> env)
|
||||
{
|
||||
String model = env.apply(OLLAMA_MODEL_ENV);
|
||||
return model == null || model.isBlank() ? OLLAMA_MODEL : model.trim();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Ollama base URL (QQQ_ASSISTANT_OLLAMA_URL, default 127.0.0.1:11434/v1).
|
||||
*******************************************************************************/
|
||||
static String ollamaBaseUrl(Function<String, String> env)
|
||||
{
|
||||
String url = env.apply(OLLAMA_URL_ENV);
|
||||
return url == null || url.isBlank() ? OLLAMA_BASE_URL : url.trim();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The two provider rows for a choice (id left out).
|
||||
*******************************************************************************/
|
||||
static List<QRecord> rows(Choice choice, Function<String, String> env)
|
||||
{
|
||||
QRecord anthropic = new QRecord()
|
||||
.withValue("name", ANTHROPIC_ROW)
|
||||
.withValue("providerType", AnthropicModelProvider.NAME)
|
||||
.withValue("apiKeyEnvName", ANTHROPIC_KEY_ENV)
|
||||
.withValue("enabled", choice == Choice.ANTHROPIC)
|
||||
.withValue("isDefault", choice == Choice.ANTHROPIC);
|
||||
QRecord ollama = new QRecord()
|
||||
.withValue("name", OLLAMA_ROW)
|
||||
.withValue("providerType", new OpenAiCompatibleProviderType().getType())
|
||||
.withValue("kind", "OLLAMA")
|
||||
.withValue("location", "LOCAL")
|
||||
.withValue("baseUrl", ollamaBaseUrl(env))
|
||||
.withValue("model", ollamaModel(env))
|
||||
.withValue("requestTimeoutSeconds", OLLAMA_REQUEST_SECONDS)
|
||||
.withValue("enabled", choice == Choice.OLLAMA)
|
||||
.withValue("isDefault", choice == Choice.OLLAMA);
|
||||
return List.of(anthropic, ollama);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Insert or update the two rows (matched by name) for this start. Runs
|
||||
** with the caller's QContext (the system session at startup).
|
||||
*******************************************************************************/
|
||||
static Choice reconcile(QInstance instance, Function<String, String> env, Predicate<String> ollamaHasModel) throws QException
|
||||
{
|
||||
if(instance.getTable(AssistantProvider.TABLE_NAME) == null)
|
||||
{
|
||||
return Choice.NONE;
|
||||
}
|
||||
Choice choice = choose(env, ollamaHasModel);
|
||||
List<QRecord> wanted = rows(choice, env);
|
||||
|
||||
QueryInput queryInput = new QueryInput(AssistantProvider.TABLE_NAME);
|
||||
queryInput.setFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.IN, List.of(ANTHROPIC_ROW, OLLAMA_ROW))));
|
||||
Map<Serializable, QRecord> existing = new java.util.HashMap<>();
|
||||
for(QRecord record : new QueryAction().execute(queryInput).getRecords())
|
||||
{
|
||||
existing.putIfAbsent(record.getValueString("name"), record);
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// the non-default row first, so marking the default one last leaves it //
|
||||
// as the only default (the qbit clears other defaults on save) //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
List<QRecord> ordered = wanted.stream().sorted((a, b) -> Boolean.compare(Boolean.TRUE.equals(a.getValueBoolean("isDefault")), Boolean.TRUE.equals(b.getValueBoolean("isDefault")))).toList();
|
||||
for(QRecord row : ordered)
|
||||
{
|
||||
QRecord current = existing.get(row.getValueString("name"));
|
||||
if(current == null)
|
||||
{
|
||||
new InsertAction().execute(new InsertInput(AssistantProvider.TABLE_NAME).withRecord(row));
|
||||
}
|
||||
else
|
||||
{
|
||||
row.setValue("id", current.getValue("id"));
|
||||
new UpdateAction().execute(new UpdateInput(AssistantProvider.TABLE_NAME).withRecord(row));
|
||||
}
|
||||
}
|
||||
return choice;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reconcile from the process environment; a failure is logged and leaves
|
||||
** the server running (the assistant then reports NOT_CONFIGURED).
|
||||
*******************************************************************************/
|
||||
static void reconcileAtStartup(QInstance instance)
|
||||
{
|
||||
try
|
||||
{
|
||||
Choice choice = reconcile(instance, System::getenv, model -> ollamaHasModel(ollamaBaseUrl(System::getenv), model));
|
||||
LOG.info("Enterprise assistant model provider: " + choice);
|
||||
}
|
||||
catch(QException | RuntimeException e)
|
||||
{
|
||||
LOG.warn("Could not set up the enterprise assistant provider rows", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** True when Ollama answers GET {base}/models within 3 s and lists the model.
|
||||
*******************************************************************************/
|
||||
static boolean ollamaHasModel(String baseUrl, String model)
|
||||
{
|
||||
try
|
||||
{
|
||||
HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)).build();
|
||||
HttpRequest request = HttpRequest.newBuilder(URI.create(baseUrl + "/models")).timeout(Duration.ofSeconds(3)).GET().build();
|
||||
HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());
|
||||
return response.statusCode() == 200 && response.body().contains("\"" + model + "\"");
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
if(e instanceof InterruptedException)
|
||||
{
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.security.KeyPair;
|
||||
import java.security.KeyPairGenerator;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.TreeMap;
|
||||
import java.util.Properties;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.alerts.AlertsQBitConfig;
|
||||
import com.kingsrook.qbits.alerts.AlertsQBitProducer;
|
||||
import com.kingsrook.qbits.assistant.AssistantQBitProducer;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitConfig;
|
||||
import com.kingsrook.qbits.backups.BackupsQBitProducer;
|
||||
import com.kingsrook.qbits.backups.storage.InMemoryBackupStorage;
|
||||
import com.kingsrook.qbits.cdn.CdnQBitConfig;
|
||||
import com.kingsrook.qbits.cdn.CdnQBitProducer;
|
||||
import com.kingsrook.qbits.cluster.ClusterQBitConfig;
|
||||
import com.kingsrook.qbits.cluster.ClusterQBitProducer;
|
||||
import com.kingsrook.qbits.collaboration.CollaborationQBitConfig;
|
||||
import com.kingsrook.qbits.collaboration.CollaborationQBitProducer;
|
||||
import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig;
|
||||
import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitProducer;
|
||||
import com.kingsrook.qbits.dbperformance.DbPerfQBitConfig;
|
||||
import com.kingsrook.qbits.dbperformance.DbPerfQBitProducer;
|
||||
import com.kingsrook.qbits.email.EmailQBitConfig;
|
||||
import com.kingsrook.qbits.email.EmailQBitProducer;
|
||||
import com.kingsrook.qbits.filegateway.FileGatewayQBitConfig;
|
||||
import com.kingsrook.qbits.filegateway.FileGatewayQBitProducer;
|
||||
import com.kingsrook.qbits.help.HelpQBitConfig;
|
||||
import com.kingsrook.qbits.help.HelpQBitProducer;
|
||||
import com.kingsrook.qbits.jobs.JobsQBitConfig;
|
||||
import com.kingsrook.qbits.jobs.JobsQBitProducer;
|
||||
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitConfig;
|
||||
import com.kingsrook.qbits.objectstorage.ObjectStorageQBitProducer;
|
||||
import com.kingsrook.qbits.objectstorage.config.AddressingStyle;
|
||||
import com.kingsrook.qbits.objectstorage.config.BucketRole;
|
||||
import com.kingsrook.qbits.objectstorage.config.ProviderType;
|
||||
import com.kingsrook.qbits.objectstorage.config.StorageBucketConfig;
|
||||
import com.kingsrook.qbits.objectstorage.config.StorageConnectionConfig;
|
||||
import com.kingsrook.qbits.qacapture.QaCaptureQBitConfig;
|
||||
import com.kingsrook.qbits.qacapture.QaCaptureQBitProducer;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig;
|
||||
import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer;
|
||||
import com.kingsrook.qbits.quicksearch.SearchableFieldConfig;
|
||||
import com.kingsrook.qbits.quicksearch.SearchableTableConfig;
|
||||
import com.kingsrook.qbits.redis.RedisConnectionConfig;
|
||||
import com.kingsrook.qbits.redis.RedisQBitConfig;
|
||||
import com.kingsrook.qbits.redis.RedisQBitProducer;
|
||||
import com.kingsrook.qbits.releasenotes.ReleaseNotesQBitConfig;
|
||||
import com.kingsrook.qbits.releasenotes.ReleaseNotesQBitProducer;
|
||||
import com.kingsrook.qbits.releasenotes.sources.ComponentVersionsSource;
|
||||
import com.kingsrook.qbits.secureddata.SecuredDataQBitConfig;
|
||||
import com.kingsrook.qbits.secureddata.SecuredDataQBitProducer;
|
||||
import com.kingsrook.qbits.sessionstore.QSessionStoreProviderType;
|
||||
import com.kingsrook.qbits.sessionstore.QSessionStoreQBitConfig;
|
||||
import com.kingsrook.qbits.sessionstore.QSessionStoreQBitProducer;
|
||||
import com.kingsrook.qbits.slack.SlackQBitConfig;
|
||||
import com.kingsrook.qbits.slack.SlackQBitProducer;
|
||||
import com.kingsrook.qbits.sqlworkbench.SqlWorkbenchQBitConfig;
|
||||
import com.kingsrook.qbits.sqlworkbench.SqlWorkbenchQBitProducer;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitConfig;
|
||||
import com.kingsrook.qbits.wormaudit.WormAuditQBitProducer;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
|
||||
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.columnstats.ColumnStatsStep;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.quartz.QuartzSchedulerMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppChildMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.querystats.QueryStatMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.savedbulkloadprofiles.SavedBulkLoadProfileMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.savedreports.SavedReportsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.savedviews.SavedViewsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.scripts.ScriptsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.actions.preferences.UserPreferencesKeyValueStoreMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.sharing.SharingMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.collaboration.RecordFeature;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.AbstractFilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemTableBackendDetails;
|
||||
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The "enterprise" profile (QQQ_ALL_PROFILE=enterprise): the core profile plus
|
||||
** every qbit and core module an enterprise rollout runs, all on the local H2
|
||||
** backend, for exercising the terminal clients against a large instance.
|
||||
**
|
||||
** Each feature is named; QQQ_ALL_ENTERPRISE_EXCLUDE (comma separated names)
|
||||
** leaves features out. Quick search needs QQQ_ALL_OPENSEARCH_HOST and redis
|
||||
** needs QQQ_ALL_REDIS_HOST; without them those two are left out.
|
||||
*******************************************************************************/
|
||||
final class EnterpriseFeatures
|
||||
{
|
||||
static final String PROFILE = "enterprise";
|
||||
|
||||
private static final String SAMPLE_PROVIDER = "com.kingsrook.sampleapp.metadata.SampleMetaDataProvider";
|
||||
|
||||
private interface Feature
|
||||
{
|
||||
void add(QInstance instance) throws Exception;
|
||||
}
|
||||
|
||||
|
||||
|
||||
private EnterpriseFeatures()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** True when the process runs the enterprise profile.
|
||||
*******************************************************************************/
|
||||
static boolean isEnabled()
|
||||
{
|
||||
return PROFILE.equals(System.getenv("QQQ_ALL_PROFILE"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Names the operator excluded.
|
||||
*******************************************************************************/
|
||||
static Set<String> excluded()
|
||||
{
|
||||
String value = System.getenv().getOrDefault("QQQ_ALL_ENTERPRISE_EXCLUDE", "");
|
||||
return Arrays.stream(value.split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toSet());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Add every enterprise feature that is not excluded.
|
||||
*******************************************************************************/
|
||||
static void add(QInstance instance, String backendName, int port) throws QException
|
||||
{
|
||||
String schedulerName = "demoScheduler";
|
||||
String baseUrl = "http://127.0.0.1:" + port;
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// qbits schedule with cron expressions, which the simple scheduler of //
|
||||
// the core profile cannot run: swap it for an in-memory Quartz //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
Properties quartzProperties = new Properties();
|
||||
quartzProperties.put("org.quartz.scheduler.instanceName", "qqqAllEnterprise");
|
||||
quartzProperties.put("org.quartz.threadPool.threadCount", "4");
|
||||
quartzProperties.put("org.quartz.jobStore.class", "org.quartz.simpl.RAMJobStore");
|
||||
instance.getSchedulers().put(schedulerName, new QuartzSchedulerMetaData().withProperties(quartzProperties).withName(schedulerName));
|
||||
MetaDataCustomizerInterface<QTableMetaData> tableCustomizer = (i, table) ->
|
||||
{
|
||||
if(table.getBackendName() == null || backendName.equals(table.getBackendName()))
|
||||
{
|
||||
table.setBackendName(backendName);
|
||||
table.setBackendDetails(new RDBMSTableBackendDetails()
|
||||
.withTableName(QInstanceEnricher.inferBackendName(table.getName())));
|
||||
QInstanceEnricher.setInferredFieldBackendNames(table);
|
||||
}
|
||||
return table;
|
||||
};
|
||||
Consumer<QTableMetaData> enricher = table ->
|
||||
{
|
||||
try
|
||||
{
|
||||
tableCustomizer.customizeMetaData(instance, table);
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
throw new IllegalStateException(e);
|
||||
}
|
||||
};
|
||||
|
||||
Map<String, Feature> features = new LinkedHashMap<>();
|
||||
|
||||
/////////////////////////
|
||||
// core qqq modules //
|
||||
/////////////////////////
|
||||
features.put("audits", i ->
|
||||
{
|
||||
new AuditsMetaDataProvider().defineAll(i, backendName, enricher);
|
||||
for(String tableName : List.of("customer", "order", "address"))
|
||||
{
|
||||
i.getTable(tableName).setAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD));
|
||||
}
|
||||
});
|
||||
features.put("scripts", i ->
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// the scripts provider defines its own "tables" possible values //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
i.getPossibleValueSources().remove("tables");
|
||||
new ScriptsMetaDataProvider().defineAll(i, backendName, enricher);
|
||||
});
|
||||
features.put("savedViews", i -> new SavedViewsMetaDataProvider().defineAll(i, backendName, enricher));
|
||||
features.put("savedReports", i ->
|
||||
{
|
||||
new SavedReportsMetaDataProvider().defineAll(i, backendName, "files", enricher);
|
||||
//////////////////////////////////////////////////////////////////
|
||||
// rendered reports are files, one per record, under the files //
|
||||
// backend (the provider leaves the storage details to the host) //
|
||||
//////////////////////////////////////////////////////////////////
|
||||
QTableMetaData reportStorage = i.getTable("reportStorage");
|
||||
if(reportStorage != null && reportStorage.getBackendDetails() == null)
|
||||
{
|
||||
reportStorage.setBackendDetails(new FilesystemTableBackendDetails().withBasePath("reportStorage").withCardinality(Cardinality.ONE).withFileNameFieldName("reference").withContentsFieldName("contents"));
|
||||
}
|
||||
ensureFilesystemDirectories(i);
|
||||
});
|
||||
features.put("bulkLoadProfiles", i -> new SavedBulkLoadProfileMetaDataProvider().defineAll(i, backendName, enricher));
|
||||
features.put("sharing", i -> new SharingMetaDataProvider().defineAll(i, null));
|
||||
features.put("queryStats", i -> new QueryStatMetaDataProvider().defineAll(i, backendName, enricher));
|
||||
features.put("userPreferences", i -> new UserPreferencesKeyValueStoreMetaDataProvider().defineAll(i, backendName, null));
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// central health: its tables turn on the health UI API extension, so //
|
||||
// /version lists it (QRun-IO/qqq-all#44). Enterprise only for now: in //
|
||||
// the core profile its backend checks share the cached sqlite //
|
||||
// connection across the test suite's runtimes //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
features.put("health", i -> new HealthMetaDataProvider().defineAll(i, backendName, enricher));
|
||||
features.put("durableJobs", i ->
|
||||
{
|
||||
new DurableJobsMetaDataProvider().defineAll(i, backendName, enricher);
|
||||
i.withSupplementalMetaData(new DurableJobsMetaData().withWorkerThreads(2));
|
||||
});
|
||||
|
||||
/////////////////////////
|
||||
// qbits //
|
||||
/////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// object storage on a local MinIO (enterprise-server.sh starts it): //
|
||||
// collaboration attachments live in its private "collaboration" //
|
||||
// bucket. Credentials are references to the server's env only. //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
String s3Endpoint = System.getenv("S3_ENDPOINT");
|
||||
if(s3Endpoint != null && !s3Endpoint.isBlank())
|
||||
{
|
||||
features.put("objectStorage", i -> new ObjectStorageQBitProducer().withQBitConfig(new ObjectStorageQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName).withAdminApp(true)
|
||||
.withConnection(new StorageConnectionConfig("local").withProviderType(ProviderType.MINIO)
|
||||
.withEndpoint("${env.S3_ENDPOINT}").withRegion("us-east-1").withAddressingStyle(AddressingStyle.PATH)
|
||||
.withAccessKeyRef("${env.S3_ACCESS_KEY}").withSecretKeyRef("${env.S3_SECRET_KEY}"))
|
||||
.withBucket(new StorageBucketConfig("collaboration").withConnection("local")
|
||||
.withBucketName("${env.S3_ATTACHMENTS_BUCKET}").withRole(BucketRole.PRIVATE).withIndexed(false)))
|
||||
.produce(i).addSelfToInstance(i));
|
||||
}
|
||||
|
||||
features.put("jobs", i -> new JobsQBitProducer().withQBitConfig(new JobsQBitConfig()
|
||||
.withSchedulerName(schedulerName)).produce(i).addSelfToInstance(i));
|
||||
features.put("collaboration", i -> new CollaborationQBitProducer().withQBitConfig(new CollaborationQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName).withUiApi(true).withAdminApp(true)
|
||||
.withDefaultFeatures(RecordFeature.values())
|
||||
.withFrontendBaseUrl(baseUrl)).produce(i).addSelfToInstance(i));
|
||||
features.put("alerts", i -> new AlertsQBitProducer().withQBitConfig(new AlertsQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName)
|
||||
.withPublicBaseUrl(baseUrl)).produce(i).addSelfToInstance(i));
|
||||
features.put("dataAnalysis", i -> new DataAnalysisQBitProducer().withQBitConfig(new DataAnalysisQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName)).produce(i).addSelfToInstance(i));
|
||||
features.put("dbPerformance", i -> new DbPerfQBitProducer().withQBitConfig(new DbPerfQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName)).produce(i).addSelfToInstance(i));
|
||||
features.put("sqlWorkbench", i -> new SqlWorkbenchQBitProducer().withQBitConfig(new SqlWorkbenchQBitConfig()
|
||||
.withBackendName(backendName).withTableMetaDataCustomizer(tableCustomizer)
|
||||
.withAllowedBackendNames(List.of(backendName))).produce(i).addSelfToInstance(i));
|
||||
features.put("help", i -> new HelpQBitProducer().withQBitConfig(new HelpQBitConfig()
|
||||
.withBackendName(backendName).withTableMetaDataCustomizer(tableCustomizer)).produce(i).addSelfToInstance(i));
|
||||
features.put("releaseNotes", i -> new ReleaseNotesQBitProducer().withQBitConfig(new ReleaseNotesQBitConfig()
|
||||
.withBackendName(backendName).withApplicationName("qqqAll").withApplicationLabel("QQQ All")
|
||||
.withAppVersion("4.1.0").withAllowNonReleaseVersions(true)
|
||||
.withSource(new ComponentVersionsSource())).produce(i).addSelfToInstance(i));
|
||||
features.put("qaCapture", i -> new QaCaptureQBitProducer().withQBitConfig(new QaCaptureQBitConfig()
|
||||
.withBackendName(backendName).withEnvironment("local").withBackendVersion("4.1.0-SNAPSHOT"))
|
||||
.produce(i).addSelfToInstance(i));
|
||||
features.put("securedData", i -> new SecuredDataQBitProducer().withQBitConfig(new SecuredDataQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName)
|
||||
.withTableMetaDataCustomizer(tableCustomizer)).produce(i).addSelfToInstance(i));
|
||||
features.put("email", i -> new EmailQBitProducer().withQBitConfig(new EmailQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName).withPublicBaseUrl(baseUrl)
|
||||
.withTableMetaDataCustomizer(tableCustomizer)).produce(i).addSelfToInstance(i));
|
||||
features.put("slack", i -> new SlackQBitProducer().withQBitConfig(new SlackQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName).withPublicBaseUrl(baseUrl)
|
||||
.withTableMetaDataCustomizer(tableCustomizer)).produce(i).addSelfToInstance(i));
|
||||
features.put("cluster", i -> new ClusterQBitProducer().withQBitConfig(new ClusterQBitConfig()
|
||||
.withBackendName(backendName)).produce(i).addSelfToInstance(i));
|
||||
features.put("cdn", i -> new CdnQBitProducer().withQBitConfig(new CdnQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName)
|
||||
.withPurgeableTables(List.of("customer")).withTableMetaDataCustomizer(tableCustomizer))
|
||||
.produce(i).addSelfToInstance(i));
|
||||
features.put("backups", i -> new BackupsQBitProducer().withQBitConfig(new BackupsQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName)
|
||||
.withStorageResolver(name -> new InMemoryBackupStorage())).produce(i).addSelfToInstance(i));
|
||||
features.put("fileGateway", i -> new FileGatewayQBitProducer().withQBitConfig(new FileGatewayQBitConfig()
|
||||
.withBackendName(backendName).withSftpEnabled(false).withSchedulerName(schedulerName)
|
||||
.withTableMetaDataCustomizer(tableCustomizer)).produce(i).addSelfToInstance(i));
|
||||
features.put("sessionStore", i ->
|
||||
{
|
||||
new QSessionStoreQBitProducer().withConfig(new QSessionStoreQBitConfig()
|
||||
.withProviderType(QSessionStoreProviderType.TABLE_BASED).withBackendName(backendName)
|
||||
.withDefaultTtl(Duration.ofHours(8)).withEnableSlidingExpiration(true)).produce(i, "session-store");
|
||||
if(i.getProcess("cleanExpiredSessions") != null && i.getProcess("cleanExpiredSessions").getSchedule() != null)
|
||||
{
|
||||
i.getProcess("cleanExpiredSessions").getSchedule().setSchedulerName(schedulerName);
|
||||
}
|
||||
});
|
||||
features.put("wormAudit", i ->
|
||||
{
|
||||
KeyPairGenerator generator = KeyPairGenerator.getInstance("Ed25519");
|
||||
KeyPair keyPair = generator.generateKeyPair();
|
||||
new WormAuditQBitProducer().withQBitConfig(new WormAuditQBitConfig()
|
||||
.withBackendName(backendName).withSchedulerName(schedulerName).withAppRole("app")
|
||||
.withSigningKey(() -> new WormAuditQBitConfig.SigningKey("local", keyPair.getPrivate()))
|
||||
.withTrustedPublicKeys(Map.of("local", keyPair.getPublic()))
|
||||
.withTableMetaDataCustomizer(tableCustomizer)).produce(i).addSelfToInstance(i);
|
||||
});
|
||||
features.put("assistant", i -> new AssistantQBitProducer().withQBitConfig(EnterpriseAssistant.config(backendName, baseUrl, System::getenv)
|
||||
.withTableMetaDataCustomizer(tableCustomizer)).produce(i).addSelfToInstance(i));
|
||||
|
||||
if(isSampleAppOnClasspath())
|
||||
{
|
||||
features.put("sampleApp", EnterpriseFeatures::mergeSampleApp);
|
||||
}
|
||||
|
||||
String opensearchHost = System.getenv("QQQ_ALL_OPENSEARCH_HOST");
|
||||
if(opensearchHost != null && !opensearchHost.isBlank())
|
||||
{
|
||||
features.put("quickSearch", i ->
|
||||
{
|
||||
QuickSearchQBitConfig searchConfig = new QuickSearchQBitConfig()
|
||||
.withBackendName(backendName)
|
||||
.withOpensearchHost(opensearchHost)
|
||||
.withOpensearchPort(Integer.valueOf(System.getenv().getOrDefault("QQQ_ALL_OPENSEARCH_PORT", "9200")))
|
||||
.withOpensearchIndexName(System.getenv().getOrDefault("QQQ_ALL_OPENSEARCH_INDEX", "qqq-all-enterprise"))
|
||||
.withUseSsl(false)
|
||||
.withAllowPlaintextCredentials(true)
|
||||
.withSchedulerName(schedulerName)
|
||||
.withTableMetaDataCustomizer(tableCustomizer)
|
||||
// new tables are queued for their first build, which the basepull runs: keep it quick
|
||||
.withBasepullRepeatSeconds(60);
|
||||
for(SearchableTableConfig table : searchableTables(i))
|
||||
{
|
||||
searchConfig.withSearchableTable(table);
|
||||
}
|
||||
new QuickSearchQBitProducer().withConfig(searchConfig).produce(i).addSelfToInstance(i);
|
||||
});
|
||||
}
|
||||
|
||||
String redisHost = System.getenv("QQQ_ALL_REDIS_HOST");
|
||||
if(redisHost != null && !redisHost.isBlank())
|
||||
{
|
||||
features.put("redis", i -> new RedisQBitProducer().withQBitConfig(new RedisQBitConfig()
|
||||
.withConnection("realtime", new RedisConnectionConfig().withHost(redisHost)
|
||||
.withPort(Integer.valueOf(System.getenv().getOrDefault("QQQ_ALL_REDIS_PORT", "6379")))
|
||||
.withTls(false))
|
||||
.withRealtime("realtime", "local", "qqq-all")).produce(i).addSelfToInstance(i));
|
||||
}
|
||||
|
||||
Set<String> excluded = excluded();
|
||||
for(Map.Entry<String, Feature> entry : features.entrySet())
|
||||
{
|
||||
if(excluded.contains(entry.getKey()))
|
||||
{
|
||||
System.out.println("[enterprise] skipped " + entry.getKey());
|
||||
continue;
|
||||
}
|
||||
try
|
||||
{
|
||||
entry.getValue().add(instance);
|
||||
System.out.println("[enterprise] added " + entry.getKey());
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
if("true".equals(System.getenv("QQQ_ALL_ENTERPRISE_LENIENT")))
|
||||
{
|
||||
System.out.println("[enterprise] FAILED " + entry.getKey() + ": " + e);
|
||||
Arrays.stream(e.getStackTrace()).limit(6).forEach(frame -> System.out.println("[enterprise] at " + frame));
|
||||
continue;
|
||||
}
|
||||
throw new QException("Enterprise feature " + entry.getKey() + " failed: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
enableColumnStatistics(instance);
|
||||
auditDataTables(instance);
|
||||
addNavigationApps(instance);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** QQQ turns QUERY_STATS (column statistics: counts, distinct values, min/max
|
||||
** and averages per field) off on every backend by default. Turn it on for the
|
||||
** backends whose module can aggregate, RDBMS (H2, SQLite, the sample's
|
||||
** in-memory H2) and the memory backend, so their tables offer statistics as
|
||||
** the web UI and the terminal clients expect; filesystem and API backends
|
||||
** cannot aggregate and stay without it. Tables of those backends that list
|
||||
** QUERY_STATS as disabled keep it off. The columnStats process that serves
|
||||
** the statistics is added when the instance lacks it; core guards it by field
|
||||
** access (QRun-IO/qqq#1367: only fully readable fields, results tagged).
|
||||
*******************************************************************************/
|
||||
static void enableColumnStatistics(QInstance instance)
|
||||
{
|
||||
////////////////////////////////////////////////////////////////////
|
||||
// the UI API serves column statistics through this process //
|
||||
////////////////////////////////////////////////////////////////////
|
||||
if(instance.getProcess(ColumnStatsStep.getProcessMetaData().getName()) == null)
|
||||
{
|
||||
instance.addProcess(ColumnStatsStep.getProcessMetaData());
|
||||
}
|
||||
|
||||
for(QBackendMetaData backend : instance.getBackends().values())
|
||||
{
|
||||
if(backend instanceof RDBMSBackendMetaData || new MemoryBackendModule().getBackendType().equals(backend.getBackendType()))
|
||||
{
|
||||
backend.withCapability(Capability.QUERY_STATS);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Field-level audits on the demo and sample data tables (customer, order and
|
||||
** address get them with the audits feature), when audits are installed.
|
||||
*******************************************************************************/
|
||||
static void auditDataTables(QInstance instance)
|
||||
{
|
||||
if(instance.getTable("audit") == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
for(String tableName : List.of("customer", "order", "orderLine", "address", "person", "pet", "petNote", "carrier"))
|
||||
{
|
||||
QTableMetaData table = instance.getTable(tableName);
|
||||
if(table != null && table.getAuditRules() == null)
|
||||
{
|
||||
table.setAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/** Text fields worth searching besides a table's record label fields. */
|
||||
private static final Set<String> SEARCHABLE_FIELD_NAMES = Set.of("name", "label", "title", "description", "email",
|
||||
"firstName", "lastName", "city", "state", "country", "code", "orderNo", "status");
|
||||
|
||||
/** The backends of the demo and sample data (QBit admin tables live on h2 too, and are left out). */
|
||||
private static final Set<String> SAMPLE_BACKENDS = Set.of("rdbms", "memory");
|
||||
private static final List<String> DEMO_TABLES = List.of("customer", "order", "address", "shipping_country",
|
||||
"shipping_stateProvince", "shipping_city");
|
||||
|
||||
/*******************************************************************************
|
||||
** Quick Search registrations for the demo tables and every visible sample-app
|
||||
** table with a record label: the label fields plus the obvious text fields,
|
||||
** STRING fields only (OpenSearch indexes them as searchable text; dates,
|
||||
** numbers and booleans stay out). Self-priming creates their index rows and
|
||||
** queues each for its first build.
|
||||
*******************************************************************************/
|
||||
static List<SearchableTableConfig> searchableTables(QInstance instance)
|
||||
{
|
||||
List<SearchableTableConfig> tables = new ArrayList<>();
|
||||
for(QTableMetaData table : instance.getTables().values())
|
||||
{
|
||||
boolean demo = DEMO_TABLES.contains(table.getName());
|
||||
boolean sample = SAMPLE_BACKENDS.contains(table.getBackendName()) && !table.getIsHidden()
|
||||
&& table.getRecordLabelFields() != null && !table.getRecordLabelFields().isEmpty();
|
||||
if(!demo && !sample)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
Set<String> names = new LinkedHashSet<>();
|
||||
for(String labelField : Objects.requireNonNullElse(table.getRecordLabelFields(), List.<String>of()))
|
||||
{
|
||||
names.add(labelField);
|
||||
}
|
||||
names.addAll(SEARCHABLE_FIELD_NAMES);
|
||||
List<SearchableFieldConfig> fields = new ArrayList<>();
|
||||
for(String name : names)
|
||||
{
|
||||
QFieldMetaData field = table.getFields().get(name);
|
||||
if(field != null && field.getType() == QFieldType.STRING)
|
||||
{
|
||||
fields.add(new SearchableFieldConfig(name).withWeight(Objects.requireNonNullElse(table.getRecordLabelFields(), List.<String>of()).contains(name) ? 3 : 1));
|
||||
}
|
||||
}
|
||||
if(fields.isEmpty())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
SearchableTableConfig config = new SearchableTableConfig(table.getName(), fields);
|
||||
if((table.getRecordLabelFields() == null || table.getRecordLabelFields().isEmpty()) && fields.stream().anyMatch(f -> "name".equals(f.getFieldName())))
|
||||
{
|
||||
///////////////////////////////////////////////////////////////
|
||||
// label hits by name where the table itself has no label //
|
||||
///////////////////////////////////////////////////////////////
|
||||
config.withRecordLabelFormat("%s", "name");
|
||||
}
|
||||
if(!table.getFields().containsKey("modifyDate"))
|
||||
{
|
||||
config.withBasepullTimestampField(null);
|
||||
}
|
||||
tables.add(config);
|
||||
}
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The core profile has no apps and several qbits only add tables. Put the
|
||||
** demo tables in a "QQQ All" app and every visible table that no app reaches
|
||||
** into an "Enterprise Admin" app, one section per table-name prefix, so the
|
||||
** clients' app tree reaches everything.
|
||||
*******************************************************************************/
|
||||
private static void addNavigationApps(QInstance instance)
|
||||
{
|
||||
List<String> demoTables = List.of("customer", "order", "orderLine", "product", "address", "apiCatalog", "demoNote",
|
||||
"shipping_country", "shipping_stateProvince", "shipping_city");
|
||||
List<QAppChildMetaData> demoChildren = new ArrayList<>();
|
||||
demoTables.stream().map(instance::getTable).filter(Objects::nonNull).forEach(demoChildren::add);
|
||||
List.of("syncOrder", "calculateOrderTotal").stream().map(instance::getProcess).filter(Objects::nonNull).forEach(demoChildren::add);
|
||||
instance.addApp(new QAppMetaData().withName("qqqAll").withLabel("QQQ All").withIcon(new QIcon("storefront"))
|
||||
.withSectionOfChildren(new QAppSection().withName("demoData").withLabel("Demo Data").withIcon(new QIcon("table_chart")), demoChildren));
|
||||
|
||||
Set<String> reached = new HashSet<>();
|
||||
for(QAppMetaData app : instance.getApps().values())
|
||||
{
|
||||
if(app.getChildren() != null)
|
||||
{
|
||||
app.getChildren().stream().filter(c -> c instanceof QTableMetaData).forEach(c -> reached.add(c.getName()));
|
||||
}
|
||||
if(app.getSections() != null)
|
||||
{
|
||||
app.getSections().stream().filter(section -> section.getTables() != null).forEach(section -> reached.addAll(section.getTables()));
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, List<QAppChildMetaData>> byPrefix = new TreeMap<>();
|
||||
for(QTableMetaData table : instance.getTables().values())
|
||||
{
|
||||
if(reached.contains(table.getName()) || table.getIsHidden())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
String backendName = QInstanceEnricher.inferBackendName(table.getName());
|
||||
String prefix = backendName.contains("_") ? backendName.substring(0, backendName.indexOf('_')) : backendName;
|
||||
byPrefix.computeIfAbsent(prefix, k -> new ArrayList<>()).add(table);
|
||||
}
|
||||
QAppMetaData admin = new QAppMetaData().withName("enterpriseAdmin").withLabel("Enterprise Admin").withIcon(new QIcon("admin_panel_settings"));
|
||||
byPrefix.forEach((prefix, tables) -> admin.withSectionOfChildren(new QAppSection().withName("admin_" + prefix)
|
||||
.withLabel(QInstanceEnricher.nameToLabel(prefix)).withIcon(new QIcon("folder")), tables));
|
||||
instance.addApp(admin);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Filesystem tables (rendered report storage) answer 500 when their directory
|
||||
** does not exist yet: create each one under the files backend.
|
||||
*******************************************************************************/
|
||||
private static void ensureFilesystemDirectories(QInstance instance) throws Exception
|
||||
{
|
||||
if(!(instance.getBackend("files") instanceof FilesystemBackendMetaData files))
|
||||
{
|
||||
return;
|
||||
}
|
||||
for(QTableMetaData table : instance.getTables().values())
|
||||
{
|
||||
if("files".equals(table.getBackendName()) && table.getBackendDetails() instanceof AbstractFilesystemTableBackendDetails details && details.getBasePath() != null)
|
||||
{
|
||||
Files.createDirectories(Path.of(files.getBasePath(), details.getBasePath()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** True when the qqq sample project's classes are on the classpath (the start
|
||||
** script adds qqq-sample-project/target/classes).
|
||||
*******************************************************************************/
|
||||
private static boolean isSampleAppOnClasspath()
|
||||
{
|
||||
try
|
||||
{
|
||||
Class.forName(SAMPLE_PROVIDER);
|
||||
return true;
|
||||
}
|
||||
catch(ClassNotFoundException e)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Merge the standard qqq sample app (person, pet, carrier, city files, the
|
||||
** field and adornment labs, its processes, widgets and apps) into this
|
||||
** instance. Its ESB wiring is dropped (this instance has its own broker) and
|
||||
** names this instance already defines win. Reflection keeps the sample
|
||||
** project out of the build; it is a classpath add-on.
|
||||
*******************************************************************************/
|
||||
private static void mergeSampleApp(QInstance instance) throws Exception
|
||||
{
|
||||
Class<?> provider = Class.forName(SAMPLE_PROVIDER);
|
||||
QInstance sample = (QInstance) provider.getMethod("defineInstance").invoke(null);
|
||||
provider.getMethod("primeTestDatabase", String.class).invoke(null, "prime-test-database.sql");
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// the sample's sales module defines its own "customer"; this instance //
|
||||
// already has one, so the sales module stays out //
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
Set<String> skip = Set.of("sales", "customer", "sku", "tag", "sales_order", "sales_order_line", "sales_order_tag",
|
||||
"salesOrderJoinLine", "salesOrderJoinTag", "salesOrderLines", "salesOrderTags");
|
||||
sample.getTables().keySet().removeAll(skip);
|
||||
sample.getPossibleValueSources().keySet().removeAll(skip);
|
||||
sample.getJoins().keySet().removeAll(skip);
|
||||
sample.getWidgets().keySet().removeAll(skip);
|
||||
sample.getApps().keySet().removeAll(skip);
|
||||
sample.getApps().values().forEach(app ->
|
||||
{
|
||||
if(app.getChildren() != null)
|
||||
{
|
||||
app.getChildren().removeIf(child -> skip.contains(child.getName()));
|
||||
}
|
||||
if(app.getSections() != null)
|
||||
{
|
||||
app.getSections().forEach(section ->
|
||||
{
|
||||
if(section.getTables() != null)
|
||||
{
|
||||
section.getTables().removeIf(skip::contains);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
sample.getBackends().forEach((name, value) -> instance.getBackends().putIfAbsent(name, value));
|
||||
sample.getPossibleValueSources().forEach((name, value) -> instance.getPossibleValueSources().putIfAbsent(name, value));
|
||||
sample.getSecurityKeyTypes().forEach((name, value) -> instance.getSecurityKeyTypes().putIfAbsent(name, value));
|
||||
sample.getTables().forEach((name, table) ->
|
||||
{
|
||||
if(table.getSupplementalMetaData() != null)
|
||||
{
|
||||
table.getSupplementalMetaData().values().removeIf(EnterpriseFeatures::isEsb);
|
||||
}
|
||||
instance.getTables().putIfAbsent(name, table);
|
||||
});
|
||||
sample.getProcesses().forEach((name, process) ->
|
||||
{
|
||||
if(process.getSupplementalMetaData() != null)
|
||||
{
|
||||
process.getSupplementalMetaData().values().removeIf(EnterpriseFeatures::isEsb);
|
||||
}
|
||||
instance.getProcesses().putIfAbsent(name, process);
|
||||
});
|
||||
sample.getJoins().forEach((name, value) -> instance.getJoins().putIfAbsent(name, value));
|
||||
sample.getReports().forEach((name, value) -> instance.getReports().putIfAbsent(name, value));
|
||||
sample.getWidgets().forEach((name, value) -> instance.getWidgets().putIfAbsent(name, value));
|
||||
sample.getApps().forEach((name, value) -> instance.getApps().putIfAbsent(name, value));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** ESB supplemental meta data of the sample points at its own destinations.
|
||||
*******************************************************************************/
|
||||
private static boolean isEsb(Object supplemental)
|
||||
{
|
||||
return supplemental != null && supplemental.getClass().getName().startsWith("com.kingsrook.qqq.esb.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import com.kingsrook.qqq.backend.core.modules.authentication.QSessionStoreHelper;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QUser;
|
||||
|
||||
/**
|
||||
* Records mock sign-ins in the session store (qbit-session-store) for the
|
||||
* enterprise profile.
|
||||
*
|
||||
* Only the OAuth2 module writes to the session store; MOCK authentication
|
||||
* builds a fresh session on every request and stores nothing, so the
|
||||
* storedSession table stayed empty and "who is online" could not count
|
||||
* terminals. This is the smallest hook: the mock module's session customizer
|
||||
* calls {@link #record(QSession)}, which stores one row per session uuid
|
||||
* (user id, a lean copy of the session, expiresAt) the first time this JVM
|
||||
* sees it and touches it (sliding expiry) at most once a minute after that.
|
||||
*/
|
||||
final class EnterpriseSessions
|
||||
{
|
||||
static final Duration TOUCH_INTERVAL = Duration.ofMinutes(1);
|
||||
|
||||
private static final Map<String, Long> LAST_SEEN = new ConcurrentHashMap<>();
|
||||
|
||||
private EnterpriseSessions()
|
||||
{
|
||||
}
|
||||
|
||||
static void record(QSession session)
|
||||
{
|
||||
String uuid = session == null ? null : session.getUuid();
|
||||
if(uuid == null || !QSessionStoreHelper.isSessionStoreAvailable())
|
||||
{
|
||||
return;
|
||||
}
|
||||
long now = System.currentTimeMillis();
|
||||
Long last = LAST_SEEN.get(uuid);
|
||||
if(last != null && now - last < TOUCH_INTERVAL.toMillis())
|
||||
{
|
||||
return;
|
||||
}
|
||||
LAST_SEEN.put(uuid, now);
|
||||
if(last == null && QSessionStoreHelper.loadSession(uuid).isEmpty())
|
||||
{
|
||||
QSessionStoreHelper.storeSession(uuid, lean(session), QSessionStoreHelper.getDefaultTtl());
|
||||
}
|
||||
else
|
||||
{
|
||||
QSessionStoreHelper.touchSession(uuid);
|
||||
}
|
||||
}
|
||||
|
||||
/** The identity only: the admin session's thousand permissions do not belong in a row. */
|
||||
static QSession lean(QSession session)
|
||||
{
|
||||
QSession lean = new QSession();
|
||||
lean.setUuid(session.getUuid());
|
||||
lean.setIdReference(session.getUuid());
|
||||
if(session.getUser() != null)
|
||||
{
|
||||
QUser user = new QUser();
|
||||
user.setIdReference(session.getUser().getIdReference());
|
||||
user.setFullName(session.getUser().getFullName());
|
||||
lean.setUser(user);
|
||||
}
|
||||
return (lean);
|
||||
}
|
||||
|
||||
/** Tests start from a clean slate. */
|
||||
static void forget()
|
||||
{
|
||||
LAST_SEEN.clear();
|
||||
}
|
||||
}
|
||||
@@ -11,16 +11,10 @@ import com.kingsrook.qqq.backend.module.filesystem.s3.model.metadata.S3BackendMe
|
||||
/** S3 metadata with an explicit endpoint for MinIO. */
|
||||
public final class MinioBackendMetaData extends S3BackendMetaData
|
||||
{
|
||||
private final String endpoint;
|
||||
|
||||
public MinioBackendMetaData(String endpoint)
|
||||
{
|
||||
this.endpoint = endpoint;
|
||||
setEndpoint(endpoint);
|
||||
setPathStyleAccess(true);
|
||||
setBackendType(MinioBackendModule.class);
|
||||
}
|
||||
|
||||
public String getEndpoint()
|
||||
{
|
||||
return endpoint;
|
||||
}
|
||||
}
|
||||
@@ -6,29 +6,11 @@
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import com.amazonaws.auth.AWSStaticCredentialsProvider;
|
||||
import com.amazonaws.auth.BasicAWSCredentials;
|
||||
import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
import com.kingsrook.qqq.backend.core.actions.interfaces.CountInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.interfaces.DeleteInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.interfaces.InsertInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.interfaces.QStorageInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.interfaces.QueryInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.QBackendModuleDispatcher;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.base.actions.AbstractBaseFilesystemAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.S3BackendModule;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.AbstractS3Action;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3CountAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3DeleteAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3InsertAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3QueryAction;
|
||||
import com.kingsrook.qqq.backend.module.filesystem.s3.actions.S3StorageAction;
|
||||
|
||||
/** Routes every S3 operation to the configured MinIO endpoint. */
|
||||
/** Uses the shared S3 actions with MinIO endpoint metadata. */
|
||||
public final class MinioBackendModule extends S3BackendModule
|
||||
{
|
||||
static
|
||||
@@ -47,92 +29,4 @@ public final class MinioBackendModule extends S3BackendModule
|
||||
{
|
||||
return MinioBackendMetaData.class;
|
||||
}
|
||||
|
||||
private static AmazonS3 client(QBackendMetaData metadata)
|
||||
{
|
||||
MinioBackendMetaData minio = (MinioBackendMetaData) metadata;
|
||||
return AmazonS3ClientBuilder.standard()
|
||||
.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(minio.getEndpoint(), minio.getRegion()))
|
||||
.withPathStyleAccessEnabled(true)
|
||||
.withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(minio.getAccessKey(), minio.getSecretKey())))
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractBaseFilesystemAction<S3ObjectSummary> getActionBase()
|
||||
{
|
||||
return new AbstractS3Action()
|
||||
{
|
||||
@Override
|
||||
protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData metadata)
|
||||
{
|
||||
return client(metadata);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public QueryInterface getQueryInterface()
|
||||
{
|
||||
return new S3QueryAction()
|
||||
{
|
||||
@Override
|
||||
protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData metadata)
|
||||
{
|
||||
return client(metadata);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public InsertInterface getInsertInterface()
|
||||
{
|
||||
return new S3InsertAction()
|
||||
{
|
||||
@Override
|
||||
protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData metadata)
|
||||
{
|
||||
return client(metadata);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public CountInterface getCountInterface()
|
||||
{
|
||||
return new S3CountAction()
|
||||
{
|
||||
@Override
|
||||
protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData metadata)
|
||||
{
|
||||
return client(metadata);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public DeleteInterface getDeleteInterface()
|
||||
{
|
||||
return new S3DeleteAction()
|
||||
{
|
||||
@Override
|
||||
protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData metadata)
|
||||
{
|
||||
return client(metadata);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public QStorageInterface getStorageInterface()
|
||||
{
|
||||
return new S3StorageAction()
|
||||
{
|
||||
@Override
|
||||
protected AmazonS3 buildAmazonS3ClientFromBackendMetaData(QBackendMetaData metadata)
|
||||
{
|
||||
return client(metadata);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -26,9 +26,9 @@ public final class ProfileLauncher
|
||||
static String profile(Map<String, String> environment)
|
||||
{
|
||||
String selected = environment.getOrDefault("QQQ_ALL_PROFILE", "core");
|
||||
if(!"core".equals(selected) && !"full".equals(selected))
|
||||
if(!"core".equals(selected) && !"full".equals(selected) && !EnterpriseFeatures.PROFILE.equals(selected))
|
||||
{
|
||||
throw new IllegalArgumentException("QQQ_ALL_PROFILE must be core or full");
|
||||
throw new IllegalArgumentException("QQQ_ALL_PROFILE must be core, full or enterprise");
|
||||
}
|
||||
return selected;
|
||||
}
|
||||
@@ -38,7 +38,7 @@ public final class ProfileLauncher
|
||||
String selected = profile(environment);
|
||||
Path dataDirectory = Path.of(environment.getOrDefault("QQQ_ALL_DATA_DIR", "./data"));
|
||||
Integer port = Integer.valueOf(environment.getOrDefault("QQQ_ALL_PORT", "8080"));
|
||||
if("core".equals(selected))
|
||||
if("core".equals(selected) || EnterpriseFeatures.PROFILE.equals(selected))
|
||||
{
|
||||
CoreRuntime.launch(dataDirectory, port, environment.getOrDefault("QQQ_ALL_BIND_HOST", "127.0.0.1"));
|
||||
return;
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
@@ -110,12 +111,13 @@ public final class QqqAllApplication extends AbstractQQQApplication
|
||||
|
||||
instance.addTable(new QTableMetaData().withName("customer").withLabel("Customer")
|
||||
.withBackendName("h2").withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s").withRecordLabelFields("name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("email", QFieldType.STRING)));
|
||||
instance.addTable(new QTableMetaData().withName("order").withLabel("Order")
|
||||
.withBackendName("h2").withBackendDetails(new RDBMSTableBackendDetails().withTableName("orders"))
|
||||
.withPrimaryKeyField("id")
|
||||
.withPrimaryKeyField("id").withRecordLabelFormat("%s").withRecordLabelFields("orderNo")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("orderNo", QFieldType.STRING).withBackendName("order_no"))
|
||||
.withField(new QFieldMetaData("customerId", QFieldType.INTEGER).withBackendName("customer_id"))
|
||||
@@ -156,12 +158,16 @@ public final class QqqAllApplication extends AbstractQQQApplication
|
||||
new EsbJavalinMetaDataProducer().produce(instance);
|
||||
|
||||
instance.withSupplementalMetaData(new HealthCheckMetaData().withEnabled(true)
|
||||
.withEndpointPath("/health").withIndicators(List.of(new MemoryHealthIndicator().withThreshold(99))));
|
||||
.withEndpointPath("/health").withIndicators(new ArrayList<>(List.of(new MemoryHealthIndicator().withThreshold(99)))));
|
||||
QJavalinMetaData.ofOrWithNew(instance)
|
||||
.withAdditionalRouteProviderReference(new QCodeReference(JavalinHealthRouteProvider.class));
|
||||
if(includeDemoQbits)
|
||||
{
|
||||
DemoQBits.addCore(instance, "h2");
|
||||
if(EnterpriseFeatures.isEnabled())
|
||||
{
|
||||
EnterpriseFeatures.add(instance, "h2", httpPort);
|
||||
}
|
||||
DemoUsers.protectDemoTables(instance);
|
||||
QJavalinMetaData.ofOrWithNew(instance)
|
||||
.withAdditionalRouteProviderReference(new QCodeReference(DemoOrderWebhookReceiver.class));
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.collaboration.customfields.FieldSecurity;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QUser;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
|
||||
/** The enterprise admin passes qbit-collaboration's Custom Fields checker (qqq-all#47). */
|
||||
class EnterpriseAccessTest
|
||||
{
|
||||
@TempDir Path dataDirectory;
|
||||
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
@Test
|
||||
void adminCanReadAndManageCustomFields() throws Exception
|
||||
{
|
||||
QInstance instance = new QqqAllApplication(dataDirectory, "tcp://127.0.0.1:61616").defineQInstance();
|
||||
QSession plain = new QSession();
|
||||
QContext.init(instance, plain);
|
||||
assertThrows(QPermissionDeniedException.class,
|
||||
() -> new FieldSecurity().checkPermissionsThrowing(new QueryInput("collaboration_field_definition"), null));
|
||||
|
||||
QSession admin = new QSession();
|
||||
DemoUsers.grantEnterpriseAccess(instance, admin, "core:admin");
|
||||
QContext.init(instance, admin);
|
||||
assertDoesNotThrow(() -> new FieldSecurity().checkPermissionsThrowing(new QueryInput("collaboration_field_definition"), null));
|
||||
assertDoesNotThrow(() -> new FieldSecurity().checkPermissionsThrowing(new InsertInput("collaboration_field_definition"), null));
|
||||
}
|
||||
|
||||
@Test
|
||||
void customizedSessionsStillCarryTheUserForTheFrontend() throws Exception
|
||||
{
|
||||
QInstance instance = new QqqAllApplication(dataDirectory, "tcp://127.0.0.1:61616").defineQInstance();
|
||||
QSession session = new QSession();
|
||||
session.setUuid(DemoUsers.ADMIN_SESSION);
|
||||
session.setUser(new QUser());
|
||||
new DemoUsers().customizeSession(instance, session, Map.of());
|
||||
assertEquals("Admin", ((Map<?, ?>) session.getValuesForFrontend().get("user")).get("name"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.assistant.AssistantQBitConfig;
|
||||
import com.kingsrook.qbits.assistant.AssistantQBitProducer;
|
||||
import com.kingsrook.qbits.assistant.model.AssistantProvider;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/** The enterprise assistant's provider rows follow the environment: Anthropic with a key, else Ollama, else none. */
|
||||
class EnterpriseAssistantTest
|
||||
{
|
||||
@TempDir Path dataDirectory;
|
||||
|
||||
@AfterEach
|
||||
void clear()
|
||||
{
|
||||
QContext.clear();
|
||||
MemoryRecordStore.fullReset();
|
||||
}
|
||||
|
||||
@Test
|
||||
void choiceFollowsKeyThenOllama()
|
||||
{
|
||||
assertEquals(EnterpriseAssistant.Choice.ANTHROPIC, EnterpriseAssistant.choose(env(Map.of("ANTHROPIC_API_KEY", "sk-test")), model -> true));
|
||||
assertEquals(EnterpriseAssistant.Choice.OLLAMA, EnterpriseAssistant.choose(env(Map.of("ANTHROPIC_API_KEY", " ")), model -> true));
|
||||
assertEquals(EnterpriseAssistant.Choice.NONE, EnterpriseAssistant.choose(env(Map.of()), model -> false));
|
||||
}
|
||||
|
||||
@Test
|
||||
void ollamaModelAndUrlFromEnvironment()
|
||||
{
|
||||
List<String> asked = new java.util.ArrayList<>();
|
||||
EnterpriseAssistant.choose(env(Map.of("QQQ_ASSISTANT_OLLAMA_MODEL", "qwen3-coder:30b")), model -> asked.add(model));
|
||||
assertEquals(List.of("qwen3-coder:30b"), asked);
|
||||
assertEquals("llama3.3:70b-instruct-q4_K_M", EnterpriseAssistant.ollamaModel(env(Map.of())));
|
||||
assertEquals("http://127.0.0.1:11434/v1", EnterpriseAssistant.ollamaBaseUrl(env(Map.of())));
|
||||
|
||||
AssistantQBitConfig config = EnterpriseAssistant.config("h2", "http://127.0.0.1:8078", env(Map.of("QQQ_ASSISTANT_OLLAMA_URL", "http://127.0.0.1:9999/v1")));
|
||||
assertTrue(config.getAllowedBaseUrls().contains("http://127.0.0.1:9999/v1"));
|
||||
assertTrue(config.getProviderTypes().containsKey("anthropic"));
|
||||
assertTrue(config.getProviderTypes().containsKey("openai-compatible"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void rowsNeverHoldAKey()
|
||||
{
|
||||
List<QRecord> rows = EnterpriseAssistant.rows(EnterpriseAssistant.Choice.ANTHROPIC, env(Map.of("ANTHROPIC_API_KEY", "sk-secret")));
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
assertFalse(row.getValues().values().stream().anyMatch(v -> String.valueOf(v).contains("sk-secret")));
|
||||
}
|
||||
assertEquals("ANTHROPIC_API_KEY", rows.get(0).getValueString("apiKeyEnvName"));
|
||||
assertNull(rows.get(1).getValue("apiKeyEnvName"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void reconcileInsertsThenSwitchesTheDefault() throws Exception
|
||||
{
|
||||
QInstance instance = new QqqAllApplication(dataDirectory, "tcp://127.0.0.1:61616").defineQInstance();
|
||||
new AssistantQBitProducer().withQBitConfig(EnterpriseAssistant.config("transientMemory", "http://127.0.0.1:1", env(Map.of()))).produce(instance);
|
||||
QContext.init(instance, new QSystemUserSession());
|
||||
|
||||
assertEquals(EnterpriseAssistant.Choice.OLLAMA, EnterpriseAssistant.reconcile(instance, env(Map.of()), model -> true));
|
||||
Map<String, QRecord> rows = providerRows();
|
||||
assertEquals(2, rows.size());
|
||||
assertTrue(rows.get(EnterpriseAssistant.OLLAMA_ROW).getValueBoolean("isDefault"));
|
||||
assertTrue(rows.get(EnterpriseAssistant.OLLAMA_ROW).getValueBoolean("enabled"));
|
||||
assertEquals("OLLAMA", rows.get(EnterpriseAssistant.OLLAMA_ROW).getValueString("kind"));
|
||||
assertEquals("llama3.3:70b-instruct-q4_K_M", rows.get(EnterpriseAssistant.OLLAMA_ROW).getValueString("model"));
|
||||
assertFalse(rows.get(EnterpriseAssistant.ANTHROPIC_ROW).getValueBoolean("enabled"));
|
||||
|
||||
assertEquals(EnterpriseAssistant.Choice.ANTHROPIC, EnterpriseAssistant.reconcile(instance, env(Map.of("ANTHROPIC_API_KEY", "sk-test")), model -> true));
|
||||
rows = providerRows();
|
||||
assertEquals(2, rows.size());
|
||||
assertTrue(rows.get(EnterpriseAssistant.ANTHROPIC_ROW).getValueBoolean("isDefault"));
|
||||
assertTrue(rows.get(EnterpriseAssistant.ANTHROPIC_ROW).getValueBoolean("enabled"));
|
||||
assertFalse(rows.get(EnterpriseAssistant.OLLAMA_ROW).getValueBoolean("isDefault"));
|
||||
assertFalse(rows.get(EnterpriseAssistant.OLLAMA_ROW).getValueBoolean("enabled"));
|
||||
|
||||
assertEquals(EnterpriseAssistant.Choice.NONE, EnterpriseAssistant.reconcile(instance, env(Map.of()), model -> false));
|
||||
assertTrue(providerRows().values().stream().noneMatch(row -> row.getValueBoolean("enabled")));
|
||||
}
|
||||
|
||||
private static Map<String, QRecord> providerRows() throws Exception
|
||||
{
|
||||
return new QueryAction().execute(new QueryInput(AssistantProvider.TABLE_NAME)).getRecords().stream()
|
||||
.collect(Collectors.toMap(row -> row.getValueString("name"), row -> row));
|
||||
}
|
||||
|
||||
private static Function<String, String> env(Map<String, String> values)
|
||||
{
|
||||
return new HashMap<>(values)::get;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.AuthScope;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.security.FieldSecurityLock;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* The enterprise profile's columnStats process (core's step, QRun-IO/qqq#1367)
|
||||
* refuses statistics for a field the caller cannot read in full and tags its
|
||||
* results with the analysed table.
|
||||
*/
|
||||
class EnterpriseColumnStatsAccessTest
|
||||
{
|
||||
private static final String KEY = "salaryViewer";
|
||||
|
||||
private QInstance instance;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws QException
|
||||
{
|
||||
MemoryRecordStore.fullReset();
|
||||
instance = new QInstance();
|
||||
instance.registerAuthenticationProvider(AuthScope.instanceDefault(), new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK));
|
||||
instance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
instance.addSecurityKeyType(new QSecurityKeyType().withName(KEY));
|
||||
instance.addTable(new QTableMetaData().withName("employee").withBackendName("memory").withPrimaryKeyField("id")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("salary", QFieldType.DECIMAL).withLabel("Salary")
|
||||
.withFieldSecurityLock(new FieldSecurityLock().withSecurityKeyType(KEY)
|
||||
.withDefaultBehavior(FieldSecurityLock.Behavior.MASK).withOverrideValues(List.of(true)))));
|
||||
EnterpriseFeatures.enableColumnStatistics(instance);
|
||||
|
||||
QContext.init(instance, new QSystemUserSession());
|
||||
new InsertAction().execute(new InsertInput("employee").withRecords(List.of(
|
||||
new QRecord().withValue("name", "Ada").withValue("salary", new BigDecimal("100")),
|
||||
new QRecord().withValue("name", "Grace").withValue("salary", new BigDecimal("200")))));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
QContext.clear();
|
||||
MemoryRecordStore.fullReset();
|
||||
}
|
||||
|
||||
@Test
|
||||
void restrictedFieldIsRefusedWithoutTheTier() throws QException
|
||||
{
|
||||
QContext.init(instance, new QSession());
|
||||
QPermissionDeniedException denied = assertThrows(QPermissionDeniedException.class, () -> run("salary"));
|
||||
assertTrue(denied.getMessage() != null && !denied.getMessage().isBlank());
|
||||
}
|
||||
|
||||
@Test
|
||||
void restrictedFieldIsAllowedWithTheTier() throws QException
|
||||
{
|
||||
QContext.init(instance, new QSession().withSecurityKeyValue(KEY, true));
|
||||
RunProcessOutput output = run("salary");
|
||||
assertTrue(output.getException().isEmpty(), () -> output.getException().get().getMessage());
|
||||
QRecord stats = (QRecord) output.getValues().get("statsRecord");
|
||||
assertEquals(2, stats.getValueInteger("count"));
|
||||
assertEquals("employee", stats.getTableName());
|
||||
}
|
||||
|
||||
@Test
|
||||
void unrestrictedFieldIsAllowed() throws QException
|
||||
{
|
||||
QContext.init(instance, new QSession());
|
||||
RunProcessOutput output = run("name");
|
||||
assertTrue(output.getException().isEmpty(), () -> output.getException().get().getMessage());
|
||||
}
|
||||
|
||||
private RunProcessOutput run(String fieldName) throws QException
|
||||
{
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName("columnStats");
|
||||
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
|
||||
input.addValue("tableName", "employee");
|
||||
input.addValue("fieldName", fieldName);
|
||||
return (new RunProcessAction().execute(input));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/** The enterprise profile offers column statistics wherever the backend can aggregate. */
|
||||
class EnterpriseColumnStatsTest
|
||||
{
|
||||
@TempDir Path dataDirectory;
|
||||
|
||||
@Test
|
||||
void queryStatsOnAggregatingBackendsOnly() throws Exception
|
||||
{
|
||||
QInstance instance = new QqqAllApplication(dataDirectory, "tcp://127.0.0.1:61616").defineQInstance();
|
||||
for(String tableName : new String[] {"customer", "order", "orderLine", "demoNote"})
|
||||
{
|
||||
assertFalse(hasQueryStats(instance, tableName), tableName + " before");
|
||||
}
|
||||
|
||||
EnterpriseFeatures.enableColumnStatistics(instance);
|
||||
assertNotNull(instance.getProcess("columnStats"));
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
// H2 (customer, order), SQLite (orderLine) and memory (demoNote) //
|
||||
///////////////////////////////////////////////////////////////////
|
||||
for(String tableName : new String[] {"customer", "order", "orderLine", "demoNote"})
|
||||
{
|
||||
assertTrue(hasQueryStats(instance, tableName), tableName);
|
||||
assertTrue(hasCapability(instance, tableName, Capability.TABLE_COUNT), tableName);
|
||||
assertTrue(hasCapability(instance, tableName, Capability.TABLE_EXPORT), tableName);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////
|
||||
// the filesystem (product) and API (apiCatalog) backends do //
|
||||
// not aggregate, so they stay without statistics //
|
||||
////////////////////////////////////////////////////////////////
|
||||
assertFalse(hasQueryStats(instance, "product"));
|
||||
assertFalse(hasQueryStats(instance, "apiCatalog"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void dataTablesAreAuditedWhenAuditsAreInstalled() throws Exception
|
||||
{
|
||||
QInstance instance = new QqqAllApplication(dataDirectory, "tcp://127.0.0.1:61616").defineQInstance();
|
||||
new AuditsMetaDataProvider().defineAll(instance, "h2", null);
|
||||
EnterpriseFeatures.auditDataTables(instance);
|
||||
for(String tableName : new String[] {"customer", "order", "orderLine", "address"})
|
||||
{
|
||||
assertEquals(AuditLevel.FIELD, instance.getTable(tableName).getAuditRules().getAuditLevel(), tableName);
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean hasQueryStats(QInstance instance, String tableName)
|
||||
{
|
||||
return hasCapability(instance, tableName, Capability.QUERY_STATS);
|
||||
}
|
||||
|
||||
private static boolean hasCapability(QInstance instance, String tableName, Capability capability)
|
||||
{
|
||||
QTableMetaData table = instance.getTable(tableName);
|
||||
QBackendMetaData backend = instance.getBackendForTable(tableName);
|
||||
return table.isCapabilityEnabled(backend, capability);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import com.kingsrook.qbits.quicksearch.SearchableFieldConfig;
|
||||
import com.kingsrook.qbits.quicksearch.SearchableTableConfig;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/** Quick Search covers the demo and sample tables, with their text fields only. */
|
||||
class EnterpriseQuickSearchTablesTest
|
||||
{
|
||||
@TempDir Path dataDirectory;
|
||||
|
||||
@Test
|
||||
void demoAndSampleTablesWithTheirTextFields() throws Exception
|
||||
{
|
||||
QInstance instance = new QqqAllApplication(dataDirectory, "tcp://127.0.0.1:61616").defineQInstance();
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// a stand-in for a sample-app table: labelled, on the sample's //
|
||||
// backend, with fields OpenSearch must not index as text //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
instance.addBackend(new QBackendMetaData().withName("rdbms").withBackendType(MemoryBackendModule.class));
|
||||
instance.addTable(new QTableMetaData().withName("countryState").withBackendName("rdbms").withPrimaryKeyField("id")
|
||||
.withRecordLabelFormat("%s").withRecordLabelFields("name")
|
||||
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("name", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("description", QFieldType.STRING))
|
||||
.withField(new QFieldMetaData("code", QFieldType.INTEGER))
|
||||
.withField(new QFieldMetaData("createDate", QFieldType.DATE_TIME)));
|
||||
|
||||
Map<String, List<String>> tables = EnterpriseFeatures.searchableTables(instance).stream()
|
||||
.collect(Collectors.toMap(SearchableTableConfig::getTableName,
|
||||
table -> table.getFields().stream().map(SearchableFieldConfig::getFieldName).toList()));
|
||||
|
||||
assertEquals(List.of("name", "email"), tables.get("customer").stream().sorted((a, b) -> a.equals("name") ? -1 : 1).toList());
|
||||
assertTrue(tables.get("shipping_stateProvince").contains("name"));
|
||||
assertEquals(List.of("name", "description"), tables.get("countryState"));
|
||||
assertFalse(tables.containsKey("webhook"), "QBit admin tables on h2 stay out");
|
||||
assertFalse(tables.containsKey("product"), "the filesystem product table is not a demo search table");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QUser;
|
||||
import com.kingsrook.qqq.backend.core.modules.authentication.QSessionStoreProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.modules.authentication.QSessionStoreRegistry;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
|
||||
/** Mock sign-ins become one stored session per uuid, touched rather than re-stored. */
|
||||
class EnterpriseSessionsTest
|
||||
{
|
||||
private final RecordingStore store = new RecordingStore();
|
||||
|
||||
@BeforeEach
|
||||
void setUp()
|
||||
{
|
||||
EnterpriseSessions.forget();
|
||||
QSessionStoreRegistry.getInstance().clear();
|
||||
QSessionStoreRegistry.getInstance().register(store);
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
QSessionStoreRegistry.getInstance().clear();
|
||||
EnterpriseSessions.forget();
|
||||
}
|
||||
|
||||
@Test
|
||||
void oneRowPerSessionUuid()
|
||||
{
|
||||
EnterpriseSessions.record(session("aaaa", "core:admin"));
|
||||
EnterpriseSessions.record(session("aaaa", "core:admin"));
|
||||
EnterpriseSessions.record(session("bbbb", "core:admin"));
|
||||
|
||||
assertEquals(List.of("aaaa", "bbbb"), new ArrayList<>(store.sessions.keySet()));
|
||||
assertEquals("core:admin", store.sessions.get("bbbb").getUser().getIdReference());
|
||||
assertNull(store.sessions.get("aaaa").getPermissions(), "only the identity is stored");
|
||||
}
|
||||
|
||||
@Test
|
||||
void aKnownSessionIsTouchedNotStoredAgain()
|
||||
{
|
||||
store.sessions.put("cccc", session("cccc", "core:admin"));
|
||||
EnterpriseSessions.record(session("cccc", "core:admin"));
|
||||
assertEquals(0, store.stores);
|
||||
assertEquals(1, store.touches);
|
||||
}
|
||||
|
||||
private static QSession session(String uuid, String userId)
|
||||
{
|
||||
QSession session = new QSession();
|
||||
session.setUuid(uuid);
|
||||
QUser user = new QUser();
|
||||
user.setIdReference(userId);
|
||||
session.setUser(user);
|
||||
session.withPermissions("customer.read", "customer.edit");
|
||||
return (session);
|
||||
}
|
||||
|
||||
/** A session store that keeps what it is given and counts calls. */
|
||||
private static class RecordingStore implements QSessionStoreProviderInterface
|
||||
{
|
||||
private final Map<String, QSession> sessions = new LinkedHashMap<>();
|
||||
private int stores;
|
||||
private int touches;
|
||||
|
||||
@Override
|
||||
public void store(String sessionUuid, QSession session, Duration ttl)
|
||||
{
|
||||
stores++;
|
||||
sessions.put(sessionUuid, session);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<QSession> load(String sessionUuid)
|
||||
{
|
||||
return (Optional.ofNullable(sessions.get(sessionUuid)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(String sessionUuid)
|
||||
{
|
||||
sessions.remove(sessionUuid);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void touch(String sessionUuid)
|
||||
{
|
||||
touches++;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Duration getDefaultTtl()
|
||||
{
|
||||
return (Duration.ofHours(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -44,7 +44,7 @@ class MinioBackendTest
|
||||
action.preAction(backend);
|
||||
S3Utils utils = utils(action);
|
||||
assertEquals("http://minio.example.test:9000/qqq-demo/example.json",
|
||||
utils.getAmazonS3().getUrl("qqq-demo", "example.json").toString());
|
||||
utils.getS3Client().utilities().getUrl(request -> request.bucket("qqq-demo").key("example.json")).toString());
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -88,8 +88,8 @@ class MinioBackendTest
|
||||
|
||||
AbstractS3Action query = (AbstractS3Action) module.getQueryInterface();
|
||||
query.preAction(backend);
|
||||
assertEquals("documents/example.json", utils(query).getAmazonS3()
|
||||
.listObjectsV2("qqq-demo").getObjectSummaries().getFirst().getKey());
|
||||
assertEquals("documents/example.json", utils(query).getS3Client()
|
||||
.listObjectsV2(request -> request.bucket("qqq-demo")).contents().getFirst().key());
|
||||
|
||||
AbstractS3Action insert = (AbstractS3Action) module.getInsertInterface();
|
||||
insert.preAction(backend);
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
|
||||
*/
|
||||
package com.kingsrook.qqq.all;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.jar.JarFile;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
/** Verifies this project's notices in the actual shaded application archive. */
|
||||
class PackagedNoticesIT
|
||||
{
|
||||
@ParameterizedTest
|
||||
@ValueSource(strings = {"LICENSE", "NOTICE"})
|
||||
void packagedProjectDocumentMatchesRoot(String document) throws Exception
|
||||
{
|
||||
Path rootDocument = Path.of(System.getProperty("qqqAllRoot"), document);
|
||||
try(JarFile jar = new JarFile(System.getProperty("qqqAllJar")))
|
||||
{
|
||||
String name = "META-INF/qqq-all/" + document;
|
||||
var entry = jar.getJarEntry(name);
|
||||
assertNotNull(entry, "Missing project document: " + name);
|
||||
assertEquals(1L, jar.stream().filter(candidate -> candidate.getName().equals(name)).count(),
|
||||
"Project document must occur exactly once: " + name);
|
||||
try(var input = jar.getInputStream(entry))
|
||||
{
|
||||
assertArrayEquals(Files.readAllBytes(rootDocument), input.readAllBytes(),
|
||||
"Packaged document must preserve root bytes: " + document);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -97,6 +97,7 @@ class StackBomTest
|
||||
List<String> missing = required.stream().filter(key -> !managedVersions.containsKey(key)).toList();
|
||||
assertEquals(List.of(), missing, "Artifacts the stack BOM does not manage");
|
||||
assertEquals("3.9.0", managedVersions.get("com.rabbitmq.jms:rabbitmq-jms"));
|
||||
assertEquals("1.0.0-RC.8", managedVersions.get("com.kingsrook.qqq:qqq-frontend-next"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
+19
-10
@@ -18,6 +18,7 @@
|
||||
|
||||
<properties>
|
||||
<qqq.version>4.1.0-SNAPSHOT</qqq.version>
|
||||
<qqq-frontend-next.version>1.0.0-RC.8</qqq-frontend-next.version>
|
||||
|
||||
<!-- qBits: develop builds on qbit-build-parent 2.0.0; none has a release on that parent yet. -->
|
||||
<qbit-quick-search.version>0.2.1-SNAPSHOT</qbit-quick-search.version>
|
||||
@@ -55,16 +56,17 @@
|
||||
<rules>
|
||||
<requireProperty>
|
||||
<property>qqq.version</property>
|
||||
<regex>^4[.]1[.]0$</regex>
|
||||
<regex>^\Q${revision}\E$</regex>
|
||||
</requireProperty>
|
||||
<requireProperty><property>qbit-quick-search.version</property><regex>^(?!.*SNAPSHOT).+$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-user-role-permissions.version</property><regex>^(?!.*SNAPSHOT).+$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-customizable-table-views.version</property><regex>^(?!.*SNAPSHOT).+$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-standard-process-trace.version</property><regex>^(?!.*SNAPSHOT).+$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-webhooks.version</property><regex>^(?!.*SNAPSHOT).+$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-workflows.version</property><regex>^(?!.*SNAPSHOT).+$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-geo-data.version</property><regex>^(?!.*SNAPSHOT).+$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-sftp-data-integration.version</property><regex>^(?!.*SNAPSHOT).+$</regex></requireProperty>
|
||||
<requireProperty><property>qqq-frontend-next.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-quick-search.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-user-role-permissions.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-customizable-table-views.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-standard-process-trace.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-webhooks.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-workflows.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-geo-data.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
<requireProperty><property>qbit-sftp-data-integration.version</property><regex>^[0-9]+[.][0-9]+[.][0-9]+(-RC[.][1-9][0-9]*)?$</regex></requireProperty>
|
||||
</rules>
|
||||
</configuration>
|
||||
</execution>
|
||||
@@ -77,7 +79,7 @@
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<!-- QQQ modules, qqq-esb, and qqq-frontend-next. -->
|
||||
<!-- QQQ modules and qqq-esb. -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-bom-pom</artifactId>
|
||||
@@ -86,6 +88,13 @@
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Keep the accepted dashboard candidate independent of framework snapshots. -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qqq</groupId>
|
||||
<artifactId>qqq-frontend-next</artifactId>
|
||||
<version>${qqq-frontend-next.version}</version>
|
||||
</dependency>
|
||||
|
||||
<!-- qBits -->
|
||||
<dependency>
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Package digest-pinned Compose with the tracked full-profile infrastructure."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
from release_gate import validate_version
|
||||
|
||||
|
||||
def create_bundle(root, version, sha, image, output):
|
||||
validate_version(version)
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", sha):
|
||||
raise ValueError("Source SHA must be a full commit ID")
|
||||
if not re.fullmatch(r"ghcr\.io/qrun-io/qqq-all@sha256:[0-9a-f]{64}", image):
|
||||
raise ValueError("Release image must be the qqq-all GHCR digest")
|
||||
environment = {key: value for key, value in os.environ.items() if not key.startswith("COMPOSE_")}
|
||||
rendered = subprocess.run(
|
||||
["docker", "compose", "--env-file", ".env.example", "--profile", "*", "-f", "compose.yaml",
|
||||
"config", "--no-interpolate", "--no-env-resolution", "--no-path-resolution", "--format", "json"],
|
||||
cwd=root, env=environment, capture_output=True, text=True, check=True,
|
||||
)
|
||||
compose = json.loads(rendered.stdout)
|
||||
compose.pop("x-app-image", None)
|
||||
for name in ("app-core", "app-full"):
|
||||
app = compose["services"][name]
|
||||
app.pop("build", None)
|
||||
app["image"] = image
|
||||
app["pull_policy"] = "always"
|
||||
compose["services"]["minio"]["image"] = "qqq-all-minio:" + version
|
||||
name = "qqq-all-" + version
|
||||
bundle = output / name
|
||||
bundle.mkdir(parents=True, exist_ok=False)
|
||||
(bundle / "compose.yaml").write_text(json.dumps(compose, indent=2) + "\n")
|
||||
tracked = subprocess.check_output(["git", "-C", str(root), "ls-files", "--", "infra"], text=True).splitlines()
|
||||
for filename in (".env.example", "LICENSE", "NOTICE", *tracked):
|
||||
source = root / filename
|
||||
if source.is_symlink() or not source.is_file():
|
||||
raise ValueError(f"Bundle input must be a regular file: {filename}")
|
||||
target = bundle / filename
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, target)
|
||||
(bundle / "release.json").write_text(json.dumps(
|
||||
{"version": version, "source_sha": sha, "image": image}, indent=2) + "\n")
|
||||
(bundle / "README.md").write_text(f"""# qqq-all {version}
|
||||
|
||||
Extract this complete bundle and run commands from this directory. The application uses
|
||||
`{image}` (source `{sha}`). No application source, Maven or app build is required.
|
||||
Docker with Compose v2 is required. Copy `.env.example` to `.env` and replace
|
||||
the demonstration passwords before sharing the environment. All fixture values remain
|
||||
demo-only; retain the infra directory next to compose.yaml.
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
docker compose -p qqq-all-demo --profile core up -d --wait --no-build
|
||||
```
|
||||
|
||||
For the full profile, stop core first. The included MinIO Dockerfile still builds locally
|
||||
and needs network access; PostgreSQL/MySQL/Mongo initialization and the Keycloak realm
|
||||
use the bundled infra files. External service images also need network access.
|
||||
|
||||
```bash
|
||||
docker compose -p qqq-all-demo --profile core down
|
||||
docker compose -p qqq-all-demo --profile full build minio
|
||||
docker compose -p qqq-all-demo --profile full up -d --wait --no-build
|
||||
```
|
||||
|
||||
Use a distinct Compose project name for each installation. The application listens on
|
||||
http://localhost:8080 by default. The full-profile identity provider uses
|
||||
http://keycloak.localhost:8081; that hostname must resolve to loopback on the client.
|
||||
Ports and fixture settings are in `.env.example`. `docker compose ... down` retains
|
||||
named data volumes; remove volumes only when intentionally resetting that installation.
|
||||
The standalone compose.yaml asset needs this bundle's infra and environment template
|
||||
for full-profile operation. See release.json for the exact application identity.
|
||||
""")
|
||||
archive = output / (name + ".tar.gz")
|
||||
with tarfile.open(archive, "x:gz") as handle:
|
||||
handle.add(bundle, arcname=name)
|
||||
return archive
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
parser.add_argument("--version", required=True)
|
||||
parser.add_argument("--sha", required=True)
|
||||
parser.add_argument("--image", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
print(create_bundle(args.root.resolve(), args.version, args.sha, args.image, args.output.resolve()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+124
-11
@@ -1,9 +1,12 @@
|
||||
"""Fail-closed checks for the manual 4.1.0 release workflow."""
|
||||
"""Fail-closed checks for the manual 4.1.0 GA / exact RC.1 release workflow."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from xml.etree import ElementTree
|
||||
|
||||
@@ -13,6 +16,16 @@ QBITS = (
|
||||
"standard-process-trace", "webhooks", "workflows", "geo-data",
|
||||
"sftp-data-integration",
|
||||
)
|
||||
RELEASE_VERSIONS = ("4.1.0", "4.1.0-RC.1")
|
||||
|
||||
|
||||
def immutable_version(value):
|
||||
return re.fullmatch(r"[0-9]+(?:\.[0-9]+){2,3}(?:-RC\.[1-9][0-9]*)?", value) is not None
|
||||
|
||||
|
||||
def validate_version(version):
|
||||
if version not in RELEASE_VERSIONS:
|
||||
raise GateError("This release workflow is limited to 4.1.0 and 4.1.0-RC.1")
|
||||
|
||||
|
||||
class GateError(ValueError):
|
||||
@@ -64,7 +77,7 @@ def managed_dependencies(path):
|
||||
reference = re.fullmatch(r"\$\{([^}]+)\}", version)
|
||||
if reference:
|
||||
version = values.get(reference.group(1), "")
|
||||
if not version or "${" in version or "SNAPSHOT" in version.upper():
|
||||
if not immutable_version(version):
|
||||
raise GateError(f"Unreleased managed dependency in {path}: "
|
||||
f"{field(dependency, 'artifactId')}")
|
||||
managed.append((field(dependency, "groupId"), field(dependency, "artifactId"),
|
||||
@@ -84,6 +97,9 @@ def validate_managed_bom(path, version):
|
||||
if entries.get(("com.kingsrook.qqq", "qqq-bom-pom")) != (version, "pom", "import"):
|
||||
raise GateError(f"QQQ BOM import differs from {version} in {path}")
|
||||
values = properties(path)
|
||||
if entries.get(("com.kingsrook.qqq", "qqq-frontend-next")) != (
|
||||
values.get("qqq-frontend-next.version"), "jar", "compile"):
|
||||
raise GateError(f"Managed Next differs from its release pin in {path}")
|
||||
for name in QBITS:
|
||||
artifact = f"qbit-{name}"
|
||||
if entries.get(("com.kingsrook.qbits", artifact)) != (
|
||||
@@ -102,14 +118,14 @@ def validate_published_poms(parent_path, bom_path, version, local_bom):
|
||||
raise GateError(f"Published {artifact} POM is not the {version} release")
|
||||
published = properties(bom_path)
|
||||
if published.get("qqq.version") != version:
|
||||
raise GateError("Published BOM does not import QQQ 4.1.0")
|
||||
raise GateError(f"Published BOM does not import QQQ {version}")
|
||||
for name in QBITS:
|
||||
key = f"qbit-{name}.version"
|
||||
value = published.get(key, "")
|
||||
if not value or "SNAPSHOT" in value.upper():
|
||||
if not immutable_version(value):
|
||||
raise GateError(f"Published BOM {key} is not a release")
|
||||
local = properties(local_bom)
|
||||
for key in ("qqq.version", *(f"qbit-{name}.version" for name in QBITS)):
|
||||
for key in ("qqq.version", "qqq-frontend-next.version", *(f"qbit-{name}.version" for name in QBITS)):
|
||||
if published.get(key) != local.get(key):
|
||||
raise GateError(f"Published BOM {key} differs from release source")
|
||||
if validate_managed_bom(bom_path, version) != validate_managed_bom(local_bom, version):
|
||||
@@ -117,20 +133,19 @@ def validate_published_poms(parent_path, bom_path, version, local_bom):
|
||||
|
||||
|
||||
def validate_poms(root, version):
|
||||
if version != "4.1.0":
|
||||
raise GateError("This release workflow is limited to 4.1.0")
|
||||
validate_version(version)
|
||||
parent = properties(root / "pom.xml")
|
||||
bom = properties(root / "qqq-all-bom/pom.xml")
|
||||
if parent.get("revision") != version or bom.get("qqq.version") != version:
|
||||
raise GateError("Parent revision and QQQ BOM pin must both equal 4.1.0")
|
||||
raise GateError(f"Parent revision and QQQ BOM pin must both equal {version}")
|
||||
for name in QBITS:
|
||||
key = f"qbit-{name}.version"
|
||||
value = bom.get(key, "")
|
||||
if not value or "SNAPSHOT" in value.upper():
|
||||
if not immutable_version(value):
|
||||
raise GateError(f"{key} must be a published non-snapshot version")
|
||||
for key, value in bom.items():
|
||||
if key.endswith(".version") and "SNAPSHOT" in value.upper():
|
||||
raise GateError(f"{key} still points to a snapshot")
|
||||
if key.endswith(".version") and not immutable_version(value):
|
||||
raise GateError(f"{key} must be an immutable release version")
|
||||
validate_managed_bom(root / "qqq-all-bom/pom.xml", version)
|
||||
|
||||
|
||||
@@ -156,6 +171,94 @@ def validate_smoke(run, jobs, sha):
|
||||
raise GateError("Smoke run must include successful core and full jobs")
|
||||
|
||||
|
||||
def validate_upstream_release(release, version):
|
||||
if (not immutable_version(version) or release.get("tag_name") != "v" + version
|
||||
or release.get("draft") is not False
|
||||
or release.get("prerelease") is not ("-RC." in version)):
|
||||
raise GateError(f"Upstream release must be public v{version} with matching prerelease state")
|
||||
|
||||
|
||||
def validate_tag(tag, version, sha, tag_sha):
|
||||
validate_version(version)
|
||||
if not all(re.fullmatch(r"[0-9a-f]{40}", value) for value in (sha, tag_sha)):
|
||||
raise GateError("Tag and commit IDs must be full Git SHAs")
|
||||
if (tag.get("sha") != tag_sha or tag.get("tag") != version
|
||||
or tag.get("object", {}).get("type") != "commit"
|
||||
or tag.get("object", {}).get("sha") != sha
|
||||
or tag.get("verification", {}).get("verified") is not True
|
||||
or tag.get("verification", {}).get("reason") != "valid"):
|
||||
raise GateError("Release tag must be signed, verified, annotated and point to the exact release commit")
|
||||
|
||||
|
||||
def download_public_inputs(bom, version, destination, central="https://repo.maven.apache.org/maven2"):
|
||||
"""Fetch the declared first-party inputs and their parents without credentials."""
|
||||
validate_version(version)
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
receipts = []
|
||||
visited = set()
|
||||
|
||||
def fetch(group, artifact, pin, kind):
|
||||
coordinate = (group, artifact, pin, kind)
|
||||
if coordinate in visited:
|
||||
return
|
||||
if (not re.fullmatch(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*", group)
|
||||
or not re.fullmatch(r"[A-Za-z0-9_-]+", artifact) or not immutable_version(pin)):
|
||||
raise GateError(f"Invalid or mutable public coordinate: {coordinate}")
|
||||
visited.add(coordinate)
|
||||
relative = f"{group.replace('.', '/')}/{artifact}/{pin}/{artifact}-{pin}.{kind}"
|
||||
url = central + "/" + relative
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as response:
|
||||
data = response.read()
|
||||
except (urllib.error.URLError, OSError) as error:
|
||||
if isinstance(error, urllib.error.HTTPError):
|
||||
error.close()
|
||||
raise GateError(f"Public input unavailable: {relative}") from error
|
||||
if not data:
|
||||
raise GateError(f"Empty public input: {relative}")
|
||||
path = destination / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
if kind == "pom":
|
||||
if coordinates(path) != (group, artifact, pin):
|
||||
raise GateError(f"Public POM coordinates differ: {relative}")
|
||||
root = ElementTree.fromstring(data)
|
||||
ns = xml_namespace(root)
|
||||
parent = root.find(f"{ns}parent")
|
||||
if parent is not None:
|
||||
parent_group = parent.findtext(f"{ns}groupId", "")
|
||||
parent_pin = parent.findtext(f"{ns}version", "")
|
||||
if not immutable_version(parent_pin):
|
||||
raise GateError(f"Mutable parent in {relative}")
|
||||
if parent_group.startswith("com.kingsrook"):
|
||||
fetch(parent_group, parent.findtext(f"{ns}artifactId", ""), parent_pin, "pom")
|
||||
if artifact == "qbit-build-parent":
|
||||
imports = root.findall(f"{ns}dependencyManagement/{ns}dependencies/{ns}dependency")
|
||||
props = root.find(f"{ns}properties")
|
||||
values = {child.tag.removeprefix(ns): child.text for child in props} if props is not None else {}
|
||||
qqq_imports = [dep for dep in imports if dep.findtext(f"{ns}groupId") == "com.kingsrook.qqq"
|
||||
and dep.findtext(f"{ns}artifactId") == "qqq-bom-pom"]
|
||||
if len(qqq_imports) != 1:
|
||||
raise GateError("Public qbit-build-parent must import the candidate QQQ BOM")
|
||||
dep = qqq_imports[0]
|
||||
actual = dep.findtext(f"{ns}version", "")
|
||||
actual = values.get(actual[2:-1], "") if actual.startswith("${") else actual
|
||||
if (actual != version or dep.findtext(f"{ns}type") != "pom"
|
||||
or dep.findtext(f"{ns}scope") != "import"):
|
||||
raise GateError("Public qbit-build-parent QQQ import differs from the candidate")
|
||||
receipts.append({"url": url, "sha256": hashlib.sha256(data).hexdigest(), "bytes": len(data)})
|
||||
|
||||
for group, artifact, pin, kind, _, classifier in validate_managed_bom(bom, version):
|
||||
if group.startswith("com.kingsrook"):
|
||||
if classifier or kind not in ("jar", "pom"):
|
||||
raise GateError(f"Unsupported first-party release input: {artifact}")
|
||||
fetch(group, artifact, pin, "pom")
|
||||
if kind == "jar":
|
||||
fetch(group, artifact, pin, "jar")
|
||||
(destination / "receipts.json").write_text(json.dumps(receipts, indent=2) + "\n")
|
||||
return receipts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--root", type=Path, required=True)
|
||||
@@ -165,13 +268,23 @@ def main():
|
||||
parser.add_argument("--smoke-jobs", type=Path, required=True)
|
||||
parser.add_argument("--published-parent", type=Path, required=True)
|
||||
parser.add_argument("--published-bom", type=Path, required=True)
|
||||
parser.add_argument("--upstream-release", type=Path, required=True)
|
||||
parser.add_argument("--next-release", type=Path, required=True)
|
||||
parser.add_argument("--tag-object", type=Path, required=True)
|
||||
parser.add_argument("--tag-sha", required=True)
|
||||
parser.add_argument("--public-inputs", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
validate_poms(args.root, args.version)
|
||||
validate_upstream_release(json.loads(args.upstream_release.read_text()), args.version)
|
||||
validate_upstream_release(json.loads(args.next_release.read_text()),
|
||||
properties(args.root / "qqq-all-bom/pom.xml")["qqq-frontend-next.version"])
|
||||
validate_tag(json.loads(args.tag_object.read_text()), args.version, args.sha, args.tag_sha)
|
||||
validate_smoke(json.loads(args.smoke_run.read_text()),
|
||||
json.loads(args.smoke_jobs.read_text()), args.sha)
|
||||
validate_published_poms(args.published_parent, args.published_bom, args.version,
|
||||
args.root / "qqq-all-bom/pom.xml")
|
||||
download_public_inputs(args.root / "qqq-all-bom/pom.xml", args.version, args.public_inputs)
|
||||
except (GateError, OSError, ElementTree.ParseError, json.JSONDecodeError) as error:
|
||||
print(f"Release gate failed: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
@@ -12,6 +12,7 @@ import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
from smoke import check_full
|
||||
from smoke_diagnostics import Diagnostics
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
@@ -95,6 +96,7 @@ def main():
|
||||
environment = os.environ.copy()
|
||||
environment.update(values)
|
||||
os.environ.update(values)
|
||||
diagnostics = Diagnostics(ROOT / "target/smoke-diagnostics", environment)
|
||||
with tempfile.TemporaryDirectory(prefix="qqq-all-full-smoke-") as directory:
|
||||
env_file = Path(directory) / "compose.env"
|
||||
env_file.write_text("".join(f"{key}={value}\n" for key, value in values.items()))
|
||||
@@ -103,10 +105,12 @@ def main():
|
||||
"-f", str(ROOT / "compose.yaml"), "-f", str(ROOT / "scripts/compose.smoke.yaml"),
|
||||
"--profile", "full"]
|
||||
try:
|
||||
started = subprocess.run(command + ["up", "-d", "--build", "--wait", "--wait-timeout", "360"],
|
||||
cwd=ROOT, env=environment, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=480, check=False)
|
||||
if started.returncode:
|
||||
diagnostics.jar_hashes(ROOT)
|
||||
diagnostics.pull(command, ROOT, environment)
|
||||
started = diagnostics.run(command + ["up", "-d", "--build", "--wait", "--wait-timeout", "360"],
|
||||
ROOT, environment, timeout=480)
|
||||
diagnostics.write("startup.json", started)
|
||||
if started["exit_code"] != 0:
|
||||
raise AssertionError("full Compose stack did not become healthy")
|
||||
prepare_sftp_import_directory(command, environment, values.get("SFTP_USER", "qqq"))
|
||||
check_full(
|
||||
@@ -115,10 +119,22 @@ def main():
|
||||
lambda index, query: query_opensearch(command, environment, index, query),
|
||||
120,
|
||||
)
|
||||
except (AssertionError, OSError, ValueError, subprocess.TimeoutExpired) as error:
|
||||
diagnostics.write("failure.json", {"type": type(error).__name__,
|
||||
"message": diagnostics.sanitize(str(error))})
|
||||
raise AssertionError(diagnostics.sanitize(str(error))) from None
|
||||
finally:
|
||||
subprocess.run(command + ["down", "--volumes", "--remove-orphans"],
|
||||
cwd=ROOT, env=environment, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=120, check=False)
|
||||
try:
|
||||
diagnostics.collect(command, ROOT, environment)
|
||||
except Exception as error:
|
||||
# Evidence failure must not suppress the smoke error or private-stack cleanup.
|
||||
print(f"smoke diagnostics collection failed: {type(error).__name__}", file=sys.stderr)
|
||||
finally:
|
||||
cleanup = diagnostics.run(command + ["down", "--volumes", "--remove-orphans"],
|
||||
ROOT, environment, timeout=120)
|
||||
diagnostics.write("cleanup.json", cleanup)
|
||||
if cleanup["exit_code"] != 0 and sys.exc_info()[0] is None:
|
||||
raise AssertionError("private Compose cleanup failed; see sanitized diagnostics")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+11
-2
@@ -119,6 +119,16 @@ class SmokeClient:
|
||||
raise AssertionError(f"missing expected {table} {field}")
|
||||
|
||||
|
||||
def require_oidc_session(client, result):
|
||||
# Next uses an HttpOnly cookie; current QQQ intentionally omits its UUID from JSON.
|
||||
cookies = [cookie for cookie in client.cookies if cookie.name == "sessionUUID"]
|
||||
if (len(cookies) != 1 or not cookies[0].value
|
||||
or not cookies[0].has_nonstandard_attr("HttpOnly")):
|
||||
raise AssertionError("QQQ did not establish an HttpOnly OIDC session")
|
||||
if "uuid" in result:
|
||||
raise AssertionError("QQQ exposed the HttpOnly session UUID in JSON")
|
||||
|
||||
|
||||
def allow_loopback_http_session(client):
|
||||
parsed = urllib.parse.urlsplit(client.base_url)
|
||||
if parsed.scheme != "http" or parsed.hostname not in ("127.0.0.1", "localhost"):
|
||||
@@ -273,8 +283,7 @@ def oidc_login(client, keycloak_url, username, password):
|
||||
result = client.json("/manageSession", {
|
||||
"code": parameters["code"][0], "redirectUri": callback, "codeVerifier": verifier
|
||||
})
|
||||
if not result.get("uuid") or not any(cookie.name == "sessionUUID" for cookie in client.cookies):
|
||||
raise AssertionError("QQQ did not establish an OIDC session")
|
||||
require_oidc_session(client, result)
|
||||
if urllib.parse.urlsplit(client.base_url).scheme == "http":
|
||||
allow_loopback_http_session(client)
|
||||
except (OSError, KeyError, ValueError, urllib.error.HTTPError) as error:
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Small, sanitized receipts for the private full-smoke stack only."""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
LIMIT = 65536
|
||||
CONTAINER_FORMAT = ('{"id":{{json .Id}},"service":{{json (index .Config.Labels '
|
||||
'"com.docker.compose.service")}},"image_id":{{json .Image}},'
|
||||
'"status":{{json .State.Status}},"exit_code":{{json .State.ExitCode}},'
|
||||
'"oom_killed":{{json .State.OOMKilled}},"error":{{json .State.Error}},'
|
||||
'"health":{{if index .State "Health"}}{{json (index .State "Health")}}{{else}}null{{end}}}')
|
||||
IMAGE_FORMAT = ('{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},'
|
||||
'"architecture":{{json .Architecture}},"os":{{json .Os}}}')
|
||||
|
||||
|
||||
class Diagnostics:
|
||||
def __init__(self, directory, environment):
|
||||
self.directory = directory
|
||||
self.secrets = set()
|
||||
for key, value in environment.items():
|
||||
if value and re.search(r'PASSWORD|PASSWD|SECRET|TOKEN|KEYFILE|ACCESS_KEY|PRIVATE_KEY|CREDENTIAL|AUTHORIZATION', key, re.I):
|
||||
self.secrets.update((value, urllib.parse.quote(value, safe=''),
|
||||
urllib.parse.quote_plus(value),
|
||||
base64.b64encode(value.encode()).decode()))
|
||||
|
||||
def sanitize(self, text):
|
||||
if isinstance(text, bytes):
|
||||
text = text.decode('utf-8', errors='replace')
|
||||
text = text or ''
|
||||
text = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', text)
|
||||
for secret in sorted(self.secrets, key=len, reverse=True):
|
||||
text = text.replace(secret, '[REDACTED]')
|
||||
text = re.sub(r'(?i)(\b[\w.-]{0,80}(?:password|passwd|secret|token|authorization|credential|api[_-]?key|access[_-]?key)\b["\s]*[:=]\s*)("[^"]*"|\x27[^\x27]*\x27|[^\s,;]+)', r'\1[REDACTED]', text)
|
||||
text = re.sub(r'(?i)\b(Bearer|Basic)\s+[^\s,;]+', r'\1 [REDACTED]', text)
|
||||
text = re.sub(r'(?i)\b([a-z][a-z0-9+.-]*://)[^/\s@]+@', r'\1[REDACTED]@', text)
|
||||
if len(text) > LIMIT:
|
||||
text = '[...truncated...]\n' + text[-(LIMIT - 18):]
|
||||
return text
|
||||
|
||||
def write(self, name, data):
|
||||
# Redact strings before JSON encoding, so escaped secrets cannot escape filtering.
|
||||
def clean(value):
|
||||
if isinstance(value, str):
|
||||
return self.sanitize(value)
|
||||
if isinstance(value, dict):
|
||||
return {key: clean(item) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [clean(item) for item in value]
|
||||
return value
|
||||
try:
|
||||
self.directory.mkdir(parents=True, exist_ok=True)
|
||||
path = self.directory / name
|
||||
path.write_text(json.dumps(clean(data), indent=2) + '\n')
|
||||
path.chmod(0o600)
|
||||
except OSError as error:
|
||||
print(f"smoke diagnostics write failed: {type(error).__name__}", file=sys.stderr)
|
||||
|
||||
def run(self, args, root, environment, timeout=15):
|
||||
try:
|
||||
result = subprocess.run(args, cwd=root, env=environment, capture_output=True,
|
||||
text=True, timeout=timeout, check=False)
|
||||
return {'exit_code': result.returncode,
|
||||
'output': self.sanitize((result.stdout or '') + (result.stderr or ''))}
|
||||
except subprocess.TimeoutExpired as error:
|
||||
return {'exit_code': None, 'error': 'TimeoutExpired',
|
||||
'output': self.sanitize(self.sanitize(error.stdout) + self.sanitize(error.stderr))}
|
||||
except OSError as error:
|
||||
return {'exit_code': None, 'error': type(error).__name__, 'output': ''}
|
||||
|
||||
def jar_hashes(self, root):
|
||||
data = {'dependencies': []}
|
||||
manifest = root / 'qqq-all-app/target/smoke-classpath.txt'
|
||||
if manifest.is_file():
|
||||
for name in sorted(set(manifest.read_text().strip().split(os.pathsep))):
|
||||
path = Path(name)
|
||||
match = re.search(r'/com/kingsrook/(qqq|qbits)/([^/]+)/([^/]+)/[^/]+\.jar$', path.as_posix())
|
||||
if not match:
|
||||
continue
|
||||
group, artifact, version = match.groups()
|
||||
entry = {'coordinate': f'com.kingsrook.{group}:{artifact}:{version}', 'file': path.name}
|
||||
if path.is_file():
|
||||
with path.open('rb') as stream:
|
||||
entry['sha256'] = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
else:
|
||||
entry['error'] = 'resolved JAR missing'
|
||||
data['dependencies'].append(entry)
|
||||
else:
|
||||
data['error'] = 'resolved classpath missing; run the full-smoke Maven command'
|
||||
app = root / 'qqq-all-app/target/qqq-all-app.jar'
|
||||
if app.is_file():
|
||||
with app.open('rb') as stream:
|
||||
data['application_sha256'] = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
self.write('jars.json', data)
|
||||
|
||||
def pull(self, command, root, environment):
|
||||
receipt = {'pulls': []}
|
||||
try:
|
||||
# Keep the uninterpolated model in memory only; retain no service environment.
|
||||
model = subprocess.run(command + ['config', '--format', 'json', '--no-interpolate',
|
||||
'--no-env-resolution'], cwd=root, env=environment,
|
||||
capture_output=True, text=True, timeout=15, check=False)
|
||||
if model.returncode:
|
||||
raise ValueError('Compose image selection failed')
|
||||
services = json.loads(model.stdout)['services']
|
||||
if not isinstance(services, dict) or not 1 <= len(services) <= 32:
|
||||
raise ValueError('unexpected service inventory')
|
||||
selected = [(name, service['image']) for name, service in sorted(services.items())
|
||||
if 'build' not in service]
|
||||
if any(not re.fullmatch(r'[a-zA-Z0-9][a-zA-Z0-9_.-]*', name)
|
||||
or not isinstance(image, str) or '${' in image for name, image in selected):
|
||||
raise ValueError('unresolved service image')
|
||||
except (KeyError, TypeError, ValueError, OSError, subprocess.TimeoutExpired) as error:
|
||||
receipt['selection_error'] = type(error).__name__
|
||||
self.write('pulls.json', receipt)
|
||||
raise AssertionError('could not select Compose service images; see pulls.json') from None
|
||||
deadline = time.monotonic() + 480
|
||||
for service, image in selected:
|
||||
first = image.split('/')[0]
|
||||
registry = first if '/' in image and ('.' in first or ':' in first or first == 'localhost') else 'docker.io'
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
result = {'exit_code': None, 'error': 'pull-stage budget exhausted', 'output': ''}
|
||||
else:
|
||||
result = self.run(command + ['--parallel', '1', '--progress', 'plain', 'pull',
|
||||
'--ignore-buildable', '--policy', 'missing', service],
|
||||
root, environment, timeout=min(120, remaining))
|
||||
receipt['pulls'].append({'service': service, 'image': image,
|
||||
'registry': registry, 'result': result})
|
||||
self.write('pulls.json', receipt)
|
||||
if result['exit_code'] != 0:
|
||||
raise AssertionError(f'Compose pull failed for {service} ({image}); see pulls.json')
|
||||
|
||||
def collect(self, command, root, environment):
|
||||
deadline = time.monotonic() + 60
|
||||
def run(args):
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return {'exit_code': None, 'error': 'collection budget exhausted', 'output': ''}
|
||||
return self.run(args, root, environment, timeout=min(15, remaining))
|
||||
inventory = run(command + ['ps', '-a', '-q'])
|
||||
ids = [line for line in inventory['output'].splitlines()
|
||||
if re.fullmatch(r'[a-f0-9]{12,64}', line)][:32]
|
||||
services, images = [], {}
|
||||
for container in ids:
|
||||
result = run(['docker', 'inspect', '--format', CONTAINER_FORMAT, container])
|
||||
try:
|
||||
raw = json.loads(result['output'])
|
||||
state = {key: raw.get(key) for key in ('id', 'service', 'image_id', 'status',
|
||||
'exit_code', 'oom_killed', 'error')}
|
||||
health = raw.get('health') or {}
|
||||
state['health'] = {'status': health.get('Status'),
|
||||
'checks': [{key: check.get(key) for key in ('Start', 'End', 'ExitCode', 'Output')}
|
||||
for check in health.get('Log', [])[-5:]]}
|
||||
services.append(state)
|
||||
image = state['image_id']
|
||||
if isinstance(image, str) and re.fullmatch(r'sha256:[a-f0-9]{64}', image) and image not in images:
|
||||
found = run(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image])
|
||||
try:
|
||||
details = json.loads(found['output'])
|
||||
images[image] = {key: details.get(key) for key in ('id', 'repo_digests', 'architecture', 'os')}
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
images[image] = {'error': 'image identity unavailable', 'command': found}
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
services.append({'id': container, 'error': 'container state unavailable', 'command': result})
|
||||
self.write(f'container-{container[:12]}.json',
|
||||
run(['docker', 'logs', '--timestamps', '--tail', '100', container]))
|
||||
self.write('services.json', {'inventory': inventory, 'services': services, 'images': images})
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Exercise the delivered archive with Compose, without running any containers."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
IMAGE = "ghcr.io/qrun-io/qqq-all@sha256:" + "a" * 64
|
||||
|
||||
|
||||
class ReleaseBundleTest(unittest.TestCase):
|
||||
def test_bundle_uses_digest_and_keeps_full_profile_infra_without_app_build(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
result = subprocess.run(["python3", str(ROOT / "scripts/release_bundle.py"),
|
||||
"--root", str(ROOT), "--version", "4.1.0-RC.1",
|
||||
"--sha", "b" * 40, "--image", IMAGE, "--output", directory],
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
archive = Path(directory) / "qqq-all-4.1.0-RC.1.tar.gz"
|
||||
extracted = Path(directory) / "extracted"
|
||||
with tarfile.open(archive) as bundle:
|
||||
names = bundle.getnames()
|
||||
self.assertIn("qqq-all-4.1.0-RC.1/infra/minio/Dockerfile", names)
|
||||
self.assertIn("qqq-all-4.1.0-RC.1/infra/keycloak/realm.json", names)
|
||||
self.assertIn("qqq-all-4.1.0-RC.1/.env.example", names)
|
||||
self.assertNotIn("qqq-all-4.1.0-RC.1/.env", names)
|
||||
self.assertNotIn("qqq-all-4.1.0-RC.1/Dockerfile", names)
|
||||
bundle.extractall(extracted, filter="data")
|
||||
root = extracted / "qqq-all-4.1.0-RC.1"
|
||||
bundled = json.loads((root / "compose.yaml").read_text())
|
||||
self.assertEqual("/home/${SFTP_USER:-qqq}/upload",
|
||||
bundled["services"]["sftp"]["volumes"][0]["target"])
|
||||
environment = {key: value for key, value in os.environ.items() if not key.startswith("COMPOSE_")}
|
||||
result = subprocess.run(["docker", "compose", "--env-file", ".env.example", "--profile", "*",
|
||||
"-f", "compose.yaml", "config", "--format", "json"],
|
||||
cwd=root, env=environment, capture_output=True, text=True)
|
||||
self.assertEqual(0, result.returncode, result.stderr)
|
||||
config = json.loads(result.stdout)
|
||||
original_result = subprocess.run(["docker", "compose", "--env-file", ".env.example", "--profile", "*",
|
||||
"-f", "compose.yaml", "config", "--format", "json"],
|
||||
cwd=ROOT, env=environment, capture_output=True, text=True, check=True)
|
||||
original = json.loads(original_result.stdout.replace(str(ROOT.resolve()), str(root.resolve())))
|
||||
for name, service in original["services"].items():
|
||||
if name in ("app-core", "app-full"):
|
||||
service.pop("build")
|
||||
service.update(image=IMAGE, pull_policy="always")
|
||||
if name == "minio":
|
||||
service["image"] = "qqq-all-minio:4.1.0-RC.1"
|
||||
self.assertEqual(service, config["services"][name], name)
|
||||
for service in ("app-core", "app-full"):
|
||||
self.assertEqual(IMAGE, config["services"][service]["image"])
|
||||
self.assertNotIn("build", config["services"][service])
|
||||
self.assertEqual((root / "infra/minio").resolve(), Path(config["services"]["minio"]["build"]["context"]).resolve())
|
||||
for service in config["services"].values():
|
||||
for volume in service.get("volumes", []):
|
||||
if volume["type"] == "bind":
|
||||
self.assertTrue(Path(volume["source"]).exists(), volume)
|
||||
manifest = json.loads((root / "release.json").read_text())
|
||||
self.assertEqual("4.1.0-RC.1", manifest["version"])
|
||||
self.assertEqual("b" * 40, manifest["source_sha"])
|
||||
self.assertEqual(IMAGE, manifest["image"])
|
||||
|
||||
def test_bundle_rejects_mutable_image_and_unapproved_version(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
for version, image in (("4.1.0-RC.2", IMAGE), ("4.1.0-RC.1", "ghcr.io/qrun-io/qqq-all:develop"),
|
||||
("4.1.0", "ghcr.io/other/qqq-all@sha256:" + "a" * 64)):
|
||||
result = subprocess.run(["python3", str(ROOT / "scripts/release_bundle.py"),
|
||||
"--root", str(ROOT), "--version", version, "--sha", "b" * 40,
|
||||
"--image", image, "--output", directory], capture_output=True, text=True)
|
||||
self.assertNotEqual(0, result.returncode)
|
||||
self.assertEqual([], list(Path(directory).iterdir()))
|
||||
@@ -26,6 +26,10 @@ def management(qqq_version="${qqq.version}", qbit_version=None):
|
||||
f"<artifactId>qbit-{name}</artifactId><version>{qbit_version(name)}</version></dependency>"
|
||||
for name in QBITS
|
||||
)
|
||||
dependencies += (
|
||||
"<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-frontend-next</artifactId>"
|
||||
"<version>1.0.0-RC.8</version></dependency>"
|
||||
)
|
||||
return f"<dependencyManagement><dependencies>{dependencies}</dependencies></dependencyManagement>"
|
||||
|
||||
|
||||
@@ -44,6 +48,7 @@ class ReleaseGateTest(unittest.TestCase):
|
||||
(self.root / "qqq-all-bom/pom.xml").write_text(
|
||||
"<project><properties>"
|
||||
f"<qqq.version>{qqq_version}</qqq.version>"
|
||||
"<qqq-frontend-next.version>1.0.0-RC.8</qqq-frontend-next.version>"
|
||||
+ "".join(f"<qbit-{name}.version>{qbit_version}</qbit-{name}.version>" for name in QBITS)
|
||||
+ "</properties>" + management() + "</project>"
|
||||
)
|
||||
@@ -59,6 +64,7 @@ class ReleaseGateTest(unittest.TestCase):
|
||||
"<groupId>com.kingsrook.qqq</groupId>"
|
||||
"<artifactId>qqq-all-parent</artifactId><version>4.1.0</version></parent>"
|
||||
"<artifactId>qqq-all-bom</artifactId><properties><qqq.version>4.1.0</qqq.version>"
|
||||
"<qqq-frontend-next.version>1.0.0-RC.8</qqq-frontend-next.version>"
|
||||
+ "".join(f"<qbit-{name}.version>1.2.3</qbit-{name}.version>" for name in QBITS)
|
||||
+ "</properties>" + managed + "</project>")
|
||||
return parent, bom
|
||||
@@ -97,6 +103,8 @@ class ReleaseGateTest(unittest.TestCase):
|
||||
({**run, "head_sha": "b" * 40}, jobs),
|
||||
({**run, "conclusion": "failure"}, jobs),
|
||||
({**run, "path": ".github/workflows/other.yml"}, jobs),
|
||||
({**run, "event": "workflow_dispatch"}, jobs),
|
||||
({**run, "head_branch": "main"}, jobs),
|
||||
(run, {"jobs": [jobs["jobs"][0]]}),
|
||||
(run, {"jobs": [jobs["jobs"][0], {"name": "full", "conclusion": "skipped"}]}),
|
||||
):
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
"""RC publication must retain the GA boundary and fail before any upload."""
|
||||
|
||||
import functools
|
||||
import http.server
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
import release_gate
|
||||
from release_gate import GateError
|
||||
import test_release_gate
|
||||
|
||||
|
||||
RC = "4.1.0-RC.1"
|
||||
TAG_SHA = "b" * 40
|
||||
RELEASE_SHA = "a" * 40
|
||||
|
||||
|
||||
class QuietHandler(http.server.SimpleHTTPRequestHandler):
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
|
||||
class ReleaseCandidateTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.fixture = test_release_gate.ReleaseGateTest()
|
||||
self.fixture.setUp()
|
||||
self.addCleanup(self.fixture.doCleanups)
|
||||
self.root = self.fixture.root
|
||||
|
||||
def test_exact_rc_and_ga_are_allowed_but_other_candidates_are_not(self):
|
||||
for version in ("4.1.0", RC):
|
||||
self.fixture.write_poms(version, version, "1.2.3-RC.1")
|
||||
release_gate.validate_poms(self.root, version)
|
||||
for version in ("4.1.0-RC.2", "4.1.0-SNAPSHOT", "4.1.1", "LATEST", ""):
|
||||
with self.subTest(version=version), self.assertRaises(GateError):
|
||||
self.fixture.write_poms(version, version, "1.2.3")
|
||||
release_gate.validate_poms(self.root, version)
|
||||
|
||||
def test_rc_rejects_wrong_framework_pin_and_mutable_dependencies(self):
|
||||
for pin in ("4.1.0", "4.1.0-RC.2", "4.1.0-SNAPSHOT"):
|
||||
self.fixture.write_poms(RC, pin, "1.2.3")
|
||||
with self.assertRaises(GateError):
|
||||
release_gate.validate_poms(self.root, RC)
|
||||
for pin in ("LATEST", "RELEASE", "[1.0,2.0)", "1.2.3-SNAPSHOT", "${unknown}"):
|
||||
self.fixture.write_poms(RC, RC, pin)
|
||||
with self.subTest(pin=pin), self.assertRaises(GateError):
|
||||
release_gate.validate_poms(self.root, RC)
|
||||
self.fixture.write_poms(RC, RC, "1.2.3")
|
||||
bom = self.root / "qqq-all-bom/pom.xml"
|
||||
bom.write_text(bom.read_text().replace("1.0.0-RC.8", "LATEST"))
|
||||
with self.assertRaises(GateError):
|
||||
release_gate.validate_poms(self.root, RC)
|
||||
|
||||
def test_public_rc_distribution_poms_must_match_source_including_next(self):
|
||||
self.fixture.write_poms(RC, RC, "1.2.3")
|
||||
parent, bom = self.fixture.published_poms(resolved=True)
|
||||
for path in (parent, bom):
|
||||
path.write_text(path.read_text().replace("4.1.0", RC))
|
||||
release_gate.validate_published_poms(parent, bom, RC, self.root / "qqq-all-bom/pom.xml")
|
||||
original = bom.read_text()
|
||||
for changed in (original.replace("1.0.0-RC.8", "1.0.0-RC.9"),
|
||||
original.replace(RC, "4.1.0-RC.2"),
|
||||
original.replace("1.2.3", "[1,2)")):
|
||||
bom.write_text(changed)
|
||||
with self.assertRaises(GateError):
|
||||
release_gate.validate_published_poms(parent, bom, RC, self.root / "qqq-all-bom/pom.xml")
|
||||
|
||||
def test_upstream_rc_requires_exact_public_prerelease_and_ga_requires_final(self):
|
||||
for version, prerelease in ((RC, True), ("4.1.0", False)):
|
||||
release = {"tag_name": "v" + version, "draft": False, "prerelease": prerelease}
|
||||
release_gate.validate_upstream_release(release, version)
|
||||
for changed in ({**release, "draft": True}, {**release, "prerelease": not prerelease},
|
||||
{**release, "tag_name": "v4.1.0-RC.2"}, {}):
|
||||
with self.subTest(changed=changed), self.assertRaises(GateError):
|
||||
release_gate.validate_upstream_release(changed, version)
|
||||
|
||||
def test_tag_requires_verified_signature_exact_name_object_and_commit(self):
|
||||
tag = {"sha": TAG_SHA, "tag": RC, "object": {"type": "commit", "sha": RELEASE_SHA},
|
||||
"verification": {"verified": True, "reason": "valid"}}
|
||||
release_gate.validate_tag(tag, RC, RELEASE_SHA, TAG_SHA)
|
||||
mutations = [
|
||||
{**tag, "tag": "4.1.0"}, {**tag, "sha": "c" * 40},
|
||||
{**tag, "object": {"type": "commit", "sha": "c" * 40}},
|
||||
{**tag, "object": {"type": "tag", "sha": RELEASE_SHA}},
|
||||
{**tag, "verification": {"verified": False, "reason": "unsigned"}},
|
||||
{**tag, "verification": {"verified": True, "reason": "unknown_key"}}, {},
|
||||
]
|
||||
for changed in mutations:
|
||||
with self.subTest(changed=changed), self.assertRaises(GateError):
|
||||
release_gate.validate_tag(changed, RC, RELEASE_SHA, TAG_SHA)
|
||||
|
||||
def test_public_download_fails_on_missing_or_wrong_coordinates_and_parent(self):
|
||||
self.fixture.write_poms(RC, RC, "1.2.3")
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
central = Path(directory)
|
||||
dependencies = [("com.kingsrook.qqq", "qqq-bom-pom", RC, "pom"),
|
||||
("com.kingsrook.qqq", "qqq-frontend-next", "1.0.0-RC.8", "jar")]
|
||||
dependencies += [("com.kingsrook.qbits", "qbit-" + name, "1.2.3", "jar")
|
||||
for name in release_gate.QBITS]
|
||||
for group, artifact, version, kind in dependencies:
|
||||
folder = central / group.replace(".", "/") / artifact / version
|
||||
folder.mkdir(parents=True)
|
||||
stem = folder / (artifact + "-" + version)
|
||||
Path(str(stem) + ".pom").write_text(
|
||||
f"<project><groupId>{group}</groupId><artifactId>{artifact}</artifactId>"
|
||||
f"<version>{version}</version></project>")
|
||||
if kind == "jar":
|
||||
Path(str(stem) + ".jar").write_bytes(b"published fixture jar")
|
||||
handler = functools.partial(QuietHandler, directory=directory)
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
self.addCleanup(server.server_close)
|
||||
self.addCleanup(server.shutdown)
|
||||
base_url = f"http://127.0.0.1:{server.server_port}"
|
||||
dest = self.root / "public"
|
||||
bom = self.root / "qqq-all-bom/pom.xml"
|
||||
receipts = release_gate.download_public_inputs(bom, RC, dest, base_url)
|
||||
self.assertEqual(19, len(receipts))
|
||||
self.assertTrue(all(len(row["sha256"]) == 64 for row in receipts))
|
||||
public_pom = central / "com/kingsrook/qqq/qqq-frontend-next/1.0.0-RC.8/qqq-frontend-next-1.0.0-RC.8.pom"
|
||||
original = public_pom.read_text()
|
||||
public_pom.write_text(original.replace("1.0.0-RC.8", "1.0.0-RC.9"))
|
||||
with self.assertRaises(GateError):
|
||||
release_gate.download_public_inputs(bom, RC, dest, base_url)
|
||||
public_pom.write_text(original.replace("</project>", "<parent><groupId>com.kingsrook</groupId>"
|
||||
"<artifactId>qbit-build-parent</artifactId><version>LATEST</version></parent></project>"))
|
||||
with self.assertRaises(GateError):
|
||||
release_gate.download_public_inputs(bom, RC, dest, base_url)
|
||||
public_pom.write_text(original.replace("</project>", "<parent><groupId>com.kingsrook</groupId>"
|
||||
"<artifactId>qbit-build-parent</artifactId><version>2.1.0-RC.1</version></parent></project>"))
|
||||
parent = central / "com/kingsrook/qbit-build-parent/2.1.0-RC.1/qbit-build-parent-2.1.0-RC.1.pom"
|
||||
parent.parent.mkdir(parents=True)
|
||||
parent.write_text("<project><groupId>com.kingsrook</groupId><artifactId>qbit-build-parent</artifactId>"
|
||||
"<version>2.1.0-RC.1</version><dependencyManagement><dependencies><dependency>"
|
||||
"<groupId>com.kingsrook.qqq</groupId><artifactId>qqq-bom-pom</artifactId>"
|
||||
"<version>4.0.0</version><type>pom</type><scope>import</scope>"
|
||||
"</dependency></dependencies></dependencyManagement></project>")
|
||||
with self.assertRaises(GateError):
|
||||
release_gate.download_public_inputs(bom, RC, dest, base_url)
|
||||
parent.write_text(parent.read_text().replace("4.0.0", RC))
|
||||
self.assertEqual(20, len(release_gate.download_public_inputs(bom, RC, dest, base_url)))
|
||||
public_pom.unlink()
|
||||
with self.assertRaises(GateError):
|
||||
release_gate.download_public_inputs(bom, RC, dest, base_url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+18
-5
@@ -148,10 +148,12 @@ class SmokeTest(unittest.TestCase):
|
||||
pom = ET.parse(ROOT / "qqq-all-app" / "pom.xml")
|
||||
final_name = pom.findtext(".//{http://maven.apache.org/POM/4.0.0}build/"
|
||||
"{http://maven.apache.org/POM/4.0.0}finalName")
|
||||
workflow = (ROOT / ".github" / "workflows" / "smoke.yml").read_text()
|
||||
command = re.search(r"run: python3 scripts/run_core_smoke.py (\S+)", workflow)
|
||||
self.assertIsNotNone(command)
|
||||
self.assertEqual(f"qqq-all-app/target/{final_name}.jar", command.group(1))
|
||||
# CI smokes the exact jar Maven builds; keep the build output and the
|
||||
# CI smoke command on the same artifact path.
|
||||
self.assertEqual("qqq-all-app", final_name)
|
||||
workflow = (ROOT / ".gitea" / "workflows" / "build.yaml").read_text()
|
||||
self.assertIn("python3 scripts/run_core_smoke.py qqq-all-app/target/qqq-all-app.jar", workflow)
|
||||
|
||||
|
||||
def test_standalone_full_smoke_requires_explicit_search_endpoint(self):
|
||||
result = subprocess.run(
|
||||
@@ -192,6 +194,7 @@ class SmokeTest(unittest.TestCase):
|
||||
with (mock.patch.object(run_full_smoke, "ROOT", root),
|
||||
mock.patch.object(run_full_smoke, "demo_environment", return_value=values),
|
||||
mock.patch.object(run_full_smoke, "published_port", return_value=20003),
|
||||
mock.patch.object(run_full_smoke.Diagnostics, "pull"),
|
||||
mock.patch.object(run_full_smoke.subprocess, "run",
|
||||
return_value=subprocess.CompletedProcess([], 0)),
|
||||
mock.patch.object(run_full_smoke, "check_full", side_effect=verify_credentials) as check,
|
||||
@@ -233,7 +236,7 @@ class SmokeTest(unittest.TestCase):
|
||||
run_full_smoke.prepare_sftp_import_directory(["docker", "compose"], {}, "qqq")
|
||||
|
||||
def test_oidc_cookie_is_sent_only_to_loopback_http_app(self):
|
||||
from smoke import allow_loopback_http_session
|
||||
from smoke import allow_loopback_http_session, require_oidc_session
|
||||
|
||||
class SecureSession(DemoHandler):
|
||||
def do_GET(self):
|
||||
@@ -257,7 +260,17 @@ class SmokeTest(unittest.TestCase):
|
||||
thread.start()
|
||||
try:
|
||||
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
|
||||
with self.assertRaisesRegex(AssertionError, "HttpOnly OIDC session"):
|
||||
require_oidc_session(client, {})
|
||||
client.request("/login")
|
||||
require_oidc_session(client, {})
|
||||
with self.assertRaisesRegex(AssertionError, "exposed"):
|
||||
require_oidc_session(client, {"uuid": "test-session"})
|
||||
cookie = next(iter(client.cookies))
|
||||
cookie._rest.clear()
|
||||
with self.assertRaisesRegex(AssertionError, "HttpOnly OIDC session"):
|
||||
require_oidc_session(client, {})
|
||||
cookie.set_nonstandard_attr("HttpOnly", None)
|
||||
with self.assertRaisesRegex(AssertionError, "HTTP 403"):
|
||||
client.request("/protected")
|
||||
allow_loopback_http_session(client)
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Failure evidence must survive cleanup without leaking runtime credentials."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import run_full_smoke
|
||||
|
||||
|
||||
class DiagnosticsTest(unittest.TestCase):
|
||||
def test_startup_failure_retains_redacted_evidence_before_cleanup(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'compose.yaml').touch()
|
||||
values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002',
|
||||
'POSTGRES_PASSWORD': 'private-password-123'}
|
||||
cleaned = []
|
||||
|
||||
def run(args, **kwargs):
|
||||
if 'config' in args:
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '')
|
||||
if 'up' in args:
|
||||
return subprocess.CompletedProcess(args, 1, 'postgres unhealthy private-password-123', '')
|
||||
if 'down' in args:
|
||||
receipt = root / 'target/smoke-diagnostics/startup.json'
|
||||
self.assertTrue(receipt.is_file(), 'diagnostics must exist before teardown')
|
||||
cleaned.append(True)
|
||||
return subprocess.CompletedProcess(args, 0, '', '')
|
||||
|
||||
with (mock.patch.object(run_full_smoke, 'ROOT', root),
|
||||
mock.patch.object(run_full_smoke, 'demo_environment', return_value=values),
|
||||
mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run),
|
||||
mock.patch.dict(os.environ)):
|
||||
with self.assertRaisesRegex(AssertionError, 'did not become healthy'):
|
||||
run_full_smoke.main()
|
||||
self.assertTrue(cleaned)
|
||||
output = (root / 'target/smoke-diagnostics/startup.json').read_text()
|
||||
self.assertIn('postgres unhealthy', output)
|
||||
self.assertNotIn('private-password-123', output)
|
||||
self.assertEqual(1, json.loads(output)['exit_code'])
|
||||
|
||||
def test_http_failure_stays_sanitized_when_collection_and_cleanup_fail(self):
|
||||
self.diagnostics_module()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'compose.yaml').touch()
|
||||
values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002',
|
||||
'DB_PASSWORD': 'hidden-fixture'}
|
||||
def run(args, **kwargs):
|
||||
if 'config' in args:
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '')
|
||||
if 'down' in args:
|
||||
raise subprocess.TimeoutExpired(args, 120, output='hidden-fixture')
|
||||
return subprocess.CompletedProcess(args, 0, '', '')
|
||||
with (mock.patch.object(run_full_smoke, 'ROOT', root),
|
||||
mock.patch.object(run_full_smoke, 'demo_environment', return_value=values),
|
||||
mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run),
|
||||
mock.patch.object(run_full_smoke.Diagnostics, 'collect', side_effect=ValueError('hidden-fixture')),
|
||||
mock.patch.object(run_full_smoke, 'check_full', side_effect=AssertionError('HTTP failed hidden-fixture')),
|
||||
mock.patch.dict(os.environ)):
|
||||
with self.assertRaisesRegex(AssertionError, r'HTTP failed \[REDACTED\]'):
|
||||
run_full_smoke.main()
|
||||
text = ''.join(f.read_text() for f in (root / 'target/smoke-diagnostics').glob('*.json'))
|
||||
self.assertNotIn('hidden-fixture', text)
|
||||
self.assertIn('TimeoutExpired', text)
|
||||
|
||||
def test_startup_timeout_keeps_partial_sanitized_output_and_tears_down(self):
|
||||
self.diagnostics_module()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'compose.yaml').touch()
|
||||
values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002',
|
||||
'DB_PASSWORD': 'hidden-fixture'}
|
||||
cleanup = []
|
||||
def run(args, **kwargs):
|
||||
if 'config' in args:
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '')
|
||||
if 'up' in args:
|
||||
raise subprocess.TimeoutExpired(args, 480, output=b'postgres unhealthy hidden-fixture')
|
||||
if 'down' in args:
|
||||
cleanup.append(True)
|
||||
return subprocess.CompletedProcess(args, 0, '', '')
|
||||
with (mock.patch.object(run_full_smoke, 'ROOT', root),
|
||||
mock.patch.object(run_full_smoke, 'demo_environment', return_value=values),
|
||||
mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run),
|
||||
mock.patch.dict(os.environ)):
|
||||
with self.assertRaisesRegex(AssertionError, 'did not become healthy'):
|
||||
run_full_smoke.main()
|
||||
self.assertTrue(cleanup)
|
||||
receipt = json.loads((root / 'target/smoke-diagnostics/startup.json').read_text())
|
||||
self.assertEqual('TimeoutExpired', receipt['error'])
|
||||
self.assertIn('postgres unhealthy', receipt['output'])
|
||||
self.assertNotIn('hidden-fixture', receipt['output'])
|
||||
|
||||
def test_pull_failure_names_service_image_registry_and_stops_before_up(self):
|
||||
module = self.diagnostics_module()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
(root / 'compose.yaml').touch()
|
||||
values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002',
|
||||
'DB_PASSWORD': 'pull-secret'}
|
||||
calls = []
|
||||
model = {'services': {
|
||||
'app-full': {'image': 'qqq-all:local', 'build': {'context': '.'}},
|
||||
'minio-init': {'image': 'public.ecr.aws/aws-cli/aws-cli:2.27.41',
|
||||
'environment': {'DO_NOT_RETAIN': 'raw-config-fixture'}},
|
||||
'sftp': {'image': 'atmoz/sftp:alpine'}}}
|
||||
def run(args, **kwargs):
|
||||
calls.append(args)
|
||||
if 'config' in args:
|
||||
self.assertIn('--no-interpolate', args)
|
||||
self.assertIn('--no-env-resolution', args)
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps(model), '')
|
||||
if 'pull' in args:
|
||||
self.assertEqual('minio-init', args[-1])
|
||||
self.assertNotIn('--ignore-pull-failures', args)
|
||||
self.assertNotIn('--include-deps', args)
|
||||
return subprocess.CompletedProcess(args, 1, '', 'toomanyrequests: Rate exceeded pull-secret')
|
||||
return subprocess.CompletedProcess(args, 0, '', '')
|
||||
with (mock.patch.object(run_full_smoke, 'ROOT', root),
|
||||
mock.patch.object(run_full_smoke, 'demo_environment', return_value=values),
|
||||
mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run),
|
||||
mock.patch.object(run_full_smoke, 'check_full'),
|
||||
mock.patch.dict(os.environ)):
|
||||
with self.assertRaisesRegex(AssertionError, 'minio-init'):
|
||||
run_full_smoke.main()
|
||||
self.assertFalse(any('up' in args for args in calls))
|
||||
receipt = json.loads((root / 'target/smoke-diagnostics/pulls.json').read_text())
|
||||
failed = receipt['pulls'][-1]
|
||||
self.assertEqual('public.ecr.aws', failed['registry'])
|
||||
self.assertEqual(model['services']['minio-init']['image'], failed['image'])
|
||||
self.assertEqual(1, failed['result']['exit_code'])
|
||||
self.assertIn('Rate exceeded', failed['result']['output'])
|
||||
text = json.dumps(receipt)
|
||||
self.assertNotIn('pull-secret', text)
|
||||
self.assertNotIn('raw-config-fixture', text)
|
||||
|
||||
def test_successful_pulls_use_same_images_without_building_or_starting(self):
|
||||
module = self.diagnostics_module()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
evidence = module.Diagnostics(Path(directory), {})
|
||||
self.assertTrue(hasattr(evidence, 'pull'), 'explicit pull diagnostics not implemented')
|
||||
calls = []
|
||||
model = {'services': {'postgres': {'image': 'postgres:16-alpine'},
|
||||
'minio': {'image': 'qqq-all-minio:local', 'build': {'context': './infra/minio'}},
|
||||
'mongo': {'image': 'mongo:7'}}}
|
||||
def run(args, **kwargs):
|
||||
calls.append(args)
|
||||
output = json.dumps(model) if 'config' in args else 'pulled'
|
||||
return subprocess.CompletedProcess(args, 0, output, '')
|
||||
with mock.patch.object(module.subprocess, 'run', side_effect=run):
|
||||
evidence.pull(['docker', 'compose', '-p', 'private'], Path(directory), {})
|
||||
pulls = [args for args in calls if 'pull' in args]
|
||||
self.assertEqual(['mongo', 'postgres'], [args[-1] for args in pulls])
|
||||
for args in pulls:
|
||||
self.assertEqual('1', args[args.index('--parallel') + 1])
|
||||
self.assertEqual('missing', args[args.index('--policy') + 1])
|
||||
self.assertFalse(any('up' in args or 'build' in args for args in calls))
|
||||
receipt = json.loads((Path(directory) / 'pulls.json').read_text())
|
||||
self.assertEqual(['mongo:7', 'postgres:16-alpine'], [r['image'] for r in receipt['pulls']])
|
||||
self.assertEqual(['docker.io', 'docker.io'], [r['registry'] for r in receipt['pulls']])
|
||||
|
||||
def diagnostics_module(self):
|
||||
try:
|
||||
import smoke_diagnostics
|
||||
except ImportError:
|
||||
self.fail('sanitized diagnostics collector is not implemented')
|
||||
return smoke_diagnostics
|
||||
|
||||
def test_redacts_secret_forms_and_bounds_output_after_redaction(self):
|
||||
module = self.diagnostics_module()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
evidence = module.Diagnostics(Path(directory), {'DB_PASSWORD': 'a+/secret',
|
||||
'GITHUB_TOKEN': 'token-1234'})
|
||||
output = evidence.sanitize('x' * 90000 + '\npassword="unknown-value" '
|
||||
'postgres://u:uri-secret@db/ Bearer bearer-secret '
|
||||
'a%2B%2Fsecret a+/secret token-1234 access_token=generated-token client_secret=generated-secret')
|
||||
for secret in ['unknown-value', 'uri-secret', 'bearer-secret',
|
||||
'a%2B%2Fsecret', 'a+/secret', 'token-1234', 'generated-token', 'generated-secret']:
|
||||
self.assertFalse(secret in output, "credential remained in sanitized output")
|
||||
self.assertLessEqual(len(output), 65536)
|
||||
self.assertIn('[REDACTED]', output)
|
||||
|
||||
def test_hashes_only_resolved_first_party_jars_not_other_cached_versions(self):
|
||||
module = self.diagnostics_module()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
jar = root / 'repository/com/kingsrook/qqq/qqq-esb/4.1.0-SNAPSHOT/qqq-esb.jar'
|
||||
jar.parent.mkdir(parents=True)
|
||||
jar.write_bytes(b'actual resolved bytes')
|
||||
(jar.parent / 'unused.jar').write_bytes(b'unused')
|
||||
manifest = root / 'qqq-all-app/target/smoke-classpath.txt'
|
||||
manifest.parent.mkdir(parents=True)
|
||||
manifest.write_text(str(jar))
|
||||
evidence = module.Diagnostics(root / 'target/smoke-diagnostics', {})
|
||||
evidence.jar_hashes(root)
|
||||
data = json.loads((evidence.directory / 'jars.json').read_text())
|
||||
self.assertEqual([{'coordinate': 'com.kingsrook.qqq:qqq-esb:4.1.0-SNAPSHOT',
|
||||
'file': 'qqq-esb.jar',
|
||||
'sha256': hashlib.sha256(b'actual resolved bytes').hexdigest()}], data['dependencies'])
|
||||
self.assertNotIn(str(root), json.dumps(data))
|
||||
|
||||
def test_container_collection_is_selected_redacted_and_survives_command_failure(self):
|
||||
module = self.diagnostics_module()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
evidence = module.Diagnostics(Path(directory), {'DB_PASSWORD': 'secret-fixture'})
|
||||
container = 'a' * 64
|
||||
completed = 'd' * 64
|
||||
image = 'sha256:' + 'b' * 64
|
||||
commands = []
|
||||
def run(args, **kwargs):
|
||||
commands.append(args)
|
||||
if 'ps' in args:
|
||||
return subprocess.CompletedProcess(args, 0, container + '\n' + completed + '\n', '')
|
||||
if 'inspect' in args and 'image' not in args:
|
||||
self.assertIn('--format', args)
|
||||
if args[-1] == completed:
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps({
|
||||
'id': completed, 'service': 'mongo-init', 'image_id': image,
|
||||
'status': 'exited', 'exit_code': 0, 'oom_killed': False,
|
||||
'error': '', 'health': None}), '')
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps({
|
||||
'id': container, 'service': 'postgres', 'image_id': image,
|
||||
'status': 'exited', 'exit_code': 1, 'oom_killed': False,
|
||||
'error': '', 'health': {'Status': 'unhealthy', 'Log': [
|
||||
{'ExitCode': 1, 'Output': 'failed secret-fixture'}]}}), '')
|
||||
if 'image' in args:
|
||||
return subprocess.CompletedProcess(args, 0, json.dumps({'id': image,
|
||||
'repo_digests': ['postgres@sha256:' + 'c' * 64],
|
||||
'architecture': 'amd64', 'os': 'linux'}), '')
|
||||
if 'logs' in args:
|
||||
raise subprocess.TimeoutExpired(args, 15, output='secret-fixture')
|
||||
self.fail('unexpected command')
|
||||
with mock.patch.object(module.subprocess, 'run', side_effect=run):
|
||||
evidence.collect(['docker', 'compose', '-p', 'private'], Path(directory), {})
|
||||
text = '\n'.join(f.read_text() for f in Path(directory).glob('*.json'))
|
||||
self.assertNotIn('secret-fixture', text)
|
||||
self.assertIn('unhealthy', text)
|
||||
self.assertIn('amd64', text)
|
||||
states = json.loads((Path(directory) / 'services.json').read_text())['services']
|
||||
self.assertEqual('mongo-init', states[1]['service'])
|
||||
self.assertEqual(0, states[1]['exit_code'])
|
||||
self.assertIsNone(states[1]['health']['status'])
|
||||
self.assertIn('TimeoutExpired', text)
|
||||
self.assertFalse(any('config' in c or '--volumes' in c for c in commands))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user