From ddad68cf022584cdb4654355ec9ccc73ebc034dc Mon Sep 17 00:00:00 2001 From: James Maes Date: Sat, 26 Sep 2026 09:37:12 -0500 Subject: [PATCH 1/5] feat(compose): add core and full demo stack (#7) --- .dockerignore | 10 + .env.example | 29 ++ .github/workflows/publish-image.yml | 43 +++ Dockerfile | 18 ++ README.md | 23 ++ compose.yaml | 285 ++++++++++++++++++ infra/keycloak/realm.json | 51 ++++ infra/minio/Dockerfile | 16 + infra/mongo/init.sh | 12 + infra/mongo/start.sh | 7 + infra/mysql/01-domain.sql | 6 + infra/postgres/01-domain.sql | 7 + infra/postgres/02-auth-and-roles.sql | 90 ++++++ qqq-all-app/README.md | 6 +- qqq-all-app/pom.xml | 1 + .../com/kingsrook/qqq/all/CoreRuntime.java | 7 +- .../kingsrook/qqq/all/ProfileLauncher.java | 2 +- 17 files changed, 609 insertions(+), 4 deletions(-) create mode 100644 .dockerignore create mode 100644 .env.example create mode 100644 .github/workflows/publish-image.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/keycloak/realm.json create mode 100644 infra/minio/Dockerfile create mode 100644 infra/mongo/init.sh create mode 100644 infra/mongo/start.sh create mode 100644 infra/mysql/01-domain.sql create mode 100644 infra/postgres/01-domain.sql create mode 100644 infra/postgres/02-auth-and-roles.sql diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..06a47b5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.git +.github +.circleci +.env +docs +qqq-all-bom +qqq-all-bom-test +qqq-all-app/src +qqq-all-app/target/* +!qqq-all-app/target/qqq-all-app.jar diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b4f9c06 --- /dev/null +++ b/.env.example @@ -0,0 +1,29 @@ +# Local demonstration values only. Copy to .env and override before exposing ports. +QQQ_ALL_PORT=8080 +KEYCLOAK_PORT=8081 +POSTGRES_USER=qqq +POSTGRES_PASSWORD=demo-postgres-password +POSTGRES_DATABASE=qqq +MYSQL_USER=qqq +MYSQL_PASSWORD=demo-mysql-password +MYSQL_ROOT_PASSWORD=demo-mysql-root-password +MYSQL_DATABASE=qqq +MONGO_USER=qqq +MONGO_PASSWORD=demo-mongo-password +MONGO_DATABASE=qqq +MONGO_REPLICA_SET=rs0 +MONGO_KEYFILE=demo-mongo-internal-replica-key-change-this-before-sharing +MINIO_ROOT_USER=qqq +MINIO_ROOT_PASSWORD=demo-minio-password +MINIO_BUCKET=qqq-demo +SFTP_USER=qqq +SFTP_PASSWORD=demo-sftp-password +RABBITMQ_USER=qqq +RABBITMQ_PASSWORD=demo-rabbitmq-password +ARTEMIS_USER=qqq +ARTEMIS_PASSWORD=demo-artemis-password +KEYCLOAK_ADMIN=admin +KEYCLOAK_ADMIN_PASSWORD=demo-keycloak-admin-password +OIDC_CLIENT_SECRET=demo-qqq-all-client-secret +DEMO_ADMIN_PASSWORD=demo-admin-password +DEMO_USER_PASSWORD=demo-user-password diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 0000000..a4840e1 --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,43 @@ +name: Publish develop image + +on: + push: + branches: [develop] + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + cache: maven + - name: Verify application + run: mvn -B -ntp verify + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish develop image + run: | + docker build \ + --label org.opencontainers.image.source=https://github.com/QRun-IO/qqq-all \ + --tag ghcr.io/qrun-io/qqq-all:develop . + docker push ghcr.io/qrun-io/qqq-all:develop + - name: Require public package visibility + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + visibility="$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility)" + test "$visibility" = public || { + echo 'Set the qqq-all container package visibility to public in GitHub package settings.' >&2 + exit 1 + } diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8c29f94 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM eclipse-temurin:21-jre-jammy + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 10001 qqq \ + && useradd --system --uid 10001 --gid qqq --home-dir /app qqq \ + && mkdir -p /app/data \ + && chown -R qqq:qqq /app + +WORKDIR /app +COPY --chown=qqq:qqq qqq-all-app/target/qqq-all-app.jar /app/qqq-all.jar +USER 10001:10001 +EXPOSE 8080 +ENV QQQ_ALL_DATA_DIR=/app/data +HEALTHCHECK --interval=10s --timeout=5s --start-period=45s --retries=6 \ + CMD curl --fail --silent http://127.0.0.1:8080/health >/dev/null || exit 1 +ENTRYPOINT ["java", "-jar", "/app/qqq-all.jar"] diff --git a/README.md b/README.md index 72e899f..7a9719a 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,29 @@ Requires Java 21 and Maven. mvn -B verify ``` +## Run locally + +The core profile needs only Java 21: + +```bash +java -jar qqq-all-app/target/qqq-all-app.jar +``` + +For containers, copy the demo settings and build the jar before starting either Compose profile: + +```bash +cp .env.example .env +mvn -B verify +docker compose --profile core up --build --wait +# Or: docker compose --profile full up --build --wait +``` + +The app is at `http://127.0.0.1:8080/health` (change `QQQ_ALL_PORT` in `.env` if needed). The full profile also publishes local-only management ports for Keycloak (`8081`), Mailpit (`8025`), MinIO (`9001`), Artemis/Jolokia (`8161`), and RabbitMQ (`15672`). The demo Keycloak users are `demo-admin` and `demo-user`; their passwords and the confidential client secret come from `.env`. The sample values in `.env.example` are for local demonstrations only. The default browser-visible issuer is `http://keycloak.localhost:8081`; if your host does not resolve `keycloak.localhost` to loopback, add that local hosts entry before trying OIDC login. + +The full stack initializes the MongoDB replica set, MinIO bucket, sample SQL tables, OAuth2 session tables, and demo roles/permissions. PostgreSQL and MySQL initialization scripts run only when their data volumes are first created. To stop, run `docker compose --profile full down`; adding `--volumes` removes demo data. See [the app guide](qqq-all-app/README.md) for the full-profile environment contract. + +On merges to `develop`, [the image workflow](.github/workflows/publish-image.yml) verifies the reactor and pushes `ghcr.io/qrun-io/qqq-all:develop`. GitHub creates a new container package private by default; an organization owner must set the `qqq-all` package visibility to **public** once in GitHub package settings. The workflow checks that visibility and fails until it is public. + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md), the [Code of Conduct](CODE_OF_CONDUCT.md), and the [security policy](SECURITY.md). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..59d5c9b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,285 @@ +name: qqq-all + +x-app-image: &app-image + build: + context: . + image: qqq-all:local + volumes: + - app-data:/app/data + ports: + - "127.0.0.1:${QQQ_ALL_PORT:-8080}:8080" + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:8080/health"] + interval: 10s + timeout: 5s + retries: 6 + start_period: 45s + +services: + app-core: + <<: *app-image + profiles: [core] + environment: + QQQ_ALL_PROFILE: core + QQQ_ALL_PORT: "8080" + QQQ_ALL_BIND_HOST: 0.0.0.0 + + app-full: + <<: *app-image + profiles: [full] + environment: + QQQ_ALL_PROFILE: full + QQQ_ALL_PORT: "8080" + QQQ_ALL_BIND_HOST: 0.0.0.0 + QQQ_ALL_POSTGRES_HOST: postgres + QQQ_ALL_POSTGRES_DATABASE: ${POSTGRES_DATABASE:-qqq} + QQQ_ALL_POSTGRES_USER: ${POSTGRES_USER:-qqq} + QQQ_ALL_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD} + QQQ_ALL_MYSQL_HOST: mysql + QQQ_ALL_MYSQL_DATABASE: ${MYSQL_DATABASE:-qqq} + QQQ_ALL_MYSQL_USER: ${MYSQL_USER:-qqq} + QQQ_ALL_MYSQL_PASSWORD: ${MYSQL_PASSWORD:?set MYSQL_PASSWORD} + QQQ_ALL_MONGO_HOST: mongo + QQQ_ALL_MONGO_DATABASE: ${MONGO_DATABASE:-qqq} + QQQ_ALL_MONGO_USER: ${MONGO_USER:-qqq} + QQQ_ALL_MONGO_PASSWORD: ${MONGO_PASSWORD:?set MONGO_PASSWORD} + QQQ_ALL_MONGO_REPLICA_SET: ${MONGO_REPLICA_SET:-rs0} + QQQ_ALL_S3_ENDPOINT: http://minio:9000 + QQQ_ALL_S3_BUCKET: ${MINIO_BUCKET:-qqq-demo} + QQQ_ALL_S3_ACCESS_KEY: ${MINIO_ROOT_USER:-qqq} + QQQ_ALL_S3_SECRET_KEY: ${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD} + QQQ_ALL_SFTP_HOST: sftp + QQQ_ALL_SFTP_USER: ${SFTP_USER:-qqq} + QQQ_ALL_SFTP_PASSWORD: ${SFTP_PASSWORD:?set SFTP_PASSWORD} + QQQ_ALL_ARTEMIS_URL: tcp://artemis:61616 + QQQ_ALL_RABBITMQ_URL: amqp://rabbitmq:5672/%2F + QQQ_ALL_RABBITMQ_USER: ${RABBITMQ_USER:-qqq} + QQQ_ALL_RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD} + QQQ_ALL_SMTP_HOST: mailpit + QQQ_ALL_OIDC_BASE_URL: http://keycloak.localhost:${KEYCLOAK_PORT:-8081}/realms/qqq-all + QQQ_ALL_OIDC_CLIENT_ID: qqq-all + QQQ_ALL_OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:?set OIDC_CLIENT_SECRET} + depends_on: + postgres: {condition: service_healthy} + mysql: {condition: service_healthy} + mongo-init: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + sftp: {condition: service_healthy} + opensearch: {condition: service_healthy} + artemis: {condition: service_healthy} + rabbitmq: {condition: service_healthy} + mailpit: {condition: service_healthy} + keycloak: {condition: service_healthy} + + postgres: + image: postgres:16-alpine + profiles: [full] + environment: + POSTGRES_DB: ${POSTGRES_DATABASE:-qqq} + POSTGRES_USER: ${POSTGRES_USER:-qqq} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD} + volumes: + - postgres-data:/var/lib/postgresql/data + - ./infra/postgres:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 5s + timeout: 5s + retries: 20 + + mysql: + image: mysql:8.4 + profiles: [full] + environment: + MYSQL_DATABASE: ${MYSQL_DATABASE:-qqq} + MYSQL_USER: ${MYSQL_USER:-qqq} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?set MYSQL_PASSWORD} + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?set MYSQL_ROOT_PASSWORD} + volumes: + - mysql-data:/var/lib/mysql + - ./infra/mysql:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root -p\"$$MYSQL_ROOT_PASSWORD\" --silent"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 20s + + mongo: + image: mongo:7 + profiles: [full] + entrypoint: ["bash", "/start.sh"] + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER:-qqq} + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD:?set MONGO_PASSWORD} + MONGO_REPLICA_SET: ${MONGO_REPLICA_SET:-rs0} + MONGO_KEYFILE: ${MONGO_KEYFILE:?set MONGO_KEYFILE} + volumes: + - mongo-data:/data/db + - mongo-config:/data/configdb + - ./infra/mongo/start.sh:/start.sh:ro + healthcheck: + test: ["CMD-SHELL", "mongosh --quiet -u \"$$MONGO_INITDB_ROOT_USERNAME\" -p \"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --eval 'db.adminCommand({ping:1}).ok' | grep -q 1"] + interval: 5s + timeout: 5s + retries: 20 + + mongo-init: + image: mongo:7 + profiles: [full] + depends_on: + mongo: {condition: service_healthy} + environment: + MONGO_REPLICA_SET: ${MONGO_REPLICA_SET:-rs0} + MONGO_USER: ${MONGO_USER:-qqq} + MONGO_PASSWORD: ${MONGO_PASSWORD:?set MONGO_PASSWORD} + volumes: + - ./infra/mongo/init.sh:/init.sh:ro + entrypoint: ["bash", "/init.sh"] + restart: "no" + + minio: + build: + context: ./infra/minio + image: qqq-all-minio:local + profiles: [full] + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-qqq} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD} + volumes: + - minio-data:/data + ports: + - "127.0.0.1:9001:9001" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/live"] + interval: 5s + timeout: 5s + retries: 20 + + minio-init: + image: public.ecr.aws/aws-cli/aws-cli:2.27.41 + profiles: [full] + depends_on: + minio: {condition: service_healthy} + environment: + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-qqq} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD} + AWS_DEFAULT_REGION: us-east-1 + MINIO_BUCKET: ${MINIO_BUCKET:-qqq-demo} + entrypoint: ["/bin/sh", "-c"] + command: ["aws --endpoint-url http://minio:9000 s3api head-bucket --bucket \"$$MINIO_BUCKET\" >/dev/null 2>&1 || aws --endpoint-url http://minio:9000 s3api create-bucket --bucket \"$$MINIO_BUCKET\""] + restart: "no" + + sftp: + image: atmoz/sftp:alpine + profiles: [full] + command: "${SFTP_USER:-qqq}:${SFTP_PASSWORD:?set SFTP_PASSWORD}:1001:1001:upload" + volumes: + - sftp-data:/home/${SFTP_USER:-qqq}/upload + healthcheck: + test: ["CMD-SHELL", "pgrep sshd >/dev/null"] + interval: 5s + timeout: 5s + retries: 20 + + opensearch: + image: opensearchproject/opensearch:2.19.1 + profiles: [full] + environment: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" + OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m + volumes: + - opensearch-data:/usr/share/opensearch/data + healthcheck: + test: ["CMD-SHELL", "curl --fail --silent http://127.0.0.1:9200/_cluster/health >/dev/null"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 30s + + artemis: + image: apache/activemq-artemis:2.42.0-alpine + profiles: [full] + environment: + ARTEMIS_USER: ${ARTEMIS_USER:-qqq} + ARTEMIS_PASSWORD: ${ARTEMIS_PASSWORD:?set ARTEMIS_PASSWORD} + ANONYMOUS_LOGIN: "true" + volumes: + - artemis-data:/var/lib/artemis-instance + ports: + - "127.0.0.1:8161:8161" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8161/console/"] + interval: 10s + timeout: 5s + retries: 20 + + rabbitmq: + image: rabbitmq:4-management-alpine + profiles: [full] + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-qqq} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD} + volumes: + - rabbitmq-data:/var/lib/rabbitmq + ports: + - "127.0.0.1:15672:15672" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 20 + + mailpit: + image: axllent/mailpit:v1.22.3 + profiles: [full] + ports: + - "127.0.0.1:8025:8025" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8025/api/v1/info"] + interval: 5s + timeout: 5s + retries: 20 + + keycloak: + image: quay.io/keycloak/keycloak:26.2.5 + profiles: [full] + networks: + default: + aliases: [keycloak.localhost] + command: ["start-dev", "--import-realm"] + environment: + KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN:-admin} + KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:?set KEYCLOAK_ADMIN_PASSWORD} + KC_HEALTH_ENABLED: "true" + KC_HOSTNAME: http://keycloak.localhost:${KEYCLOAK_PORT:-8081} + KC_HTTP_ENABLED: "true" + KC_HTTP_PORT: ${KEYCLOAK_PORT:-8081} + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:?set OIDC_CLIENT_SECRET} + DEMO_ADMIN_PASSWORD: ${DEMO_ADMIN_PASSWORD:?set DEMO_ADMIN_PASSWORD} + DEMO_USER_PASSWORD: ${DEMO_USER_PASSWORD:?set DEMO_USER_PASSWORD} + QQQ_ALL_PORT: ${QQQ_ALL_PORT:-8080} + volumes: + - ./infra/keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro + ports: + - "127.0.0.1:${KEYCLOAK_PORT:-8081}:${KEYCLOAK_PORT:-8081}" + healthcheck: + test: ["CMD", "bash", "-c", "{ printf 'HEAD /health/ready HTTP/1.0\r\n\r\n' >&0; grep 'HTTP/1.0 200'; } 0<>/dev/tcp/127.0.0.1/9000"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 30s + +volumes: + app-data: + postgres-data: + mysql-data: + mongo-data: + mongo-config: + minio-data: + sftp-data: + opensearch-data: + artemis-data: + rabbitmq-data: diff --git a/infra/keycloak/realm.json b/infra/keycloak/realm.json new file mode 100644 index 0000000..807375c --- /dev/null +++ b/infra/keycloak/realm.json @@ -0,0 +1,51 @@ +{ + "realm": "qqq-all", + "enabled": true, + "displayName": "QQQ All Demo", + "registrationAllowed": false, + "loginWithEmailAllowed": true, + "roles": { + "realm": [ + {"name": "admin", "description": "Full demo permissions"}, + {"name": "viewer", "description": "Read-only demo permissions"} + ] + }, + "clients": [ + { + "clientId": "qqq-all", + "name": "QQQ All Demo", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "clientAuthenticatorType": "client-secret", + "secret": "${OIDC_CLIENT_SECRET}", + "standardFlowEnabled": true, + "directAccessGrantsEnabled": false, + "redirectUris": ["http://localhost:${QQQ_ALL_PORT}/*", "http://127.0.0.1:${QQQ_ALL_PORT}/*"], + "webOrigins": ["http://localhost:${QQQ_ALL_PORT}", "http://127.0.0.1:${QQQ_ALL_PORT}"], + "defaultClientScopes": ["web-origins", "acr", "profile", "roles", "email"] + } + ], + "users": [ + { + "username": "demo-admin", + "email": "admin@example.test", + "firstName": "Demo", + "lastName": "Admin", + "enabled": true, + "emailVerified": true, + "realmRoles": ["admin"], + "credentials": [{"type": "password", "value": "${DEMO_ADMIN_PASSWORD}", "temporary": false}] + }, + { + "username": "demo-user", + "email": "user@example.test", + "firstName": "Demo", + "lastName": "User", + "enabled": true, + "emailVerified": true, + "realmRoles": ["viewer"], + "credentials": [{"type": "password", "value": "${DEMO_USER_PASSWORD}", "temporary": false}] + } + ] +} diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 0000000..743eff4 --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.24-alpine AS build +RUN apk add --no-cache git +RUN git clone --depth 1 --branch RELEASE.2025-09-06T17-38-46Z https://github.com/minio/minio.git /src +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath -o /minio . + +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates \ + && addgroup -S -g 10001 minio \ + && adduser -S -u 10001 -G minio minio \ + && mkdir /data \ + && chown minio:minio /data +COPY --from=build /minio /usr/local/bin/minio +USER 10001:10001 +EXPOSE 9000 9001 +ENTRYPOINT ["minio"] diff --git a/infra/mongo/init.sh b/infra/mongo/init.sh new file mode 100644 index 0000000..f04753c --- /dev/null +++ b/infra/mongo/init.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +for attempt in {1..30}; do + if mongosh --host mongo --quiet -u "${MONGO_USER}" -p "${MONGO_PASSWORD}" --authenticationDatabase admin \ + --eval 'try { rs.status().ok } catch (e) { if (e.codeName === "NotYetInitialized") { rs.initiate({_id: process.env.MONGO_REPLICA_SET, members: [{_id: 0, host: "mongo:27017"}]}).ok } else { throw e } }' | grep -q 1; then + exit 0 + fi + sleep 2 +done +echo 'Mongo replica set did not initialize' >&2 +exit 1 diff --git a/infra/mongo/start.sh b/infra/mongo/start.sh new file mode 100644 index 0000000..c3385ab --- /dev/null +++ b/infra/mongo/start.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +printf '%s' "${MONGO_KEYFILE}" | base64 > /data/configdb/keyfile +chmod 400 /data/configdb/keyfile +chown mongodb:mongodb /data/configdb/keyfile +exec docker-entrypoint.sh mongod --replSet "${MONGO_REPLICA_SET}" --bind_ip_all --auth --keyFile /data/configdb/keyfile diff --git a/infra/mysql/01-domain.sql b/infra/mysql/01-domain.sql new file mode 100644 index 0000000..6b789c6 --- /dev/null +++ b/infra/mysql/01-domain.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS supplier_order ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + reference VARCHAR(120) NOT NULL +); + +INSERT IGNORE INTO supplier_order (id, reference) VALUES (1, 'DEMO-SUPPLIER-001'); diff --git a/infra/postgres/01-domain.sql b/infra/postgres/01-domain.sql new file mode 100644 index 0000000..1fbbafa --- /dev/null +++ b/infra/postgres/01-domain.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS warehouse_customer ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name VARCHAR(120) NOT NULL +); + +INSERT INTO warehouse_customer (id, name) VALUES (1, 'Demo Warehouse Customer') +ON CONFLICT (id) DO NOTHING; diff --git a/infra/postgres/02-auth-and-roles.sql b/infra/postgres/02-auth-and-roles.sql new file mode 100644 index 0000000..8469f9b --- /dev/null +++ b/infra/postgres/02-auth-and-roles.sql @@ -0,0 +1,90 @@ +-- C4's OAuth2 session tables and qbit-user-role-permissions tables. +-- Names follow QInstanceEnricher's camelCase-to-snake_case backend mapping. +CREATE TABLE IF NOT EXISTS user_session ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + uuid VARCHAR(40) UNIQUE, + access_token TEXT, + user_id VARCHAR(100) +); + +CREATE TABLE IF NOT EXISTS redirect_state ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + state VARCHAR(45) UNIQUE, + redirect_uri VARCHAR(4096), + create_date TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS "user" ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + email VARCHAR(250), + full_name VARCHAR(250) +); + +CREATE TABLE IF NOT EXISTS role ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + name VARCHAR(100) NOT NULL UNIQUE, + description VARCHAR(500) +); + +CREATE TABLE IF NOT EXISTS permission ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + name VARCHAR(250) NOT NULL UNIQUE, + description VARCHAR(500), + object_type VARCHAR(50), + object_label VARCHAR(250) +); + +CREATE TABLE IF NOT EXISTS role_permission_int ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + role_id INTEGER NOT NULL REFERENCES role(id), + permission_id INTEGER NOT NULL REFERENCES permission(id), + UNIQUE (role_id, permission_id) +); + +CREATE TABLE IF NOT EXISTS user_role_int ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + user_id INTEGER NOT NULL REFERENCES "user"(id), + role_id INTEGER NOT NULL REFERENCES role(id), + UNIQUE (user_id, role_id) +); + +CREATE TABLE IF NOT EXISTS user_permission_int ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + user_id INTEGER NOT NULL REFERENCES "user"(id), + permission_id INTEGER NOT NULL REFERENCES permission(id), + UNIQUE (user_id, permission_id) +); + +INSERT INTO role (name, description) VALUES + ('admin', 'All QQQ All demo tables'), + ('viewer', 'Read-only QQQ All demo tables') +ON CONFLICT (name) DO NOTHING; + +INSERT INTO permission (name, object_type, object_label) +SELECT table_name || '.' || action, 'table', table_name +FROM (VALUES + ('customer'), ('order'), ('orderLine'), ('product'), + ('warehouseCustomer'), ('supplierOrder'), ('shipment'), ('document'), ('importFile') +) AS tables(table_name) +CROSS JOIN (VALUES ('read'), ('insert'), ('edit'), ('delete')) AS actions(action) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO role_permission_int (role_id, permission_id) +SELECT role.id, permission.id +FROM role CROSS JOIN permission +WHERE role.name = 'admin' OR (role.name = 'viewer' AND permission.name LIKE '%.read') +ON CONFLICT (role_id, permission_id) DO NOTHING; diff --git a/qqq-all-app/README.md b/qqq-all-app/README.md index e5b5583..a78a1db 100644 --- a/qqq-all-app/README.md +++ b/qqq-all-app/README.md @@ -4,11 +4,13 @@ Build with Java 21 and Maven, then start the standalone core profile: ```sh mvn -pl qqq-all-app -am verify -java -jar qqq-all-app/target/qqq-all-app-4.1.0-SNAPSHOT.jar +java -jar qqq-all-app/target/qqq-all-app.jar ``` The core app binds to `127.0.0.1` and serves the Next dashboard at `http://127.0.0.1:8080/` and health at `/health`. Set `QQQ_ALL_PORT` to change the port and `QQQ_ALL_DATA_DIR` to change the default `./data` directory. On first start it seeds customers and orders in H2, order lines in SQLite, and products as JSON files. Artemis runs in the same JVM; inserts and updates to `order` publish to the `orderEvents` topic and trigger `syncOrder`. +`QQQ_ALL_BIND_HOST` can select another interface when intentionally running behind a container port mapping. The Compose core service sets it to `0.0.0.0` inside the container while publishing only on host loopback. + Core uses mock authentication. The `sessionId` cookie selects the two fixed demo identities: `aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa` for Admin and `dddddddd-dddd-4ddd-8ddd-dddddddddddd` for Demo User. Any other session ID uses Demo User. These identities are for local demonstrations; role-based permissions are wired in task C5. ## Full profile @@ -28,4 +30,4 @@ Configure the following environment variables before starting the same jar: | Mailpit SMTP | `QQQ_ALL_SMTP_HOST` | `QQQ_ALL_SMTP_PORT` (1025) | | Keycloak OIDC | `QQQ_ALL_OIDC_BASE_URL`, `QQQ_ALL_OIDC_CLIENT_ID`, `QQQ_ALL_OIDC_CLIENT_SECRET` | `QQQ_ALL_OIDC_EXTERNAL_BASE_URL` (base URL), `QQQ_ALL_OIDC_SCOPES` (`openid email profile`) | -The full app adds sample tables on each external backend, `orderEvents` on Artemis, and `orderSyncEvents` on RabbitMQ. Keycloak realm roles resolve by name against the PostgreSQL `role` table from `qbit-user-role-permissions`, then become QQQ session permissions. Unknown roles receive no permissions. PostgreSQL must also contain the qbit permission tables and OAuth session and redirect-state tables. External database schema, demo roles, the MinIO bucket, and the service stack are provisioned by the forthcoming compose and seed work in C6; this C4 profile expects those resources to exist. +The full app adds sample tables on each external backend, `orderEvents` on Artemis, and `orderSyncEvents` on RabbitMQ. Keycloak realm roles resolve by name against the PostgreSQL `role` table from `qbit-user-role-permissions`, then become QQQ session permissions. Unknown roles receive no permissions. The Compose profile provisions demo SQL tables, OAuth session and redirect-state tables, qbit permission tables and roles, and the MinIO bucket on first startup. diff --git a/qqq-all-app/pom.xml b/qqq-all-app/pom.xml index a5c6648..b0394eb 100644 --- a/qqq-all-app/pom.xml +++ b/qqq-all-app/pom.xml @@ -48,6 +48,7 @@ + qqq-all-app org.apache.maven.plugins diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/CoreRuntime.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/CoreRuntime.java index 1492c66..3ef7b7c 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/CoreRuntime.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/CoreRuntime.java @@ -31,6 +31,11 @@ final class CoreRuntime implements AutoCloseable } static CoreRuntime launch(Path dataDirectory, Integer port) throws Exception + { + return launch(dataDirectory, port, "127.0.0.1"); + } + + static CoreRuntime launch(Path dataDirectory, Integer port, String bindHost) throws Exception { CoreData.seed(dataDirectory); Integer brokerPort = findFreePort(); @@ -51,7 +56,7 @@ final class CoreRuntime implements AutoCloseable new QqqAllApplication(dataDirectory, brokerUrl), new QApplicationLauncherConfig().withRegisterShutdownHook(false) .withServerCustomizer(server -> server.withPort(port) - .withJavalinConfigCustomizer(config -> config.jetty.host = "127.0.0.1"))); + .withJavalinConfigCustomizer(config -> config.jetty.host = bindHost))); return new CoreRuntime(broker, launcher); } catch(Exception | LinkageError e) diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java index 3ff460c..ecd2d7b 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java @@ -40,7 +40,7 @@ public final class ProfileLauncher Integer port = Integer.valueOf(environment.getOrDefault("QQQ_ALL_PORT", "8080")); if("core".equals(selected)) { - CoreRuntime.launch(dataDirectory, port); + CoreRuntime.launch(dataDirectory, port, environment.getOrDefault("QQQ_ALL_BIND_HOST", "127.0.0.1")); return; } -- 2.54.0 From d5f7d780351c913d3686513379911ee7dc1fbf1e Mon Sep 17 00:00:00 2001 From: James Maes Date: Sat, 26 Sep 2026 09:51:41 -0500 Subject: [PATCH 2/5] fix(compose): secure broker management and gate image publication (#7) --- .github/workflows/publish-image.yml | 21 +++-- README.md | 4 +- compose.yaml | 6 +- qqq-all-app/README.md | 13 ++- .../qqq/all/FullProfileApplication.java | 12 ++- .../qqq/all/FullProfileSettings.java | 3 +- .../all/FullProfileBrokerIntegrationTest.java | 91 +++++++++++++++++++ .../kingsrook/qqq/all/FullProfileTest.java | 12 +++ 8 files changed, 148 insertions(+), 14 deletions(-) create mode 100644 qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileBrokerIntegrationTest.java diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index a4840e1..5bbc218 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -14,6 +14,18 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Require public package visibility before publishing + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + visibility="$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility)" || { + echo 'The qqq-all container package must exist and be public before develop images can be published.' >&2 + exit 1 + } + test "$visibility" = public || { + echo 'Set the qqq-all container package visibility to public in GitHub package settings before publishing.' >&2 + exit 1 + } - uses: actions/setup-java@v4 with: distribution: temurin @@ -32,12 +44,3 @@ jobs: --label org.opencontainers.image.source=https://github.com/QRun-IO/qqq-all \ --tag ghcr.io/qrun-io/qqq-all:develop . docker push ghcr.io/qrun-io/qqq-all:develop - - name: Require public package visibility - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - visibility="$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility)" - test "$visibility" = public || { - echo 'Set the qqq-all container package visibility to public in GitHub package settings.' >&2 - exit 1 - } diff --git a/README.md b/README.md index 7a9719a..8923e55 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,9 @@ The app is at `http://127.0.0.1:8080/health` (change `QQQ_ALL_PORT` in `.env` if The full stack initializes the MongoDB replica set, MinIO bucket, sample SQL tables, OAuth2 session tables, and demo roles/permissions. PostgreSQL and MySQL initialization scripts run only when their data volumes are first created. To stop, run `docker compose --profile full down`; adding `--volumes` removes demo data. See [the app guide](qqq-all-app/README.md) for the full-profile environment contract. -On merges to `develop`, [the image workflow](.github/workflows/publish-image.yml) verifies the reactor and pushes `ghcr.io/qrun-io/qqq-all:develop`. GitHub creates a new container package private by default; an organization owner must set the `qqq-all` package visibility to **public** once in GitHub package settings. The workflow checks that visibility and fails until it is public. +Artemis requires a configured login on a fresh volume. If an older demo volume was created with anonymous login enabled, recreate that volume before relying on the new setting; `docker compose --profile full down --volumes` resets all local demo data. + +On merges to `develop`, [the image workflow](.github/workflows/publish-image.yml) checks that the `qqq-all` container package is already public, verifies the reactor, and pushes `ghcr.io/qrun-io/qqq-all:develop`. GitHub creates a new container package private by default, so the first image requires a deliberate, one-time bootstrap: an organization owner publishes an initial image, then changes the package visibility to **public** in GitHub package settings. Until that is done, the workflow fails before any image push. Do not enable automatic develop publication until the package is public. ## Contributing diff --git a/compose.yaml b/compose.yaml index 59d5c9b..8cb90d6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -52,9 +52,13 @@ services: QQQ_ALL_SFTP_USER: ${SFTP_USER:-qqq} QQQ_ALL_SFTP_PASSWORD: ${SFTP_PASSWORD:?set SFTP_PASSWORD} QQQ_ALL_ARTEMIS_URL: tcp://artemis:61616 + QQQ_ALL_ARTEMIS_USER: ${ARTEMIS_USER:-qqq} + QQQ_ALL_ARTEMIS_PASSWORD: ${ARTEMIS_PASSWORD:?set ARTEMIS_PASSWORD} + QQQ_ALL_ARTEMIS_MANAGEMENT_URL: http://artemis:8161 QQQ_ALL_RABBITMQ_URL: amqp://rabbitmq:5672/%2F QQQ_ALL_RABBITMQ_USER: ${RABBITMQ_USER:-qqq} QQQ_ALL_RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD} + QQQ_ALL_RABBITMQ_MANAGEMENT_URL: http://rabbitmq:15672 QQQ_ALL_SMTP_HOST: mailpit QQQ_ALL_OIDC_BASE_URL: http://keycloak.localhost:${KEYCLOAK_PORT:-8081}/realms/qqq-all QQQ_ALL_OIDC_CLIENT_ID: qqq-all @@ -205,7 +209,7 @@ services: environment: ARTEMIS_USER: ${ARTEMIS_USER:-qqq} ARTEMIS_PASSWORD: ${ARTEMIS_PASSWORD:?set ARTEMIS_PASSWORD} - ANONYMOUS_LOGIN: "true" + ANONYMOUS_LOGIN: "false" volumes: - artemis-data:/var/lib/artemis-instance ports: diff --git a/qqq-all-app/README.md b/qqq-all-app/README.md index a78a1db..d253122 100644 --- a/qqq-all-app/README.md +++ b/qqq-all-app/README.md @@ -26,8 +26,19 @@ Configure the following environment variables before starting the same jar: | MongoDB replica set | `QQQ_ALL_MONGO_HOST`, `QQQ_ALL_MONGO_DATABASE`, `QQQ_ALL_MONGO_USER`, `QQQ_ALL_MONGO_PASSWORD`, `QQQ_ALL_MONGO_REPLICA_SET` | `QQQ_ALL_MONGO_PORT` (27017), `QQQ_ALL_MONGO_AUTH_DATABASE` (admin) | | MinIO S3 | `QQQ_ALL_S3_ENDPOINT`, `QQQ_ALL_S3_BUCKET`, `QQQ_ALL_S3_ACCESS_KEY`, `QQQ_ALL_S3_SECRET_KEY` | `QQQ_ALL_S3_REGION` (us-east-1) | | SFTP | `QQQ_ALL_SFTP_HOST`, `QQQ_ALL_SFTP_USER`, `QQQ_ALL_SFTP_PASSWORD` | `QQQ_ALL_SFTP_PORT` (22), `QQQ_ALL_SFTP_BASE_PATH` (/upload) | -| ESB | `QQQ_ALL_ARTEMIS_URL`, `QQQ_ALL_RABBITMQ_URL`, `QQQ_ALL_RABBITMQ_USER`, `QQQ_ALL_RABBITMQ_PASSWORD` | — | +| ESB | `QQQ_ALL_ARTEMIS_URL`, `QQQ_ALL_ARTEMIS_USER`, `QQQ_ALL_ARTEMIS_PASSWORD`, `QQQ_ALL_ARTEMIS_MANAGEMENT_URL`, `QQQ_ALL_RABBITMQ_URL`, `QQQ_ALL_RABBITMQ_USER`, `QQQ_ALL_RABBITMQ_PASSWORD`, `QQQ_ALL_RABBITMQ_MANAGEMENT_URL` | — | | Mailpit SMTP | `QQQ_ALL_SMTP_HOST` | `QQQ_ALL_SMTP_PORT` (1025) | | Keycloak OIDC | `QQQ_ALL_OIDC_BASE_URL`, `QQQ_ALL_OIDC_CLIENT_ID`, `QQQ_ALL_OIDC_CLIENT_SECRET` | `QQQ_ALL_OIDC_EXTERNAL_BASE_URL` (base URL), `QQQ_ALL_OIDC_SCOPES` (`openid email profile`) | The full app adds sample tables on each external backend, `orderEvents` on Artemis, and `orderSyncEvents` on RabbitMQ. Keycloak realm roles resolve by name against the PostgreSQL `role` table from `qbit-user-role-permissions`, then become QQQ session permissions. Unknown roles receive no permissions. The Compose profile provisions demo SQL tables, OAuth session and redirect-state tables, qbit permission tables and roles, and the MinIO bucket on first startup. + +After starting the full Compose profile, run the opt-in broker management smoke test with the same broker credentials. For the supplied demo environment: + +```sh +set -a +. ./.env.example +set +a +QQQ_ALL_COMPOSE_BROKERS=true mvn -pl qqq-all-app -am -Dtest=FullProfileBrokerIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false test +``` + +The test uses the loopback management ports to read and pause/resume the Artemis subscription queue, then creates, reads, purges, and deletes a temporary RabbitMQ queue. For a customized `.env`, source that file instead. It is skipped during ordinary `mvn verify`. diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java index 3e81a93..3845241 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java @@ -161,10 +161,20 @@ final class FullProfileApplication extends AbstractQQQApplication private void addEsb(QInstance instance) { + EsbInstanceMetaData.of(instance) + .getProvider("artemis") + .withUsername(settings.get("ARTEMIS_USER")) + .withPassword(settings.get("ARTEMIS_PASSWORD")) + .withManagementUrl(settings.get("ARTEMIS_MANAGEMENT_URL")) + .withManagementUsername(settings.get("ARTEMIS_USER")) + .withManagementPassword(settings.get("ARTEMIS_PASSWORD")); EsbInstanceMetaData.of(instance) .withProvider(new QEsbProviderMetaData().withName("rabbitmq") .withType(EsbProviderType.RABBITMQ).withUrl(settings.get("RABBITMQ_URL")) - .withUsername(settings.get("RABBITMQ_USER")).withPassword(settings.get("RABBITMQ_PASSWORD"))) + .withUsername(settings.get("RABBITMQ_USER")).withPassword(settings.get("RABBITMQ_PASSWORD")) + .withManagementUrl(settings.get("RABBITMQ_MANAGEMENT_URL")) + .withManagementUsername(settings.get("RABBITMQ_USER")) + .withManagementPassword(settings.get("RABBITMQ_PASSWORD"))) .withDestination(new QEsbDestinationMetaData().withName("orderSyncEvents") .withType(EsbDestinationType.QUEUE).withProviderName("rabbitmq") .withDestinationName("qqq.all.orderSyncEvents")); diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileSettings.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileSettings.java index 9842732..08c9725 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileSettings.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileSettings.java @@ -19,7 +19,8 @@ final class FullProfileSettings "MONGO_HOST", "MONGO_DATABASE", "MONGO_USER", "MONGO_PASSWORD", "MONGO_REPLICA_SET", "S3_ENDPOINT", "S3_BUCKET", "S3_ACCESS_KEY", "S3_SECRET_KEY", "SFTP_HOST", "SFTP_USER", "SFTP_PASSWORD", - "ARTEMIS_URL", "RABBITMQ_URL", "RABBITMQ_USER", "RABBITMQ_PASSWORD", + "ARTEMIS_URL", "ARTEMIS_USER", "ARTEMIS_PASSWORD", "ARTEMIS_MANAGEMENT_URL", + "RABBITMQ_URL", "RABBITMQ_USER", "RABBITMQ_PASSWORD", "RABBITMQ_MANAGEMENT_URL", "SMTP_HOST", "OIDC_BASE_URL", "OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET"); private final Map values; diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileBrokerIntegrationTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileBrokerIntegrationTest.java new file mode 100644 index 0000000..464f7ed --- /dev/null +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileBrokerIntegrationTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 2026 Kingsrook, LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.kingsrook.qqq.all; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Base64; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.esb.management.ArtemisJolokiaAdapter; +import com.kingsrook.qqq.esb.management.EsbBrokerNames; +import com.kingsrook.qqq.esb.management.RabbitManagementAdapter; +import com.kingsrook.qqq.esb.model.EsbInstanceMetaData; +import com.kingsrook.qqq.esb.model.EsbProviderType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Opt-in smoke test against the local full Compose brokers and their management APIs. */ +@EnabledIfEnvironmentVariable(named = "QQQ_ALL_COMPOSE_BROKERS", matches = "true") +class FullProfileBrokerIntegrationTest +{ + @TempDir + Path dataDirectory; + + @Test + void managesQueuesWithFullProfileCredentials() throws Exception + { + Map environment = FullProfileTest.environment(); + environment.put("QQQ_ALL_ARTEMIS_USER", System.getenv("ARTEMIS_USER")); + environment.put("QQQ_ALL_ARTEMIS_PASSWORD", System.getenv("ARTEMIS_PASSWORD")); + environment.put("QQQ_ALL_ARTEMIS_MANAGEMENT_URL", "http://127.0.0.1:8161"); + environment.put("QQQ_ALL_RABBITMQ_USER", System.getenv("RABBITMQ_USER")); + environment.put("QQQ_ALL_RABBITMQ_PASSWORD", System.getenv("RABBITMQ_PASSWORD")); + environment.put("QQQ_ALL_RABBITMQ_MANAGEMENT_URL", "http://127.0.0.1:15672"); + QInstance instance = new FullProfileApplication(dataDirectory, FullProfileSettings.from(environment)).defineQInstance(); + EsbInstanceMetaData esb = EsbInstanceMetaData.of(instance); + + ArtemisJolokiaAdapter artemis = new ArtemisJolokiaAdapter(esb.getProvider("artemis")); + String subscription = EsbBrokerNames.subscriptionQueue(EsbProviderType.ACTIVEMQ_ARTEMIS, + "qqq.all.orderEvents", "syncOrder.qqq.all.orderEvents"); + assertTrue(artemis.getQueueInfo(subscription).isPresent()); + try + { + artemis.pauseQueue(subscription); + assertTrue(artemis.getQueueInfo(subscription).orElseThrow().paused()); + } + finally + { + artemis.resumeQueue(subscription); + } + assertFalse(artemis.getQueueInfo(subscription).orElseThrow().paused()); + + RabbitManagementAdapter rabbit = new RabbitManagementAdapter(esb.getProvider("rabbitmq")); + String queueName = "qqq.all.c6.managementProbe." + UUID.randomUUID(); + String queueUrl = "http://127.0.0.1:15672/api/queues/%2F/" + queueName; + String credentials = environment.get("QQQ_ALL_RABBITMQ_USER") + ":" + environment.get("QQQ_ALL_RABBITMQ_PASSWORD"); + String authorization = "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + try(HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build()) + { + HttpRequest create = HttpRequest.newBuilder(URI.create(queueUrl)) + .header("Authorization", authorization).header("Content-Type", "application/json") + .PUT(HttpRequest.BodyPublishers.ofString("{\"durable\":true,\"auto_delete\":false,\"arguments\":{}}")) + .build(); + assertEquals(201, client.send(create, HttpResponse.BodyHandlers.discarding()).statusCode()); + try + { + assertTrue(rabbit.getQueueInfo(queueName).isPresent()); + assertEquals(0L, rabbit.purgeQueue(queueName)); + } + finally + { + HttpRequest delete = HttpRequest.newBuilder(URI.create(queueUrl)) + .header("Authorization", authorization).DELETE().build(); + assertEquals(204, client.send(delete, HttpResponse.BodyHandlers.discarding()).statusCode()); + } + } + } +} diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java index 1519893..74656d7 100644 --- a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java @@ -56,7 +56,15 @@ class FullProfileTest EsbInstanceMetaData esb = EsbInstanceMetaData.of(instance); assertEquals("tcp://artemis.example.test:61616", esb.getProvider("artemis").getUrl()); + assertEquals("fixture", esb.getProvider("artemis").getUsername()); + assertEquals("fixture", esb.getProvider("artemis").getPassword()); + assertEquals("http://artemis.example.test:8161", esb.getProvider("artemis").getManagementUrl()); + assertEquals("fixture", esb.getProvider("artemis").getManagementUsername()); + assertEquals("fixture", esb.getProvider("artemis").getManagementPassword()); assertEquals("amqp://rabbit.example.test:5672/%2F", esb.getProvider("rabbitmq").getUrl()); + assertEquals("http://rabbit.example.test:15672", esb.getProvider("rabbitmq").getManagementUrl()); + assertEquals("fixture", esb.getProvider("rabbitmq").getManagementUsername()); + assertEquals("fixture", esb.getProvider("rabbitmq").getManagementPassword()); assertEquals("artemis", esb.getDestination("orderEvents").getProviderName()); assertEquals("rabbitmq", esb.getDestination("orderSyncEvents").getProviderName()); @@ -94,7 +102,11 @@ class FullProfileTest values.put("QQQ_ALL_SFTP_USER", "fixture"); values.put("QQQ_ALL_SFTP_PASSWORD", "fixture"); values.put("QQQ_ALL_ARTEMIS_URL", "tcp://artemis.example.test:61616"); + values.put("QQQ_ALL_ARTEMIS_USER", "fixture"); + values.put("QQQ_ALL_ARTEMIS_PASSWORD", "fixture"); + values.put("QQQ_ALL_ARTEMIS_MANAGEMENT_URL", "http://artemis.example.test:8161"); values.put("QQQ_ALL_RABBITMQ_URL", "amqp://rabbit.example.test:5672/%2F"); + values.put("QQQ_ALL_RABBITMQ_MANAGEMENT_URL", "http://rabbit.example.test:15672"); values.put("QQQ_ALL_RABBITMQ_USER", "fixture"); values.put("QQQ_ALL_RABBITMQ_PASSWORD", "fixture"); values.put("QQQ_ALL_SMTP_HOST", "mailpit.example.test"); -- 2.54.0 From 7e2e23a60d4408a387f4887e6feccc3ebe5cf36c Mon Sep 17 00:00:00 2001 From: James Maes Date: Sat, 26 Sep 2026 09:53:16 -0500 Subject: [PATCH 3/5] fix(compose): pass OpenSearch host to full app (#7) --- compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yaml b/compose.yaml index 8cb90d6..1dbf5d6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -59,6 +59,7 @@ services: QQQ_ALL_RABBITMQ_USER: ${RABBITMQ_USER:-qqq} QQQ_ALL_RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD} QQQ_ALL_RABBITMQ_MANAGEMENT_URL: http://rabbitmq:15672 + QQQ_ALL_OPENSEARCH_HOST: opensearch QQQ_ALL_SMTP_HOST: mailpit QQQ_ALL_OIDC_BASE_URL: http://keycloak.localhost:${KEYCLOAK_PORT:-8081}/realms/qqq-all QQQ_ALL_OIDC_CLIENT_ID: qqq-all -- 2.54.0 From 4b396534dea4af668fc07a3d742cacb0322a27e1 Mon Sep 17 00:00:00 2001 From: James Maes Date: Sat, 26 Sep 2026 09:56:36 -0500 Subject: [PATCH 4/5] ci(compose): skip image push until package is public (#7) --- .github/workflows/publish-image.yml | 20 +++++++++++--------- README.md | 2 +- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml index 5bbc218..52ed3d7 100644 --- a/.github/workflows/publish-image.yml +++ b/.github/workflows/publish-image.yml @@ -14,18 +14,18 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Require public package visibility before publishing + - name: Check public package visibility + id: package_visibility env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - visibility="$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility)" || { - echo 'The qqq-all container package must exist and be public before develop images can be published.' >&2 - exit 1 - } - test "$visibility" = public || { - echo 'Set the qqq-all container package visibility to public in GitHub package settings before publishing.' >&2 - exit 1 - } + if visibility="$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility 2>/dev/null)" && [ "$visibility" = public ]; then + echo 'publish=true' >> "$GITHUB_OUTPUT" + echo 'The qqq-all container package is public; develop image publication is enabled.' >> "$GITHUB_STEP_SUMMARY" + else + echo 'publish=false' >> "$GITHUB_OUTPUT" + echo 'Skipped develop image publication: the qqq-all container package is not yet public or cannot be read. An owner must bootstrap the package and set it to Public before C9 enables publication.' >> "$GITHUB_STEP_SUMMARY" + fi - uses: actions/setup-java@v4 with: distribution: temurin @@ -34,11 +34,13 @@ jobs: - name: Verify application run: mvn -B -ntp verify - uses: docker/login-action@v3 + if: steps.package_visibility.outputs.publish == 'true' with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and publish develop image + if: steps.package_visibility.outputs.publish == 'true' run: | docker build \ --label org.opencontainers.image.source=https://github.com/QRun-IO/qqq-all \ diff --git a/README.md b/README.md index 8923e55..ebf6492 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ The full stack initializes the MongoDB replica set, MinIO bucket, sample SQL tab Artemis requires a configured login on a fresh volume. If an older demo volume was created with anonymous login enabled, recreate that volume before relying on the new setting; `docker compose --profile full down --volumes` resets all local demo data. -On merges to `develop`, [the image workflow](.github/workflows/publish-image.yml) checks that the `qqq-all` container package is already public, verifies the reactor, and pushes `ghcr.io/qrun-io/qqq-all:develop`. GitHub creates a new container package private by default, so the first image requires a deliberate, one-time bootstrap: an organization owner publishes an initial image, then changes the package visibility to **public** in GitHub package settings. Until that is done, the workflow fails before any image push. Do not enable automatic develop publication until the package is public. +On merges to `develop`, [the image workflow](.github/workflows/publish-image.yml) verifies the reactor and publishes `ghcr.io/qrun-io/qqq-all:develop` only when the `qqq-all` container package is already public. If the package is absent, private, or cannot be read, image publication is skipped with a job summary while verification still runs. GitHub creates a new container package private by default, so the first image requires a deliberate, one-time bootstrap: an organization owner publishes an initial image, then changes the package visibility to **public** in GitHub package settings. Automatic develop publication starts only after that bootstrap. ## Contributing -- 2.54.0 From 77e72ae0688233666aeed29de2040266b0c81901 Mon Sep 17 00:00:00 2001 From: James Maes Date: Sat, 26 Sep 2026 10:45:18 -0500 Subject: [PATCH 5/5] fix: provision full-profile Postgres qbit schema after C5 merge --- qqq-all-app/README.md | 2 + qqq-all-app/pom.xml | 1 + .../com/kingsrook/qqq/all/DemoQBitData.java | 139 ++++++++++++++++++ .../qqq/all/FullProfileApplication.java | 38 +++++ .../qqq/all/PostgresActionStrategy.java | 39 +++++ .../kingsrook/qqq/all/ProfileLauncher.java | 2 +- .../kingsrook/qqq/all/FullProfileTest.java | 8 +- .../com/kingsrook/qqq/all/FullQBitsTest.java | 14 ++ 8 files changed, 241 insertions(+), 2 deletions(-) create mode 100644 qqq-all-app/src/main/java/com/kingsrook/qqq/all/PostgresActionStrategy.java diff --git a/qqq-all-app/README.md b/qqq-all-app/README.md index 6d1f513..e392191 100644 --- a/qqq-all-app/README.md +++ b/qqq-all-app/README.md @@ -35,6 +35,8 @@ Configure the following environment variables before starting the same jar: The full app adds sample tables on each external backend, `orderEvents` on Artemis, and `orderSyncEvents` on RabbitMQ. Quick search is configured for customer `name` and `email`; run `quickSearchFullReindex` after OpenSearch and the customer data are ready. The SFTP integration adds its import source and staging tables plus `SFTPImportFileSyncProcess`; `externalImportFile` remains the direct SFTP file sample. Keycloak realm roles resolve by name against the PostgreSQL `role` table from `qbit-user-role-permissions`, then become QQQ session permissions. Unknown roles receive no permissions. Webhook tables and delivery processes require explicit permissions: the viewer role receives read access, while the admin role can edit destinations and run delivery. The Compose profile provisions demo SQL tables, OAuth session and redirect-state tables, qbit permission tables and roles, and the MinIO bucket on first startup; `infra/postgres/03-webhook-permissions.sql` adds webhook grants after the role seed. +On startup, the full demo creates its remaining PostgreSQL qbit tables from the registered metadata and checks every expected column and type. The hand-maintained domain, OAuth, and role tables stay under `infra/postgres`. If a persisted qbit table is missing a required column or has a different type, startup fails with a migration/reset message instead of silently accepting it. The full-profile RabbitMQ completion-event smoke remains gated on QQQ ESB Task 9's metadata-driven process lifecycle publication; the broker management smoke below can run independently. + After starting the full Compose profile, run the opt-in broker management smoke test with the same broker credentials. For the supplied demo environment: ```sh diff --git a/qqq-all-app/pom.xml b/qqq-all-app/pom.xml index 103568f..79aa9d3 100644 --- a/qqq-all-app/pom.xml +++ b/qqq-all-app/pom.xml @@ -51,6 +51,7 @@ com.h2databaseh2 org.xerialsqlite-jdbc org.apache.artemisartemis-jakarta-server2.57.0 + com.rabbitmq.jmsrabbitmq-jms org.junit.jupiterjunit-jupiter6.0.2test diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBitData.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBitData.java index db52bc9..695b2df 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBitData.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBitData.java @@ -9,10 +9,17 @@ package com.kingsrook.qqq.all; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -22,6 +29,11 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDe /** Creates repeatable local tables for the qbit metadata exposed by core. */ final class DemoQBitData { + /** These tables have hand-maintained PostgreSQL DDL in infra/postgres/01 and 02. */ + private static final Set POSTGRES_MANAGED_TABLES = Set.of("warehouse_customer", "user_session", + "redirect_state", "user", "role", "permission", "role_permission_int", "user_role_int", + "user_permission_int"); + private DemoQBitData() { } @@ -86,6 +98,133 @@ final class DemoQBitData } } + static void ensurePostgresSchema(QInstance instance) throws SQLException + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) instance.getBackend("postgres"); + try(Connection connection = DriverManager.getConnection(ConnectionManager.getJdbcUrl(backend), + backend.getUsername(), backend.getPassword()); + Statement statement = connection.createStatement()) + { + for(String sql : postgresSchemaStatements(instance)) + { + statement.execute(sql); + } + for(QTableMetaData table : postgresQBitTables(instance)) + { + assertPostgresColumns(connection, table); + } + } + } + + static List postgresSchemaStatements(QInstance instance) + { + List statements = new ArrayList<>(); + for(QTableMetaData table : postgresQBitTables(instance)) + { + String tableName = postgresTableName(table); + List columns = new ArrayList<>(); + for(QFieldMetaData field : table.getFields().values()) + { + String name = field.getBackendName() == null ? field.getName() : field.getBackendName(); + String identity = field.getName().equals(table.getPrimaryKeyField()) + && (field.getType() == QFieldType.INTEGER || field.getType() == QFieldType.LONG) + ? " GENERATED BY DEFAULT AS IDENTITY" : ""; + String primaryKey = field.getName().equals(table.getPrimaryKeyField()) ? " PRIMARY KEY" : ""; + columns.add(postgresIdentifier(name) + " " + postgresSqlType(field.getType()) + identity + primaryKey); + } + statements.add("CREATE TABLE IF NOT EXISTS " + postgresIdentifier(tableName) + + " (" + String.join(", ", columns) + ")"); + } + return statements; + } + + private static List postgresQBitTables(QInstance instance) + { + return instance.getTables().values().stream() + .filter(table -> "postgres".equals(table.getBackendName()) && !table.getFields().isEmpty()) + .filter(table -> !POSTGRES_MANAGED_TABLES.contains(postgresTableName(table))) + .toList(); + } + + private static String postgresTableName(QTableMetaData table) + { + return table.getBackendDetails() instanceof RDBMSTableBackendDetails details + ? details.getTableName() : table.getName(); + } + + private static void assertPostgresColumns(Connection connection, QTableMetaData table) throws SQLException + { + String tableName = postgresTableName(table); + Map actual = new HashMap<>(); + try(PreparedStatement query = connection.prepareStatement("SELECT column_name, data_type " + + "FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ?")) + { + query.setString(1, tableName); + try(ResultSet rows = query.executeQuery()) + { + while(rows.next()) + { + actual.put(rows.getString(1), rows.getString(2)); + } + } + } + for(QFieldMetaData field : table.getFields().values()) + { + String columnName = field.getBackendName() == null ? field.getName() : field.getBackendName(); + String expectedType = postgresInformationSchemaType(field.getType()); + String actualType = actual.get(columnName); + if(!expectedType.equals(actualType)) + { + throw new SQLException("Full-profile table " + tableName + " column " + columnName + + " must be " + expectedType + "; found " + (actualType == null ? "missing" : actualType) + + ". Recreate the demo volume or migrate the persisted schema."); + } + } + } + + private static String postgresInformationSchemaType(QFieldType type) + { + return switch(type) + { + case INTEGER -> "integer"; + case LONG -> "bigint"; + case DECIMAL -> "numeric"; + case BOOLEAN -> "boolean"; + case DATE -> "date"; + case TIME -> "time without time zone"; + case DATE_TIME -> "timestamp without time zone"; + case BLOB -> "bytea"; + case STRING, PASSWORD -> "character varying"; + case TEXT, HTML -> "text"; + }; + } + + private static String postgresIdentifier(String value) + { + if(value == null || !value.matches("[A-Za-z][A-Za-z0-9_]*")) + { + throw new IllegalArgumentException("Unsupported demo schema identifier"); + } + return "\"" + value + "\""; + } + + private static String postgresSqlType(QFieldType type) + { + return switch(type) + { + case INTEGER -> "INTEGER"; + case LONG -> "BIGINT"; + case DECIMAL -> "DECIMAL(20,6)"; + case BOOLEAN -> "BOOLEAN"; + case DATE -> "DATE"; + case TIME -> "TIME"; + case DATE_TIME -> "TIMESTAMP"; + case BLOB -> "BYTEA"; + case STRING, PASSWORD -> "VARCHAR(4000)"; + case TEXT, HTML -> "TEXT"; + }; + } + private static String identifier(String value) { if(value == null || !value.matches("[A-Za-z][A-Za-z0-9_]*")) diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java index 2b2cda4..65b6659 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java @@ -7,6 +7,7 @@ package com.kingsrook.qqq.all; import java.nio.file.Path; +import java.sql.SQLException; import java.util.List; import com.kingsrook.qbits.userrolepermissions.UserRolePermissionsQBitConfig; import com.kingsrook.qbits.userrolepermissions.UserRolePermissionsQBitProducer; @@ -48,11 +49,18 @@ final class FullProfileApplication extends AbstractQQQApplication { private final Path dataDirectory; private final FullProfileSettings settings; + private final boolean provisionSchema; FullProfileApplication(Path dataDirectory, FullProfileSettings settings) + { + this(dataDirectory, settings, false); + } + + FullProfileApplication(Path dataDirectory, FullProfileSettings settings, boolean provisionSchema) { this.dataDirectory = dataDirectory; this.settings = settings; + this.provisionSchema = provisionSchema; } @Override @@ -65,13 +73,39 @@ final class FullProfileApplication extends AbstractQQQApplication addEsb(instance); DemoQBits.addCore(instance, "postgres"); DemoQBits.addFull(instance, settings); + // OIDC and ESB sessions use string user IDs, unlike the qbit's default integer user ID. + instance.getTable("processTrace").getField("userId").setType(QFieldType.STRING); DemoUsers.protectWebhookResources(instance); + for(QTableMetaData table : instance.getTables().values()) + { + if(!"postgres".equals(table.getBackendName())) + { + continue; + } + if(table.getBackendDetails() == null) + { + table.setBackendDetails(new RDBMSTableBackendDetails() + .withTableName(QInstanceEnricher.inferBackendName(table.getName()))); + } + QInstanceEnricher.setInferredFieldBackendNames(table); + } EmailMessagingProviderMetaData mail = new EmailMessagingProviderMetaData() .withSmtpServer(settings.get("SMTP_HOST")) .withSmtpPort(settings.getOrDefault("SMTP_PORT", "1025")); mail.setName("mailpit"); instance.addMessagingProvider(mail); + if(provisionSchema) + { + try + { + DemoQBitData.ensurePostgresSchema(instance); + } + catch(SQLException e) + { + throw new QException("Could not provision full-profile qbit tables", e); + } + } return instance; } @@ -80,6 +114,10 @@ final class FullProfileApplication extends AbstractQQQApplication instance.addBackend(new RDBMSBackendMetaData().withName("postgres").withVendor("postgres") .withHostName(settings.get("POSTGRES_HOST")).withPort(settings.port("POSTGRES_PORT", "5432")) .withDatabaseName(settings.get("POSTGRES_DATABASE")) + .withJdbcUrl("jdbc:postgresql://" + settings.get("POSTGRES_HOST") + ":" + + settings.port("POSTGRES_PORT", "5432") + "/" + settings.get("POSTGRES_DATABASE")) + .withJdbcDriverClassName("org.postgresql.Driver") + .withActionStrategyCodeReference(new QCodeReference(PostgresActionStrategy.class)) .withUsername(settings.get("POSTGRES_USER")).withPassword(settings.get("POSTGRES_PASSWORD"))); instance.addBackend(new RDBMSBackendMetaData().withName("mysql").withVendor("mysql") .withHostName(settings.get("MYSQL_HOST")).withPort(settings.port("MYSQL_PORT", "3306")) diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/PostgresActionStrategy.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/PostgresActionStrategy.java new file mode 100644 index 0000000..c9bce8c --- /dev/null +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/PostgresActionStrategy.java @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Kingsrook, LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.kingsrook.qqq.all; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; + +/** PostgreSQL syntax for the shared QQQ RDBMS actions. */ +public class PostgresActionStrategy extends BaseRDBMSActionStrategy +{ + @Override + public String getIdentifierQuoteString() + { + return "\""; + } + + @Override + public String getInsertDefaultValuesClause() + { + return "DEFAULT VALUES"; + } + + @Override + protected int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.NULL); + return 1; + } + return super.bindParamObject(statement, index, value); + } +} diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java index ecd2d7b..ee8b22c 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java @@ -46,7 +46,7 @@ public final class ProfileLauncher FullProfileSettings settings = FullProfileSettings.from(environment); CoreData.seed(dataDirectory); - QApplicationLauncher.run(new FullProfileApplication(dataDirectory, settings), + QApplicationLauncher.run(new FullProfileApplication(dataDirectory, settings, true), new QApplicationLauncherConfig().withRegisterShutdownHook(true) .withServerCustomizer(server -> server.withPort(port) .withJavalinConfigCustomizer(config -> config.jetty.host = settings.get("BIND_HOST")))); diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java index 949175b..f45b85b 100644 --- a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java @@ -15,6 +15,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.authentication.OAuth2Authen import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.messaging.email.EmailMessagingProviderMetaData; import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.qqq.esb.model.EsbInstanceMetaData; import org.junit.jupiter.api.Test; @@ -43,7 +44,12 @@ class FullProfileTest QInstance instance = new FullProfileApplication(dataDirectory, settings).defineQInstance(); new QInstanceValidator().validate(instance); - assertEquals("postgres.example.test", ((RDBMSBackendMetaData) instance.getBackend("postgres")).getHostName()); + RDBMSBackendMetaData postgres = (RDBMSBackendMetaData) instance.getBackend("postgres"); + assertEquals("postgres.example.test", postgres.getHostName()); + assertEquals("jdbc:postgresql://postgres.example.test:5432/qqq", ConnectionManager.getJdbcUrl(postgres)); + assertEquals("org.postgresql.Driver", ConnectionManager.getJdbcDriverClassName(postgres)); + assertEquals("\"", postgres.getActionStrategy().getIdentifierQuoteString()); + assertEquals("DEFAULT VALUES", postgres.getActionStrategy().getInsertDefaultValuesClause()); assertEquals("mysql.example.test", ((RDBMSBackendMetaData) instance.getBackend("mysql")).getHostName()); assertEquals("replicaSet=rs0", ((MongoDBBackendMetaData) instance.getBackend("mongo")).getUrlSuffix()); assertEquals("postgres", instance.getTable("warehouseCustomer").getBackendName()); diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java index 1460427..8b14d3b 100644 --- a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java @@ -18,6 +18,8 @@ import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; import com.kingsrook.qqq.backend.core.model.session.QSession; import org.junit.jupiter.api.Test; @@ -50,6 +52,18 @@ class FullQBitsTest assertNotNull(instance.getProcess("SFTPImportFileSyncProcess")); assertNotNull(instance.getTable("SFTPImportSourceFileTable")); assertEquals("sftp", instance.getTable("SFTPImportSourceFileTable").getBackendName()); + assertEquals("scheduled_job", ((RDBMSTableBackendDetails) instance.getTable("scheduledJob") + .getBackendDetails()).getTableName()); + assertEquals("create_date", instance.getTable("scheduledJob").getField("createDate").getBackendName()); + assertEquals(QFieldType.STRING, instance.getTable("processTrace").getField("userId").getType()); + assertTrue(DemoQBitData.postgresSchemaStatements(instance).stream() + .anyMatch(sql -> sql.startsWith("CREATE TABLE IF NOT EXISTS \"scheduled_job\"") + && sql.contains("\"create_date\" TIMESTAMP"))); + assertTrue(DemoQBitData.postgresSchemaStatements(instance).stream() + .anyMatch(sql -> sql.startsWith("CREATE TABLE IF NOT EXISTS \"webhook_subscription\"") + && sql.contains("\"webhook_event_type_name\" VARCHAR(4000)"))); + assertFalse(DemoQBitData.postgresSchemaStatements(instance).stream() + .anyMatch(sql -> sql.contains("\"user_session\"") || sql.contains("\"warehouse_customer\""))); for(String tableName : Set.of("webhook", "webhookSubscription", "webhookEvent", "webhookEventContent", "webhookEventSendLog", "scheduledJob", "scheduledJobParameter")) { -- 2.54.0