diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..06a47b5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.git +.github +.circleci +.env +docs +qqq-all-bom +qqq-all-bom-test +qqq-all-app/src +qqq-all-app/target/* +!qqq-all-app/target/qqq-all-app.jar diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b4f9c06 --- /dev/null +++ b/.env.example @@ -0,0 +1,29 @@ +# Local demonstration values only. Copy to .env and override before exposing ports. +QQQ_ALL_PORT=8080 +KEYCLOAK_PORT=8081 +POSTGRES_USER=qqq +POSTGRES_PASSWORD=demo-postgres-password +POSTGRES_DATABASE=qqq +MYSQL_USER=qqq +MYSQL_PASSWORD=demo-mysql-password +MYSQL_ROOT_PASSWORD=demo-mysql-root-password +MYSQL_DATABASE=qqq +MONGO_USER=qqq +MONGO_PASSWORD=demo-mongo-password +MONGO_DATABASE=qqq +MONGO_REPLICA_SET=rs0 +MONGO_KEYFILE=demo-mongo-internal-replica-key-change-this-before-sharing +MINIO_ROOT_USER=qqq +MINIO_ROOT_PASSWORD=demo-minio-password +MINIO_BUCKET=qqq-demo +SFTP_USER=qqq +SFTP_PASSWORD=demo-sftp-password +RABBITMQ_USER=qqq +RABBITMQ_PASSWORD=demo-rabbitmq-password +ARTEMIS_USER=qqq +ARTEMIS_PASSWORD=demo-artemis-password +KEYCLOAK_ADMIN=admin +KEYCLOAK_ADMIN_PASSWORD=demo-keycloak-admin-password +OIDC_CLIENT_SECRET=demo-qqq-all-client-secret +DEMO_ADMIN_PASSWORD=demo-admin-password +DEMO_USER_PASSWORD=demo-user-password diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 0000000..52ed3d7 --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,48 @@ +name: Publish develop image + +on: + push: + branches: [develop] + workflow_dispatch: + +permissions: + contents: read + packages: write + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check public package visibility + id: package_visibility + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if visibility="$(gh api /orgs/QRun-IO/packages/container/qqq-all --jq .visibility 2>/dev/null)" && [ "$visibility" = public ]; then + echo 'publish=true' >> "$GITHUB_OUTPUT" + echo 'The qqq-all container package is public; develop image publication is enabled.' >> "$GITHUB_STEP_SUMMARY" + else + echo 'publish=false' >> "$GITHUB_OUTPUT" + echo 'Skipped develop image publication: the qqq-all container package is not yet public or cannot be read. An owner must bootstrap the package and set it to Public before C9 enables publication.' >> "$GITHUB_STEP_SUMMARY" + fi + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + cache: maven + - name: Verify application + run: mvn -B -ntp verify + - uses: docker/login-action@v3 + if: steps.package_visibility.outputs.publish == 'true' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish develop image + if: steps.package_visibility.outputs.publish == 'true' + run: | + docker build \ + --label org.opencontainers.image.source=https://github.com/QRun-IO/qqq-all \ + --tag ghcr.io/qrun-io/qqq-all:develop . + docker push ghcr.io/qrun-io/qqq-all:develop diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8c29f94 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM eclipse-temurin:21-jre-jammy + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system --gid 10001 qqq \ + && useradd --system --uid 10001 --gid qqq --home-dir /app qqq \ + && mkdir -p /app/data \ + && chown -R qqq:qqq /app + +WORKDIR /app +COPY --chown=qqq:qqq qqq-all-app/target/qqq-all-app.jar /app/qqq-all.jar +USER 10001:10001 +EXPOSE 8080 +ENV QQQ_ALL_DATA_DIR=/app/data +HEALTHCHECK --interval=10s --timeout=5s --start-period=45s --retries=6 \ + CMD curl --fail --silent http://127.0.0.1:8080/health >/dev/null || exit 1 +ENTRYPOINT ["java", "-jar", "/app/qqq-all.jar"] diff --git a/README.md b/README.md index 72e899f..ebf6492 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,31 @@ Requires Java 21 and Maven. mvn -B verify ``` +## Run locally + +The core profile needs only Java 21: + +```bash +java -jar qqq-all-app/target/qqq-all-app.jar +``` + +For containers, copy the demo settings and build the jar before starting either Compose profile: + +```bash +cp .env.example .env +mvn -B verify +docker compose --profile core up --build --wait +# Or: docker compose --profile full up --build --wait +``` + +The app is at `http://127.0.0.1:8080/health` (change `QQQ_ALL_PORT` in `.env` if needed). The full profile also publishes local-only management ports for Keycloak (`8081`), Mailpit (`8025`), MinIO (`9001`), Artemis/Jolokia (`8161`), and RabbitMQ (`15672`). The demo Keycloak users are `demo-admin` and `demo-user`; their passwords and the confidential client secret come from `.env`. The sample values in `.env.example` are for local demonstrations only. The default browser-visible issuer is `http://keycloak.localhost:8081`; if your host does not resolve `keycloak.localhost` to loopback, add that local hosts entry before trying OIDC login. + +The full stack initializes the MongoDB replica set, MinIO bucket, sample SQL tables, OAuth2 session tables, and demo roles/permissions. PostgreSQL and MySQL initialization scripts run only when their data volumes are first created. To stop, run `docker compose --profile full down`; adding `--volumes` removes demo data. See [the app guide](qqq-all-app/README.md) for the full-profile environment contract. + +Artemis requires a configured login on a fresh volume. If an older demo volume was created with anonymous login enabled, recreate that volume before relying on the new setting; `docker compose --profile full down --volumes` resets all local demo data. + +On merges to `develop`, [the image workflow](.github/workflows/publish-image.yml) verifies the reactor and publishes `ghcr.io/qrun-io/qqq-all:develop` only when the `qqq-all` container package is already public. If the package is absent, private, or cannot be read, image publication is skipped with a job summary while verification still runs. GitHub creates a new container package private by default, so the first image requires a deliberate, one-time bootstrap: an organization owner publishes an initial image, then changes the package visibility to **public** in GitHub package settings. Automatic develop publication starts only after that bootstrap. + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md), the [Code of Conduct](CODE_OF_CONDUCT.md), and the [security policy](SECURITY.md). diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..1dbf5d6 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,290 @@ +name: qqq-all + +x-app-image: &app-image + build: + context: . + image: qqq-all:local + volumes: + - app-data:/app/data + ports: + - "127.0.0.1:${QQQ_ALL_PORT:-8080}:8080" + healthcheck: + test: ["CMD", "curl", "--fail", "--silent", "http://127.0.0.1:8080/health"] + interval: 10s + timeout: 5s + retries: 6 + start_period: 45s + +services: + app-core: + <<: *app-image + profiles: [core] + environment: + QQQ_ALL_PROFILE: core + QQQ_ALL_PORT: "8080" + QQQ_ALL_BIND_HOST: 0.0.0.0 + + app-full: + <<: *app-image + profiles: [full] + environment: + QQQ_ALL_PROFILE: full + QQQ_ALL_PORT: "8080" + QQQ_ALL_BIND_HOST: 0.0.0.0 + QQQ_ALL_POSTGRES_HOST: postgres + QQQ_ALL_POSTGRES_DATABASE: ${POSTGRES_DATABASE:-qqq} + QQQ_ALL_POSTGRES_USER: ${POSTGRES_USER:-qqq} + QQQ_ALL_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD} + QQQ_ALL_MYSQL_HOST: mysql + QQQ_ALL_MYSQL_DATABASE: ${MYSQL_DATABASE:-qqq} + QQQ_ALL_MYSQL_USER: ${MYSQL_USER:-qqq} + QQQ_ALL_MYSQL_PASSWORD: ${MYSQL_PASSWORD:?set MYSQL_PASSWORD} + QQQ_ALL_MONGO_HOST: mongo + QQQ_ALL_MONGO_DATABASE: ${MONGO_DATABASE:-qqq} + QQQ_ALL_MONGO_USER: ${MONGO_USER:-qqq} + QQQ_ALL_MONGO_PASSWORD: ${MONGO_PASSWORD:?set MONGO_PASSWORD} + QQQ_ALL_MONGO_REPLICA_SET: ${MONGO_REPLICA_SET:-rs0} + QQQ_ALL_S3_ENDPOINT: http://minio:9000 + QQQ_ALL_S3_BUCKET: ${MINIO_BUCKET:-qqq-demo} + QQQ_ALL_S3_ACCESS_KEY: ${MINIO_ROOT_USER:-qqq} + QQQ_ALL_S3_SECRET_KEY: ${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD} + QQQ_ALL_SFTP_HOST: sftp + QQQ_ALL_SFTP_USER: ${SFTP_USER:-qqq} + QQQ_ALL_SFTP_PASSWORD: ${SFTP_PASSWORD:?set SFTP_PASSWORD} + QQQ_ALL_ARTEMIS_URL: tcp://artemis:61616 + QQQ_ALL_ARTEMIS_USER: ${ARTEMIS_USER:-qqq} + QQQ_ALL_ARTEMIS_PASSWORD: ${ARTEMIS_PASSWORD:?set ARTEMIS_PASSWORD} + QQQ_ALL_ARTEMIS_MANAGEMENT_URL: http://artemis:8161 + QQQ_ALL_RABBITMQ_URL: amqp://rabbitmq:5672/%2F + QQQ_ALL_RABBITMQ_USER: ${RABBITMQ_USER:-qqq} + QQQ_ALL_RABBITMQ_PASSWORD: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD} + QQQ_ALL_RABBITMQ_MANAGEMENT_URL: http://rabbitmq:15672 + QQQ_ALL_OPENSEARCH_HOST: opensearch + QQQ_ALL_SMTP_HOST: mailpit + QQQ_ALL_OIDC_BASE_URL: http://keycloak.localhost:${KEYCLOAK_PORT:-8081}/realms/qqq-all + QQQ_ALL_OIDC_CLIENT_ID: qqq-all + QQQ_ALL_OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:?set OIDC_CLIENT_SECRET} + depends_on: + postgres: {condition: service_healthy} + mysql: {condition: service_healthy} + mongo-init: {condition: service_completed_successfully} + minio-init: {condition: service_completed_successfully} + sftp: {condition: service_healthy} + opensearch: {condition: service_healthy} + artemis: {condition: service_healthy} + rabbitmq: {condition: service_healthy} + mailpit: {condition: service_healthy} + keycloak: {condition: service_healthy} + + postgres: + image: postgres:16-alpine + profiles: [full] + environment: + POSTGRES_DB: ${POSTGRES_DATABASE:-qqq} + POSTGRES_USER: ${POSTGRES_USER:-qqq} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD} + volumes: + - postgres-data:/var/lib/postgresql/data + - ./infra/postgres:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 5s + timeout: 5s + retries: 20 + + mysql: + image: mysql:8.4 + profiles: [full] + environment: + MYSQL_DATABASE: ${MYSQL_DATABASE:-qqq} + MYSQL_USER: ${MYSQL_USER:-qqq} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?set MYSQL_PASSWORD} + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?set MYSQL_ROOT_PASSWORD} + volumes: + - mysql-data:/var/lib/mysql + - ./infra/mysql:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -u root -p\"$$MYSQL_ROOT_PASSWORD\" --silent"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 20s + + mongo: + image: mongo:7 + profiles: [full] + entrypoint: ["bash", "/start.sh"] + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGO_USER:-qqq} + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_PASSWORD:?set MONGO_PASSWORD} + MONGO_REPLICA_SET: ${MONGO_REPLICA_SET:-rs0} + MONGO_KEYFILE: ${MONGO_KEYFILE:?set MONGO_KEYFILE} + volumes: + - mongo-data:/data/db + - mongo-config:/data/configdb + - ./infra/mongo/start.sh:/start.sh:ro + healthcheck: + test: ["CMD-SHELL", "mongosh --quiet -u \"$$MONGO_INITDB_ROOT_USERNAME\" -p \"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --eval 'db.adminCommand({ping:1}).ok' | grep -q 1"] + interval: 5s + timeout: 5s + retries: 20 + + mongo-init: + image: mongo:7 + profiles: [full] + depends_on: + mongo: {condition: service_healthy} + environment: + MONGO_REPLICA_SET: ${MONGO_REPLICA_SET:-rs0} + MONGO_USER: ${MONGO_USER:-qqq} + MONGO_PASSWORD: ${MONGO_PASSWORD:?set MONGO_PASSWORD} + volumes: + - ./infra/mongo/init.sh:/init.sh:ro + entrypoint: ["bash", "/init.sh"] + restart: "no" + + minio: + build: + context: ./infra/minio + image: qqq-all-minio:local + profiles: [full] + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-qqq} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD} + volumes: + - minio-data:/data + ports: + - "127.0.0.1:9001:9001" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/live"] + interval: 5s + timeout: 5s + retries: 20 + + minio-init: + image: public.ecr.aws/aws-cli/aws-cli:2.27.41 + profiles: [full] + depends_on: + minio: {condition: service_healthy} + environment: + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-qqq} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:?set MINIO_ROOT_PASSWORD} + AWS_DEFAULT_REGION: us-east-1 + MINIO_BUCKET: ${MINIO_BUCKET:-qqq-demo} + entrypoint: ["/bin/sh", "-c"] + command: ["aws --endpoint-url http://minio:9000 s3api head-bucket --bucket \"$$MINIO_BUCKET\" >/dev/null 2>&1 || aws --endpoint-url http://minio:9000 s3api create-bucket --bucket \"$$MINIO_BUCKET\""] + restart: "no" + + sftp: + image: atmoz/sftp:alpine + profiles: [full] + command: "${SFTP_USER:-qqq}:${SFTP_PASSWORD:?set SFTP_PASSWORD}:1001:1001:upload" + volumes: + - sftp-data:/home/${SFTP_USER:-qqq}/upload + healthcheck: + test: ["CMD-SHELL", "pgrep sshd >/dev/null"] + interval: 5s + timeout: 5s + retries: 20 + + opensearch: + image: opensearchproject/opensearch:2.19.1 + profiles: [full] + environment: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: "true" + OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m + volumes: + - opensearch-data:/usr/share/opensearch/data + healthcheck: + test: ["CMD-SHELL", "curl --fail --silent http://127.0.0.1:9200/_cluster/health >/dev/null"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 30s + + artemis: + image: apache/activemq-artemis:2.42.0-alpine + profiles: [full] + environment: + ARTEMIS_USER: ${ARTEMIS_USER:-qqq} + ARTEMIS_PASSWORD: ${ARTEMIS_PASSWORD:?set ARTEMIS_PASSWORD} + ANONYMOUS_LOGIN: "false" + volumes: + - artemis-data:/var/lib/artemis-instance + ports: + - "127.0.0.1:8161:8161" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8161/console/"] + interval: 10s + timeout: 5s + retries: 20 + + rabbitmq: + image: rabbitmq:4-management-alpine + profiles: [full] + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-qqq} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:?set RABBITMQ_PASSWORD} + volumes: + - rabbitmq-data:/var/lib/rabbitmq + ports: + - "127.0.0.1:15672:15672" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 20 + + mailpit: + image: axllent/mailpit:v1.22.3 + profiles: [full] + ports: + - "127.0.0.1:8025:8025" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8025/api/v1/info"] + interval: 5s + timeout: 5s + retries: 20 + + keycloak: + image: quay.io/keycloak/keycloak:26.2.5 + profiles: [full] + networks: + default: + aliases: [keycloak.localhost] + command: ["start-dev", "--import-realm"] + environment: + KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN:-admin} + KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD:?set KEYCLOAK_ADMIN_PASSWORD} + KC_HEALTH_ENABLED: "true" + KC_HOSTNAME: http://keycloak.localhost:${KEYCLOAK_PORT:-8081} + KC_HTTP_ENABLED: "true" + KC_HTTP_PORT: ${KEYCLOAK_PORT:-8081} + OIDC_CLIENT_SECRET: ${OIDC_CLIENT_SECRET:?set OIDC_CLIENT_SECRET} + DEMO_ADMIN_PASSWORD: ${DEMO_ADMIN_PASSWORD:?set DEMO_ADMIN_PASSWORD} + DEMO_USER_PASSWORD: ${DEMO_USER_PASSWORD:?set DEMO_USER_PASSWORD} + QQQ_ALL_PORT: ${QQQ_ALL_PORT:-8080} + volumes: + - ./infra/keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro + ports: + - "127.0.0.1:${KEYCLOAK_PORT:-8081}:${KEYCLOAK_PORT:-8081}" + healthcheck: + test: ["CMD", "bash", "-c", "{ printf 'HEAD /health/ready HTTP/1.0\r\n\r\n' >&0; grep 'HTTP/1.0 200'; } 0<>/dev/tcp/127.0.0.1/9000"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 30s + +volumes: + app-data: + postgres-data: + mysql-data: + mongo-data: + mongo-config: + minio-data: + sftp-data: + opensearch-data: + artemis-data: + rabbitmq-data: diff --git a/infra/keycloak/realm.json b/infra/keycloak/realm.json new file mode 100644 index 0000000..807375c --- /dev/null +++ b/infra/keycloak/realm.json @@ -0,0 +1,51 @@ +{ + "realm": "qqq-all", + "enabled": true, + "displayName": "QQQ All Demo", + "registrationAllowed": false, + "loginWithEmailAllowed": true, + "roles": { + "realm": [ + {"name": "admin", "description": "Full demo permissions"}, + {"name": "viewer", "description": "Read-only demo permissions"} + ] + }, + "clients": [ + { + "clientId": "qqq-all", + "name": "QQQ All Demo", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "clientAuthenticatorType": "client-secret", + "secret": "${OIDC_CLIENT_SECRET}", + "standardFlowEnabled": true, + "directAccessGrantsEnabled": false, + "redirectUris": ["http://localhost:${QQQ_ALL_PORT}/*", "http://127.0.0.1:${QQQ_ALL_PORT}/*"], + "webOrigins": ["http://localhost:${QQQ_ALL_PORT}", "http://127.0.0.1:${QQQ_ALL_PORT}"], + "defaultClientScopes": ["web-origins", "acr", "profile", "roles", "email"] + } + ], + "users": [ + { + "username": "demo-admin", + "email": "admin@example.test", + "firstName": "Demo", + "lastName": "Admin", + "enabled": true, + "emailVerified": true, + "realmRoles": ["admin"], + "credentials": [{"type": "password", "value": "${DEMO_ADMIN_PASSWORD}", "temporary": false}] + }, + { + "username": "demo-user", + "email": "user@example.test", + "firstName": "Demo", + "lastName": "User", + "enabled": true, + "emailVerified": true, + "realmRoles": ["viewer"], + "credentials": [{"type": "password", "value": "${DEMO_USER_PASSWORD}", "temporary": false}] + } + ] +} diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile new file mode 100644 index 0000000..743eff4 --- /dev/null +++ b/infra/minio/Dockerfile @@ -0,0 +1,16 @@ +FROM golang:1.24-alpine AS build +RUN apk add --no-cache git +RUN git clone --depth 1 --branch RELEASE.2025-09-06T17-38-46Z https://github.com/minio/minio.git /src +WORKDIR /src +RUN CGO_ENABLED=0 go build -trimpath -o /minio . + +FROM alpine:3.20 +RUN apk add --no-cache ca-certificates \ + && addgroup -S -g 10001 minio \ + && adduser -S -u 10001 -G minio minio \ + && mkdir /data \ + && chown minio:minio /data +COPY --from=build /minio /usr/local/bin/minio +USER 10001:10001 +EXPOSE 9000 9001 +ENTRYPOINT ["minio"] diff --git a/infra/mongo/init.sh b/infra/mongo/init.sh new file mode 100644 index 0000000..f04753c --- /dev/null +++ b/infra/mongo/init.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +for attempt in {1..30}; do + if mongosh --host mongo --quiet -u "${MONGO_USER}" -p "${MONGO_PASSWORD}" --authenticationDatabase admin \ + --eval 'try { rs.status().ok } catch (e) { if (e.codeName === "NotYetInitialized") { rs.initiate({_id: process.env.MONGO_REPLICA_SET, members: [{_id: 0, host: "mongo:27017"}]}).ok } else { throw e } }' | grep -q 1; then + exit 0 + fi + sleep 2 +done +echo 'Mongo replica set did not initialize' >&2 +exit 1 diff --git a/infra/mongo/start.sh b/infra/mongo/start.sh new file mode 100644 index 0000000..c3385ab --- /dev/null +++ b/infra/mongo/start.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +printf '%s' "${MONGO_KEYFILE}" | base64 > /data/configdb/keyfile +chmod 400 /data/configdb/keyfile +chown mongodb:mongodb /data/configdb/keyfile +exec docker-entrypoint.sh mongod --replSet "${MONGO_REPLICA_SET}" --bind_ip_all --auth --keyFile /data/configdb/keyfile diff --git a/infra/mysql/01-domain.sql b/infra/mysql/01-domain.sql new file mode 100644 index 0000000..6b789c6 --- /dev/null +++ b/infra/mysql/01-domain.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS supplier_order ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + reference VARCHAR(120) NOT NULL +); + +INSERT IGNORE INTO supplier_order (id, reference) VALUES (1, 'DEMO-SUPPLIER-001'); diff --git a/infra/postgres/01-domain.sql b/infra/postgres/01-domain.sql new file mode 100644 index 0000000..1fbbafa --- /dev/null +++ b/infra/postgres/01-domain.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS warehouse_customer ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name VARCHAR(120) NOT NULL +); + +INSERT INTO warehouse_customer (id, name) VALUES (1, 'Demo Warehouse Customer') +ON CONFLICT (id) DO NOTHING; diff --git a/infra/postgres/02-auth-and-roles.sql b/infra/postgres/02-auth-and-roles.sql new file mode 100644 index 0000000..8469f9b --- /dev/null +++ b/infra/postgres/02-auth-and-roles.sql @@ -0,0 +1,90 @@ +-- C4's OAuth2 session tables and qbit-user-role-permissions tables. +-- Names follow QInstanceEnricher's camelCase-to-snake_case backend mapping. +CREATE TABLE IF NOT EXISTS user_session ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + uuid VARCHAR(40) UNIQUE, + access_token TEXT, + user_id VARCHAR(100) +); + +CREATE TABLE IF NOT EXISTS redirect_state ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + state VARCHAR(45) UNIQUE, + redirect_uri VARCHAR(4096), + create_date TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS "user" ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + email VARCHAR(250), + full_name VARCHAR(250) +); + +CREATE TABLE IF NOT EXISTS role ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + name VARCHAR(100) NOT NULL UNIQUE, + description VARCHAR(500) +); + +CREATE TABLE IF NOT EXISTS permission ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + name VARCHAR(250) NOT NULL UNIQUE, + description VARCHAR(500), + object_type VARCHAR(50), + object_label VARCHAR(250) +); + +CREATE TABLE IF NOT EXISTS role_permission_int ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + role_id INTEGER NOT NULL REFERENCES role(id), + permission_id INTEGER NOT NULL REFERENCES permission(id), + UNIQUE (role_id, permission_id) +); + +CREATE TABLE IF NOT EXISTS user_role_int ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + user_id INTEGER NOT NULL REFERENCES "user"(id), + role_id INTEGER NOT NULL REFERENCES role(id), + UNIQUE (user_id, role_id) +); + +CREATE TABLE IF NOT EXISTS user_permission_int ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + create_date TIMESTAMPTZ, + modify_date TIMESTAMPTZ, + user_id INTEGER NOT NULL REFERENCES "user"(id), + permission_id INTEGER NOT NULL REFERENCES permission(id), + UNIQUE (user_id, permission_id) +); + +INSERT INTO role (name, description) VALUES + ('admin', 'All QQQ All demo tables'), + ('viewer', 'Read-only QQQ All demo tables') +ON CONFLICT (name) DO NOTHING; + +INSERT INTO permission (name, object_type, object_label) +SELECT table_name || '.' || action, 'table', table_name +FROM (VALUES + ('customer'), ('order'), ('orderLine'), ('product'), + ('warehouseCustomer'), ('supplierOrder'), ('shipment'), ('document'), ('importFile') +) AS tables(table_name) +CROSS JOIN (VALUES ('read'), ('insert'), ('edit'), ('delete')) AS actions(action) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO role_permission_int (role_id, permission_id) +SELECT role.id, permission.id +FROM role CROSS JOIN permission +WHERE role.name = 'admin' OR (role.name = 'viewer' AND permission.name LIKE '%.read') +ON CONFLICT (role_id, permission_id) DO NOTHING; diff --git a/qqq-all-app/README.md b/qqq-all-app/README.md index 09a0251..e392191 100644 --- a/qqq-all-app/README.md +++ b/qqq-all-app/README.md @@ -4,14 +4,16 @@ Build with Java 21 and Maven, then start the standalone core profile: ```sh mvn -pl qqq-all-app -am verify -java -jar qqq-all-app/target/qqq-all-app-4.1.0-SNAPSHOT.jar +java -jar qqq-all-app/target/qqq-all-app.jar ``` The core app binds to `127.0.0.1` and serves the Next dashboard at `http://127.0.0.1:8080/` and health at `/health`. Set `QQQ_ALL_PORT` to change the port and `QQQ_ALL_DATA_DIR` to change the default `./data` directory. On first start it seeds customers and orders in H2, order lines in SQLite, and products as JSON files. Artemis runs in the same JVM; inserts and updates to `order` publish to the `orderEvents` topic and trigger `syncOrder`. +`QQQ_ALL_BIND_HOST` can select another interface when intentionally running behind a container port mapping. The Compose core service sets it to `0.0.0.0` inside the container while publishing only on host loopback. + Core uses mock authentication. The `sessionId` cookie selects the two fixed demo identities: `aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa` for Admin and `dddddddd-dddd-4ddd-8ddd-dddddddddddd` for Demo User. Any other session ID uses Demo User. Admin can read and write the customer, order, order-line, product, and address tables; Demo User can read them but cannot write them. These identities are for local demonstrations. Quick search is disabled in core because it requires OpenSearch; the customer list remains available through the normal table query API. -The core demo seeds a customer table view, a shipping address linked to the geo country/state/city tables, and an order review workflow. The `RunRecordWorkflow` process can run that workflow against an order; `syncOrder` writes a process trace after an order event. It also seeds an active `orderStored` webhook subscription to its loopback receiver at `/demo/order-webhook`. Insert an order as Admin, run `SendWebhookEvent` as Admin, and inspect `/demo/order-webhook-receipts` to see the delivered payload. Demo User can read webhook records but cannot edit their configuration or invoke the delivery process. The receiver is available only in the loopback-bound core profile. +The core demo seeds a customer table view, a shipping address linked to the geo country/state/city tables, and an order review workflow. The `RunRecordWorkflow` process can run that workflow against an order; `syncOrder` writes a process trace after an order event. It also seeds an active `orderStored` webhook subscription to its receiver at `/demo/order-webhook`. Insert an order as Admin, run `SendWebhookEvent` as Admin, and inspect `/demo/order-webhook-receipts` to see the delivered payload. Demo User can read webhook records but cannot edit their configuration or invoke the delivery process. The receiver is available only in the core profile, which Compose publishes on host loopback. ## Full profile @@ -26,9 +28,22 @@ Configure the following environment variables before starting the same jar: | MongoDB replica set | `QQQ_ALL_MONGO_HOST`, `QQQ_ALL_MONGO_DATABASE`, `QQQ_ALL_MONGO_USER`, `QQQ_ALL_MONGO_PASSWORD`, `QQQ_ALL_MONGO_REPLICA_SET` | `QQQ_ALL_MONGO_PORT` (27017), `QQQ_ALL_MONGO_AUTH_DATABASE` (admin) | | MinIO S3 | `QQQ_ALL_S3_ENDPOINT`, `QQQ_ALL_S3_BUCKET`, `QQQ_ALL_S3_ACCESS_KEY`, `QQQ_ALL_S3_SECRET_KEY` | `QQQ_ALL_S3_REGION` (us-east-1) | | SFTP | `QQQ_ALL_SFTP_HOST`, `QQQ_ALL_SFTP_USER`, `QQQ_ALL_SFTP_PASSWORD` | `QQQ_ALL_SFTP_PORT` (22), `QQQ_ALL_SFTP_BASE_PATH` (/upload) | -| ESB | `QQQ_ALL_ARTEMIS_URL`, `QQQ_ALL_RABBITMQ_URL`, `QQQ_ALL_RABBITMQ_USER`, `QQQ_ALL_RABBITMQ_PASSWORD` | — | +| ESB | `QQQ_ALL_ARTEMIS_URL`, `QQQ_ALL_ARTEMIS_USER`, `QQQ_ALL_ARTEMIS_PASSWORD`, `QQQ_ALL_ARTEMIS_MANAGEMENT_URL`, `QQQ_ALL_RABBITMQ_URL`, `QQQ_ALL_RABBITMQ_USER`, `QQQ_ALL_RABBITMQ_PASSWORD`, `QQQ_ALL_RABBITMQ_MANAGEMENT_URL` | — | | Mailpit SMTP | `QQQ_ALL_SMTP_HOST` | `QQQ_ALL_SMTP_PORT` (1025) | | Keycloak OIDC | `QQQ_ALL_OIDC_BASE_URL`, `QQQ_ALL_OIDC_CLIENT_ID`, `QQQ_ALL_OIDC_CLIENT_SECRET` | `QQQ_ALL_OIDC_EXTERNAL_BASE_URL` (base URL), `QQQ_ALL_OIDC_SCOPES` (`openid email profile`) | | OpenSearch quick search | `QQQ_ALL_OPENSEARCH_HOST` | `QQQ_ALL_OPENSEARCH_PORT` (9200), `QQQ_ALL_OPENSEARCH_INDEX` (`qqq-all-customers`), `QQQ_ALL_OPENSEARCH_USER`, `QQQ_ALL_OPENSEARCH_PASSWORD`, `QQQ_ALL_OPENSEARCH_SSL` (false) | -The full app adds sample tables on each external backend, `orderEvents` on Artemis, and `orderSyncEvents` on RabbitMQ. Quick search is configured for customer `name` and `email`; run `quickSearchFullReindex` after OpenSearch and the customer data are ready. The SFTP integration adds its import source and staging tables plus `SFTPImportFileSyncProcess`; `externalImportFile` remains the direct SFTP file sample. Keycloak realm roles resolve by name against the PostgreSQL `role` table from `qbit-user-role-permissions`, then become QQQ session permissions. Unknown roles receive no permissions. Webhook tables and delivery processes require explicit permissions: the viewer role receives read access, while the admin role can edit destinations and run delivery. The additive `infra/postgres/03-webhook-permissions.sql` grant file runs after C6's role seed. PostgreSQL must also contain the qbit permission tables and OAuth session and redirect-state tables. External database schema, demo records, demo roles, the MinIO bucket, and the service stack are provisioned by the forthcoming compose and seed work in C6; this profile expects those resources to exist. +The full app adds sample tables on each external backend, `orderEvents` on Artemis, and `orderSyncEvents` on RabbitMQ. Quick search is configured for customer `name` and `email`; run `quickSearchFullReindex` after OpenSearch and the customer data are ready. The SFTP integration adds its import source and staging tables plus `SFTPImportFileSyncProcess`; `externalImportFile` remains the direct SFTP file sample. Keycloak realm roles resolve by name against the PostgreSQL `role` table from `qbit-user-role-permissions`, then become QQQ session permissions. Unknown roles receive no permissions. Webhook tables and delivery processes require explicit permissions: the viewer role receives read access, while the admin role can edit destinations and run delivery. The Compose profile provisions demo SQL tables, OAuth session and redirect-state tables, qbit permission tables and roles, and the MinIO bucket on first startup; `infra/postgres/03-webhook-permissions.sql` adds webhook grants after the role seed. + +On startup, the full demo creates its remaining PostgreSQL qbit tables from the registered metadata and checks every expected column and type. The hand-maintained domain, OAuth, and role tables stay under `infra/postgres`. If a persisted qbit table is missing a required column or has a different type, startup fails with a migration/reset message instead of silently accepting it. The full-profile RabbitMQ completion-event smoke remains gated on QQQ ESB Task 9's metadata-driven process lifecycle publication; the broker management smoke below can run independently. + +After starting the full Compose profile, run the opt-in broker management smoke test with the same broker credentials. For the supplied demo environment: + +```sh +set -a +. ./.env.example +set +a +QQQ_ALL_COMPOSE_BROKERS=true mvn -pl qqq-all-app -am -Dtest=FullProfileBrokerIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false test +``` + +The test uses the loopback management ports to read and pause/resume the Artemis subscription queue, then creates, reads, purges, and deletes a temporary RabbitMQ queue. For a customized `.env`, source that file instead. It is skipped during ordinary `mvn verify`. diff --git a/qqq-all-app/pom.xml b/qqq-all-app/pom.xml index 056cd78..79aa9d3 100644 --- a/qqq-all-app/pom.xml +++ b/qqq-all-app/pom.xml @@ -51,10 +51,12 @@ com.h2databaseh2 org.xerialsqlite-jdbc org.apache.artemisartemis-jakarta-server2.57.0 + com.rabbitmq.jmsrabbitmq-jms org.junit.jupiterjunit-jupiter6.0.2test + qqq-all-app org.apache.maven.plugins diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/CoreRuntime.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/CoreRuntime.java index 267204f..b4afbd5 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/CoreRuntime.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/CoreRuntime.java @@ -34,6 +34,11 @@ final class CoreRuntime implements AutoCloseable } static CoreRuntime launch(Path dataDirectory, Integer port) throws Exception + { + return launch(dataDirectory, port, "127.0.0.1"); + } + + static CoreRuntime launch(Path dataDirectory, Integer port, String bindHost) throws Exception { CoreData.seed(dataDirectory); Integer brokerPort = findFreePort(); @@ -56,7 +61,7 @@ final class CoreRuntime implements AutoCloseable new QqqAllApplication(dataDirectory, brokerUrl), new QApplicationLauncherConfig().withRegisterShutdownHook(false) .withServerCustomizer(server -> server.withPort(port) - .withJavalinConfigCustomizer(config -> config.jetty.host = "127.0.0.1"))); + .withJavalinConfigCustomizer(config -> config.jetty.host = bindHost))); QContext.init(launcher.getQInstance(), new QSystemUserSession()); try { diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBitData.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBitData.java index db52bc9..695b2df 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBitData.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/DemoQBitData.java @@ -9,10 +9,17 @@ package com.kingsrook.qqq.all; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; @@ -22,6 +29,11 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDe /** Creates repeatable local tables for the qbit metadata exposed by core. */ final class DemoQBitData { + /** These tables have hand-maintained PostgreSQL DDL in infra/postgres/01 and 02. */ + private static final Set POSTGRES_MANAGED_TABLES = Set.of("warehouse_customer", "user_session", + "redirect_state", "user", "role", "permission", "role_permission_int", "user_role_int", + "user_permission_int"); + private DemoQBitData() { } @@ -86,6 +98,133 @@ final class DemoQBitData } } + static void ensurePostgresSchema(QInstance instance) throws SQLException + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) instance.getBackend("postgres"); + try(Connection connection = DriverManager.getConnection(ConnectionManager.getJdbcUrl(backend), + backend.getUsername(), backend.getPassword()); + Statement statement = connection.createStatement()) + { + for(String sql : postgresSchemaStatements(instance)) + { + statement.execute(sql); + } + for(QTableMetaData table : postgresQBitTables(instance)) + { + assertPostgresColumns(connection, table); + } + } + } + + static List postgresSchemaStatements(QInstance instance) + { + List statements = new ArrayList<>(); + for(QTableMetaData table : postgresQBitTables(instance)) + { + String tableName = postgresTableName(table); + List columns = new ArrayList<>(); + for(QFieldMetaData field : table.getFields().values()) + { + String name = field.getBackendName() == null ? field.getName() : field.getBackendName(); + String identity = field.getName().equals(table.getPrimaryKeyField()) + && (field.getType() == QFieldType.INTEGER || field.getType() == QFieldType.LONG) + ? " GENERATED BY DEFAULT AS IDENTITY" : ""; + String primaryKey = field.getName().equals(table.getPrimaryKeyField()) ? " PRIMARY KEY" : ""; + columns.add(postgresIdentifier(name) + " " + postgresSqlType(field.getType()) + identity + primaryKey); + } + statements.add("CREATE TABLE IF NOT EXISTS " + postgresIdentifier(tableName) + + " (" + String.join(", ", columns) + ")"); + } + return statements; + } + + private static List postgresQBitTables(QInstance instance) + { + return instance.getTables().values().stream() + .filter(table -> "postgres".equals(table.getBackendName()) && !table.getFields().isEmpty()) + .filter(table -> !POSTGRES_MANAGED_TABLES.contains(postgresTableName(table))) + .toList(); + } + + private static String postgresTableName(QTableMetaData table) + { + return table.getBackendDetails() instanceof RDBMSTableBackendDetails details + ? details.getTableName() : table.getName(); + } + + private static void assertPostgresColumns(Connection connection, QTableMetaData table) throws SQLException + { + String tableName = postgresTableName(table); + Map actual = new HashMap<>(); + try(PreparedStatement query = connection.prepareStatement("SELECT column_name, data_type " + + "FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ?")) + { + query.setString(1, tableName); + try(ResultSet rows = query.executeQuery()) + { + while(rows.next()) + { + actual.put(rows.getString(1), rows.getString(2)); + } + } + } + for(QFieldMetaData field : table.getFields().values()) + { + String columnName = field.getBackendName() == null ? field.getName() : field.getBackendName(); + String expectedType = postgresInformationSchemaType(field.getType()); + String actualType = actual.get(columnName); + if(!expectedType.equals(actualType)) + { + throw new SQLException("Full-profile table " + tableName + " column " + columnName + + " must be " + expectedType + "; found " + (actualType == null ? "missing" : actualType) + + ". Recreate the demo volume or migrate the persisted schema."); + } + } + } + + private static String postgresInformationSchemaType(QFieldType type) + { + return switch(type) + { + case INTEGER -> "integer"; + case LONG -> "bigint"; + case DECIMAL -> "numeric"; + case BOOLEAN -> "boolean"; + case DATE -> "date"; + case TIME -> "time without time zone"; + case DATE_TIME -> "timestamp without time zone"; + case BLOB -> "bytea"; + case STRING, PASSWORD -> "character varying"; + case TEXT, HTML -> "text"; + }; + } + + private static String postgresIdentifier(String value) + { + if(value == null || !value.matches("[A-Za-z][A-Za-z0-9_]*")) + { + throw new IllegalArgumentException("Unsupported demo schema identifier"); + } + return "\"" + value + "\""; + } + + private static String postgresSqlType(QFieldType type) + { + return switch(type) + { + case INTEGER -> "INTEGER"; + case LONG -> "BIGINT"; + case DECIMAL -> "DECIMAL(20,6)"; + case BOOLEAN -> "BOOLEAN"; + case DATE -> "DATE"; + case TIME -> "TIME"; + case DATE_TIME -> "TIMESTAMP"; + case BLOB -> "BYTEA"; + case STRING, PASSWORD -> "VARCHAR(4000)"; + case TEXT, HTML -> "TEXT"; + }; + } + private static String identifier(String value) { if(value == null || !value.matches("[A-Za-z][A-Za-z0-9_]*")) diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java index 04d6a85..65b6659 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileApplication.java @@ -7,6 +7,7 @@ package com.kingsrook.qqq.all; import java.nio.file.Path; +import java.sql.SQLException; import java.util.List; import com.kingsrook.qbits.userrolepermissions.UserRolePermissionsQBitConfig; import com.kingsrook.qbits.userrolepermissions.UserRolePermissionsQBitProducer; @@ -48,11 +49,18 @@ final class FullProfileApplication extends AbstractQQQApplication { private final Path dataDirectory; private final FullProfileSettings settings; + private final boolean provisionSchema; FullProfileApplication(Path dataDirectory, FullProfileSettings settings) + { + this(dataDirectory, settings, false); + } + + FullProfileApplication(Path dataDirectory, FullProfileSettings settings, boolean provisionSchema) { this.dataDirectory = dataDirectory; this.settings = settings; + this.provisionSchema = provisionSchema; } @Override @@ -65,13 +73,39 @@ final class FullProfileApplication extends AbstractQQQApplication addEsb(instance); DemoQBits.addCore(instance, "postgres"); DemoQBits.addFull(instance, settings); + // OIDC and ESB sessions use string user IDs, unlike the qbit's default integer user ID. + instance.getTable("processTrace").getField("userId").setType(QFieldType.STRING); DemoUsers.protectWebhookResources(instance); + for(QTableMetaData table : instance.getTables().values()) + { + if(!"postgres".equals(table.getBackendName())) + { + continue; + } + if(table.getBackendDetails() == null) + { + table.setBackendDetails(new RDBMSTableBackendDetails() + .withTableName(QInstanceEnricher.inferBackendName(table.getName()))); + } + QInstanceEnricher.setInferredFieldBackendNames(table); + } EmailMessagingProviderMetaData mail = new EmailMessagingProviderMetaData() .withSmtpServer(settings.get("SMTP_HOST")) .withSmtpPort(settings.getOrDefault("SMTP_PORT", "1025")); mail.setName("mailpit"); instance.addMessagingProvider(mail); + if(provisionSchema) + { + try + { + DemoQBitData.ensurePostgresSchema(instance); + } + catch(SQLException e) + { + throw new QException("Could not provision full-profile qbit tables", e); + } + } return instance; } @@ -80,6 +114,10 @@ final class FullProfileApplication extends AbstractQQQApplication instance.addBackend(new RDBMSBackendMetaData().withName("postgres").withVendor("postgres") .withHostName(settings.get("POSTGRES_HOST")).withPort(settings.port("POSTGRES_PORT", "5432")) .withDatabaseName(settings.get("POSTGRES_DATABASE")) + .withJdbcUrl("jdbc:postgresql://" + settings.get("POSTGRES_HOST") + ":" + + settings.port("POSTGRES_PORT", "5432") + "/" + settings.get("POSTGRES_DATABASE")) + .withJdbcDriverClassName("org.postgresql.Driver") + .withActionStrategyCodeReference(new QCodeReference(PostgresActionStrategy.class)) .withUsername(settings.get("POSTGRES_USER")).withPassword(settings.get("POSTGRES_PASSWORD"))); instance.addBackend(new RDBMSBackendMetaData().withName("mysql").withVendor("mysql") .withHostName(settings.get("MYSQL_HOST")).withPort(settings.port("MYSQL_PORT", "3306")) @@ -164,10 +202,20 @@ final class FullProfileApplication extends AbstractQQQApplication private void addEsb(QInstance instance) { + EsbInstanceMetaData.of(instance) + .getProvider("artemis") + .withUsername(settings.get("ARTEMIS_USER")) + .withPassword(settings.get("ARTEMIS_PASSWORD")) + .withManagementUrl(settings.get("ARTEMIS_MANAGEMENT_URL")) + .withManagementUsername(settings.get("ARTEMIS_USER")) + .withManagementPassword(settings.get("ARTEMIS_PASSWORD")); EsbInstanceMetaData.of(instance) .withProvider(new QEsbProviderMetaData().withName("rabbitmq") .withType(EsbProviderType.RABBITMQ).withUrl(settings.get("RABBITMQ_URL")) - .withUsername(settings.get("RABBITMQ_USER")).withPassword(settings.get("RABBITMQ_PASSWORD"))) + .withUsername(settings.get("RABBITMQ_USER")).withPassword(settings.get("RABBITMQ_PASSWORD")) + .withManagementUrl(settings.get("RABBITMQ_MANAGEMENT_URL")) + .withManagementUsername(settings.get("RABBITMQ_USER")) + .withManagementPassword(settings.get("RABBITMQ_PASSWORD"))) .withDestination(new QEsbDestinationMetaData().withName("orderSyncEvents") .withType(EsbDestinationType.QUEUE).withProviderName("rabbitmq") .withDestinationName("qqq.all.orderSyncEvents")); diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileSettings.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileSettings.java index f89b94b..32d964d 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileSettings.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/FullProfileSettings.java @@ -19,7 +19,8 @@ final class FullProfileSettings "MONGO_HOST", "MONGO_DATABASE", "MONGO_USER", "MONGO_PASSWORD", "MONGO_REPLICA_SET", "S3_ENDPOINT", "S3_BUCKET", "S3_ACCESS_KEY", "S3_SECRET_KEY", "SFTP_HOST", "SFTP_USER", "SFTP_PASSWORD", - "ARTEMIS_URL", "RABBITMQ_URL", "RABBITMQ_USER", "RABBITMQ_PASSWORD", + "ARTEMIS_URL", "ARTEMIS_USER", "ARTEMIS_PASSWORD", "ARTEMIS_MANAGEMENT_URL", + "RABBITMQ_URL", "RABBITMQ_USER", "RABBITMQ_PASSWORD", "RABBITMQ_MANAGEMENT_URL", "SMTP_HOST", "OIDC_BASE_URL", "OIDC_CLIENT_ID", "OIDC_CLIENT_SECRET", "OPENSEARCH_HOST"); diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/PostgresActionStrategy.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/PostgresActionStrategy.java new file mode 100644 index 0000000..c9bce8c --- /dev/null +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/PostgresActionStrategy.java @@ -0,0 +1,39 @@ +/* + * Copyright 2026 Kingsrook, LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.kingsrook.qqq.all; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy; + +/** PostgreSQL syntax for the shared QQQ RDBMS actions. */ +public class PostgresActionStrategy extends BaseRDBMSActionStrategy +{ + @Override + public String getIdentifierQuoteString() + { + return "\""; + } + + @Override + public String getInsertDefaultValuesClause() + { + return "DEFAULT VALUES"; + } + + @Override + protected int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException + { + if(value == null) + { + statement.setNull(index, Types.NULL); + return 1; + } + return super.bindParamObject(statement, index, value); + } +} diff --git a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java index 3ff460c..ee8b22c 100644 --- a/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java +++ b/qqq-all-app/src/main/java/com/kingsrook/qqq/all/ProfileLauncher.java @@ -40,13 +40,13 @@ public final class ProfileLauncher Integer port = Integer.valueOf(environment.getOrDefault("QQQ_ALL_PORT", "8080")); if("core".equals(selected)) { - CoreRuntime.launch(dataDirectory, port); + CoreRuntime.launch(dataDirectory, port, environment.getOrDefault("QQQ_ALL_BIND_HOST", "127.0.0.1")); return; } FullProfileSettings settings = FullProfileSettings.from(environment); CoreData.seed(dataDirectory); - QApplicationLauncher.run(new FullProfileApplication(dataDirectory, settings), + QApplicationLauncher.run(new FullProfileApplication(dataDirectory, settings, true), new QApplicationLauncherConfig().withRegisterShutdownHook(true) .withServerCustomizer(server -> server.withPort(port) .withJavalinConfigCustomizer(config -> config.jetty.host = settings.get("BIND_HOST")))); diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileBrokerIntegrationTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileBrokerIntegrationTest.java new file mode 100644 index 0000000..464f7ed --- /dev/null +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileBrokerIntegrationTest.java @@ -0,0 +1,91 @@ +/* + * Copyright 2026 Kingsrook, LLC + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0 + */ +package com.kingsrook.qqq.all; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Base64; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.esb.management.ArtemisJolokiaAdapter; +import com.kingsrook.qqq.esb.management.EsbBrokerNames; +import com.kingsrook.qqq.esb.management.RabbitManagementAdapter; +import com.kingsrook.qqq.esb.model.EsbInstanceMetaData; +import com.kingsrook.qqq.esb.model.EsbProviderType; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** Opt-in smoke test against the local full Compose brokers and their management APIs. */ +@EnabledIfEnvironmentVariable(named = "QQQ_ALL_COMPOSE_BROKERS", matches = "true") +class FullProfileBrokerIntegrationTest +{ + @TempDir + Path dataDirectory; + + @Test + void managesQueuesWithFullProfileCredentials() throws Exception + { + Map environment = FullProfileTest.environment(); + environment.put("QQQ_ALL_ARTEMIS_USER", System.getenv("ARTEMIS_USER")); + environment.put("QQQ_ALL_ARTEMIS_PASSWORD", System.getenv("ARTEMIS_PASSWORD")); + environment.put("QQQ_ALL_ARTEMIS_MANAGEMENT_URL", "http://127.0.0.1:8161"); + environment.put("QQQ_ALL_RABBITMQ_USER", System.getenv("RABBITMQ_USER")); + environment.put("QQQ_ALL_RABBITMQ_PASSWORD", System.getenv("RABBITMQ_PASSWORD")); + environment.put("QQQ_ALL_RABBITMQ_MANAGEMENT_URL", "http://127.0.0.1:15672"); + QInstance instance = new FullProfileApplication(dataDirectory, FullProfileSettings.from(environment)).defineQInstance(); + EsbInstanceMetaData esb = EsbInstanceMetaData.of(instance); + + ArtemisJolokiaAdapter artemis = new ArtemisJolokiaAdapter(esb.getProvider("artemis")); + String subscription = EsbBrokerNames.subscriptionQueue(EsbProviderType.ACTIVEMQ_ARTEMIS, + "qqq.all.orderEvents", "syncOrder.qqq.all.orderEvents"); + assertTrue(artemis.getQueueInfo(subscription).isPresent()); + try + { + artemis.pauseQueue(subscription); + assertTrue(artemis.getQueueInfo(subscription).orElseThrow().paused()); + } + finally + { + artemis.resumeQueue(subscription); + } + assertFalse(artemis.getQueueInfo(subscription).orElseThrow().paused()); + + RabbitManagementAdapter rabbit = new RabbitManagementAdapter(esb.getProvider("rabbitmq")); + String queueName = "qqq.all.c6.managementProbe." + UUID.randomUUID(); + String queueUrl = "http://127.0.0.1:15672/api/queues/%2F/" + queueName; + String credentials = environment.get("QQQ_ALL_RABBITMQ_USER") + ":" + environment.get("QQQ_ALL_RABBITMQ_PASSWORD"); + String authorization = "Basic " + Base64.getEncoder().encodeToString(credentials.getBytes(StandardCharsets.UTF_8)); + try(HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build()) + { + HttpRequest create = HttpRequest.newBuilder(URI.create(queueUrl)) + .header("Authorization", authorization).header("Content-Type", "application/json") + .PUT(HttpRequest.BodyPublishers.ofString("{\"durable\":true,\"auto_delete\":false,\"arguments\":{}}")) + .build(); + assertEquals(201, client.send(create, HttpResponse.BodyHandlers.discarding()).statusCode()); + try + { + assertTrue(rabbit.getQueueInfo(queueName).isPresent()); + assertEquals(0L, rabbit.purgeQueue(queueName)); + } + finally + { + HttpRequest delete = HttpRequest.newBuilder(URI.create(queueUrl)) + .header("Authorization", authorization).DELETE().build(); + assertEquals(204, client.send(delete, HttpResponse.BodyHandlers.discarding()).statusCode()); + } + } + } +} diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java index 32fd00a..f45b85b 100644 --- a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullProfileTest.java @@ -15,6 +15,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.authentication.OAuth2Authen import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; import com.kingsrook.qqq.backend.core.model.metadata.messaging.email.EmailMessagingProviderMetaData; import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; import com.kingsrook.qqq.esb.model.EsbInstanceMetaData; import org.junit.jupiter.api.Test; @@ -43,7 +44,12 @@ class FullProfileTest QInstance instance = new FullProfileApplication(dataDirectory, settings).defineQInstance(); new QInstanceValidator().validate(instance); - assertEquals("postgres.example.test", ((RDBMSBackendMetaData) instance.getBackend("postgres")).getHostName()); + RDBMSBackendMetaData postgres = (RDBMSBackendMetaData) instance.getBackend("postgres"); + assertEquals("postgres.example.test", postgres.getHostName()); + assertEquals("jdbc:postgresql://postgres.example.test:5432/qqq", ConnectionManager.getJdbcUrl(postgres)); + assertEquals("org.postgresql.Driver", ConnectionManager.getJdbcDriverClassName(postgres)); + assertEquals("\"", postgres.getActionStrategy().getIdentifierQuoteString()); + assertEquals("DEFAULT VALUES", postgres.getActionStrategy().getInsertDefaultValuesClause()); assertEquals("mysql.example.test", ((RDBMSBackendMetaData) instance.getBackend("mysql")).getHostName()); assertEquals("replicaSet=rs0", ((MongoDBBackendMetaData) instance.getBackend("mongo")).getUrlSuffix()); assertEquals("postgres", instance.getTable("warehouseCustomer").getBackendName()); @@ -56,7 +62,15 @@ class FullProfileTest EsbInstanceMetaData esb = EsbInstanceMetaData.of(instance); assertEquals("tcp://artemis.example.test:61616", esb.getProvider("artemis").getUrl()); + assertEquals("fixture", esb.getProvider("artemis").getUsername()); + assertEquals("fixture", esb.getProvider("artemis").getPassword()); + assertEquals("http://artemis.example.test:8161", esb.getProvider("artemis").getManagementUrl()); + assertEquals("fixture", esb.getProvider("artemis").getManagementUsername()); + assertEquals("fixture", esb.getProvider("artemis").getManagementPassword()); assertEquals("amqp://rabbit.example.test:5672/%2F", esb.getProvider("rabbitmq").getUrl()); + assertEquals("http://rabbit.example.test:15672", esb.getProvider("rabbitmq").getManagementUrl()); + assertEquals("fixture", esb.getProvider("rabbitmq").getManagementUsername()); + assertEquals("fixture", esb.getProvider("rabbitmq").getManagementPassword()); assertEquals("artemis", esb.getDestination("orderEvents").getProviderName()); assertEquals("rabbitmq", esb.getDestination("orderSyncEvents").getProviderName()); @@ -94,7 +108,11 @@ class FullProfileTest values.put("QQQ_ALL_SFTP_USER", "fixture"); values.put("QQQ_ALL_SFTP_PASSWORD", "fixture"); values.put("QQQ_ALL_ARTEMIS_URL", "tcp://artemis.example.test:61616"); + values.put("QQQ_ALL_ARTEMIS_USER", "fixture"); + values.put("QQQ_ALL_ARTEMIS_PASSWORD", "fixture"); + values.put("QQQ_ALL_ARTEMIS_MANAGEMENT_URL", "http://artemis.example.test:8161"); values.put("QQQ_ALL_RABBITMQ_URL", "amqp://rabbit.example.test:5672/%2F"); + values.put("QQQ_ALL_RABBITMQ_MANAGEMENT_URL", "http://rabbit.example.test:15672"); values.put("QQQ_ALL_RABBITMQ_USER", "fixture"); values.put("QQQ_ALL_RABBITMQ_PASSWORD", "fixture"); values.put("QQQ_ALL_SMTP_HOST", "mailpit.example.test"); diff --git a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java index 1460427..8b14d3b 100644 --- a/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java +++ b/qqq-all-app/src/test/java/com/kingsrook/qqq/all/FullQBitsTest.java @@ -18,6 +18,8 @@ import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; import com.kingsrook.qqq.backend.core.model.session.QSession; import org.junit.jupiter.api.Test; @@ -50,6 +52,18 @@ class FullQBitsTest assertNotNull(instance.getProcess("SFTPImportFileSyncProcess")); assertNotNull(instance.getTable("SFTPImportSourceFileTable")); assertEquals("sftp", instance.getTable("SFTPImportSourceFileTable").getBackendName()); + assertEquals("scheduled_job", ((RDBMSTableBackendDetails) instance.getTable("scheduledJob") + .getBackendDetails()).getTableName()); + assertEquals("create_date", instance.getTable("scheduledJob").getField("createDate").getBackendName()); + assertEquals(QFieldType.STRING, instance.getTable("processTrace").getField("userId").getType()); + assertTrue(DemoQBitData.postgresSchemaStatements(instance).stream() + .anyMatch(sql -> sql.startsWith("CREATE TABLE IF NOT EXISTS \"scheduled_job\"") + && sql.contains("\"create_date\" TIMESTAMP"))); + assertTrue(DemoQBitData.postgresSchemaStatements(instance).stream() + .anyMatch(sql -> sql.startsWith("CREATE TABLE IF NOT EXISTS \"webhook_subscription\"") + && sql.contains("\"webhook_event_type_name\" VARCHAR(4000)"))); + assertFalse(DemoQBitData.postgresSchemaStatements(instance).stream() + .anyMatch(sql -> sql.contains("\"user_session\"") || sql.contains("\"warehouse_customer\""))); for(String tableName : Set.of("webhook", "webhookSubscription", "webhookEvent", "webhookEventContent", "webhookEventSendLog", "scheduledJob", "scheduledJobParameter")) {