Compare commits

...
Author SHA1 Message Date
james.maes 6df9f08643 ci(#8): pin settings-safe QQQ orb 0.6.8 2026-09-30 10:49:56 -05:00
james.maes a511834edd fix(GH-21): follow listener address for the API catalog 2026-09-27 05:06:47 -05:00
james.maes 6e9419f342 feat(GH-21): complete runtime backend and JavaScript demos 2026-09-27 04:51:56 -05:00
james.maes e1625d2000 feat(esb): expose service observability in core profile 2026-09-26 13:44:39 -05:00
james.maes cd9168fc9d test(smoke): verify Rabbit completion through ESB API
Grant full-profile admin ESB access and keep the exact process-run assertion. Part of #8.
2026-09-26 13:25:39 -05:00
james.maes 3ba8aea90e fix(smoke): use Maven final jar in core CI (#8) 2026-09-26 11:17:09 -05:00
james.maes 68d56ddedd fix(smoke): require explicit standalone full search endpoint (#8) 2026-09-26 11:08:17 -05:00
james.maes 4e81bda8ec merge: reconcile C7 with merged C8 develop (#8) 2026-09-26 11:05:18 -05:00
james.maes 82891b54fd fix(smoke): exercise private full Compose services over HTTP (#8) 2026-09-26 11:05:09 -05:00
james.maes f9777ae373 merge: integrate C6 full Compose profile into C7 smoke (#8) 2026-09-26 10:48:20 -05:00
james.maes e8870e33ac fix(smoke): require order-correlated Artemis trace and full CI job (#8) 2026-09-26 10:35:46 -05:00
james.maes 1584a5d879 fix(smoke): query SFTP qbit and isolate Compose ports (#8) 2026-09-26 10:24:35 -05:00
james.maes b74dd284a8 ci(smoke): add core CI and prepare full conformance harness (#8) 2026-09-26 10:20:22 -05:00
28 changed files with 2106 additions and 68 deletions

No files matched your search

+1 -1
View File
@@ -3,7 +3,7 @@
version: 2.1
orbs:
qqq-orb: kingsrook/qqq-orb@0.6.5
qqq-orb: kingsrook/qqq-orb@0.6.8
commands:
configure-package-install:
+47
View File
@@ -0,0 +1,47 @@
name: Smoke
on:
pull_request:
push:
branches: [develop]
workflow_dispatch:
permissions:
contents: read
jobs:
core:
runs-on: ubuntu-latest
timeout-minutes: 25
env:
PYTHONDONTWRITEBYTECODE: "1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "21"
cache: maven
- name: Check smoke assertions
run: python3 -m unittest discover -s scripts -p 'test_*.py' -v
- name: Verify and package core app
run: mvn -B -ntp -pl qqq-all-app -am verify
- name: Smoke packaged core over HTTP
run: python3 scripts/run_core_smoke.py qqq-all-app/target/qqq-all-app.jar
full:
runs-on: ubuntu-latest
timeout-minutes: 45
env:
PYTHONDONTWRITEBYTECODE: "1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "21"
cache: maven
- name: Package full app for Docker image
run: mvn -B -ntp -pl qqq-all-app -am -DskipTests package
- name: Smoke full Compose over HTTP
run: python3 scripts/run_full_smoke.py
+2
View File
@@ -1,4 +1,6 @@
target/
__pycache__/
*.pyc
*.iml
.env
/.env.local
+15
View File
@@ -0,0 +1,15 @@
/* Give the full-profile demo admin service-level ESB visibility and controls.
The viewer role keeps table READ without broker-wide message access. */
INSERT INTO permission (name, object_type, object_label)
VALUES
('esbView.hasAccess', 'app', 'ESB'),
('esbOperate.hasAccess', 'process', 'ESB operations'),
('esbDelete.hasAccess', 'process', 'ESB message deletion')
ON CONFLICT (name) DO NOTHING;
INSERT INTO role_permission_int (role_id, permission_id)
SELECT role.id, permission.id
FROM role CROSS JOIN permission
WHERE role.name = 'admin'
AND permission.name IN ('esbView.hasAccess', 'esbOperate.hasAccess', 'esbDelete.hasAccess')
ON CONFLICT (role_id, permission_id) DO NOTHING;
@@ -0,0 +1,18 @@
-- Read-only API catalog, transient notes, and the fixed JavaScript calculation.
INSERT INTO permission (name, object_type, object_label)
VALUES
('apiCatalog.read', 'table', 'API Catalog'),
('demoNote.read', 'table', 'Demo Note (Transient)'),
('demoNote.insert', 'table', 'Demo Note (Transient)'),
('demoNote.edit', 'table', 'Demo Note (Transient)'),
('demoNote.delete', 'table', 'Demo Note (Transient)'),
('calculateOrderTotal.hasAccess', 'process', 'Calculate Order Total (JavaScript)')
ON CONFLICT (name) DO NOTHING;
INSERT INTO role_permission_int (role_id, permission_id)
SELECT role.id, permission.id
FROM role CROSS JOIN permission
WHERE permission.name IN ('apiCatalog.read', 'demoNote.read', 'demoNote.insert',
'demoNote.edit', 'demoNote.delete', 'calculateOrderTotal.hasAccess')
AND (role.name = 'admin' OR (role.name = 'viewer' AND permission.name IN ('apiCatalog.read', 'demoNote.read')))
ON CONFLICT (role_id, permission_id) DO NOTHING;
+71 -2
View File
@@ -11,7 +11,7 @@ The core app binds to `127.0.0.1` and serves the Next dashboard at `http://127.0
`QQQ_ALL_BIND_HOST` can select another interface when intentionally running behind a container port mapping. The Compose core service sets it to `0.0.0.0` inside the container while publishing only on host loopback.
Core uses mock authentication. The `sessionId` cookie selects the two fixed demo identities: `aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa` for Admin and `dddddddd-dddd-4ddd-8ddd-dddddddddddd` for Demo User. Any other session ID uses Demo User. Admin can read and write the customer, order, order-line, product, and address tables; Demo User can read them but cannot write them. These identities are for local demonstrations. Quick search is disabled in core because it requires OpenSearch; the customer list remains available through the normal table query API.
Core uses mock authentication. The `sessionId` cookie selects the two fixed demo identities: `aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa` for Admin and `dddddddd-dddd-4ddd-8ddd-dddddddddddd` for Demo User. Any other session ID uses Demo User. Admin can read and write the customer, order, order-line, product, and address tables and open the ESB overview at `/qqq/v1/esb/overview`; Demo User can read those tables but cannot write them or open the ESB overview. These identities are for local demonstrations. Quick search is disabled in core because it requires OpenSearch; the customer list remains available through the normal table query API.
The core demo seeds a customer table view, a shipping address linked to the geo country/state/city tables, and an order review workflow. The `RunRecordWorkflow` process can run that workflow against an order; `syncOrder` writes a process trace after an order event. It also seeds an active `orderStored` webhook subscription to its receiver at `/demo/order-webhook`. Insert an order as Admin, run `SendWebhookEvent` as Admin, and inspect `/demo/order-webhook-receipts` to see the delivered payload. Demo User can read webhook records but cannot edit their configuration or invoke the delivery process. The receiver is available only in the core profile, which Compose publishes on host loopback.
@@ -35,7 +35,15 @@ Configure the following environment variables before starting the same jar:
The full app adds sample tables on each external backend, `orderEvents` on Artemis, and `orderSyncEvents` on RabbitMQ. Quick search is configured for customer `name` and `email`; run `quickSearchFullReindex` after OpenSearch and the customer data are ready. The SFTP integration adds its import source and staging tables plus `SFTPImportFileSyncProcess`; `externalImportFile` remains the direct SFTP file sample. Keycloak realm roles resolve by name against the PostgreSQL `role` table from `qbit-user-role-permissions`, then become QQQ session permissions. Unknown roles receive no permissions. Webhook tables and delivery processes require explicit permissions: the viewer role receives read access, while the admin role can edit destinations and run delivery. The Compose profile provisions demo SQL tables, OAuth session and redirect-state tables, qbit permission tables and roles, and the MinIO bucket on first startup; `infra/postgres/03-webhook-permissions.sql` adds webhook grants after the role seed.
On startup, the full demo creates its remaining PostgreSQL qbit tables from the registered metadata and checks every expected column and type. The hand-maintained domain, OAuth, and role tables stay under `infra/postgres`. If a persisted qbit table is missing a required column or has a different type, startup fails with a migration/reset message instead of silently accepting it. The full-profile RabbitMQ completion-event smoke remains gated on QQQ ESB Task 9's metadata-driven process lifecycle publication; the broker management smoke below can run independently.
The full-profile admin receives ESB service visibility, queue controls, and message deletion through `infra/postgres/04-esb-permissions.sql`; the viewer role does not. The full smoke checks the RabbitMQ process-completion event through QQQ's authenticated ESB message endpoint, which decodes the broker's JMS message. Fresh PostgreSQL volumes apply this SQL automatically. For a running full-profile stack with an existing volume, apply the idempotent grant file once:
```bash
COMPOSE_PROFILES=full docker compose exec -T postgres sh -c 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < infra/postgres/04-esb-permissions.sql
```
Then sign in as Admin and open the ESB overview. The Viewer role must remain unable to open it.
On startup, the full demo creates its remaining PostgreSQL qbit tables from the registered metadata and checks every expected column and type. The hand-maintained domain, OAuth, and role tables stay under `infra/postgres`. If a persisted qbit table is missing a required column or has a different type, startup fails with a migration/reset message instead of silently accepting it. The full-profile RabbitMQ completion-event smoke requires QQQ's process publishing and permission-scoped ESB message endpoint; the broker management smoke below can run independently.
After starting the full Compose profile, run the opt-in broker management smoke test with the same broker credentials. For the supplied demo environment:
@@ -47,3 +55,64 @@ QQQ_ALL_COMPOSE_BROKERS=true mvn -pl qqq-all-app -am -Dtest=FullProfileBrokerInt
```
The test uses the loopback management ports to read and pause/resume the Artemis subscription queue, then creates, reads, purges, and deletes a temporary RabbitMQ queue. For a customized `.env`, source that file instead. It is skipped during ordinary `mvn verify`.
## Local runtime demonstrations (#21)
Both profiles include the BOM-managed API backend and JavaScript module. The shaded
JAR retains Nashorn's `ScriptEngineFactory` service registration. No extra service,
API key, script editor, or user-supplied code is needed.
| Demo | Behavior | Access |
| --- | --- | --- |
| `apiCatalog` | QQQ API-backed table reads three synthetic products from this app's `/demo/catalog` HTTP endpoint; native queries page through the endpoint and get-by-ID uses `/demo/catalog/{id}`. | Admin and Demo/Viewer can read; neither receives write grants. |
| `calculateOrderTotal` | Executes fixed JavaScript through `ExecuteCodeAction`: quantity × unit price in cents → `totalCents`. It changes no records. | Admin only. |
| `demoNote` | Memory-backed table, seeded with `Transient demo note` on every start and cleared on shutdown. | Admin can edit; Demo/Viewer can read. |
The internal catalog URL follows `QQQ_ALL_BIND_HOST` and the configured HTTP port.
IPv4/IPv6 wildcard listeners use their corresponding loopback address; literal
IPv6 hosts are bracketed correctly in the URL.
The catalog endpoint exposes **only synthetic public demo data**. It is not an
outbound proxy and does not forward session credentials. The adapter supports
ascending ID order, limit/skip pagination, and one exact ID filter; unsupported
filters and sorts fail explicitly. Native get of an absent ID returns no record.
The endpoint rejects invalid or excessive page sizes (maximum 1000).
With the core JAR running on port 8080, exercise the real QQQ routes:
```sh
curl -fsS -H 'Content-Type: application/json' -d '{}' http://127.0.0.1:8080/qqq/v1/table/apiCatalog/query
curl -fsS http://127.0.0.1:8080/data/apiCatalog/103
curl -fsS -H 'Cookie: sessionId=aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' -H 'Content-Type: application/json' -d '{}' 'http://127.0.0.1:8080/processes/calculateOrderTotal/run?quantity=3&unitPriceCents=1250'
```
The last response contains `values.totalCents: 3750`. Quantity must be an integer
in 1..1000; unit price must be an integer in 0..1000000 cents. The process does not
accept code from the caller. This demonstration does not alter stored-script or
Test-mode behavior.
Memory notes are deliberately **not persisted** with the data directory or a
Compose volume. Restarting discards edits and restores the seed. Like the existing
QQQ memory and ESB runtime singletons, this demo assumes one application per JVM.
The memory lifecycle touches only `demoNote`; it never resets the global store.
Full-profile PostgreSQL now uses QQQ's first-party `PostgreSQLBackendMetaData` and
strategy, including driver/URL selection, quoted identifiers, default-value
inserts, generated IDs, null binding, and UTC timestamp handling. Existing qbit
schema provisioning and compatibility checks remain in place.
Fresh full-profile volumes apply `infra/postgres/05-runtime-demo-permissions.sql`.
For an existing volume, apply the idempotent grants and then sign in again:
```sh
COMPOSE_PROFILES=full docker compose exec -T postgres sh -c 'psql -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB"' < infra/postgres/05-runtime-demo-permissions.sql
```
`mvn -pl qqq-all-app -am verify` includes native local API, calculation/permission,
memory restart, and packaged-JAR checks. `scripts/run_core_smoke.py` and
`scripts/run_full_smoke.py` verify these routes with actual Admin and restricted
sessions; full smoke also retains the OIDC, qbit, and broker checks. The opt-in
`PostgresCompatibilityTest` uses `QQQ_ALL_TEST_POSTGRES_PORT` and
`QQQ_ALL_TEST_POSTGRES_PASSWORD` against a local `qqq` database/user; it creates and
drops only a uniquely named test table. It is skipped during ordinary verify.
All eight included qbits, their explicit deferrals, and release gates are unchanged.
+3
View File
@@ -30,7 +30,10 @@
<dependencies>
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-backend-core</artifactId></dependency>
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-backend-module-api</artifactId></dependency>
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-language-support-javascript</artifactId></dependency>
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-backend-module-rdbms</artifactId></dependency>
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-backend-module-postgres</artifactId></dependency>
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-backend-module-mongodb</artifactId></dependency>
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-backend-module-sqlite</artifactId></dependency>
<dependency><groupId>com.kingsrook.qqq</groupId><artifactId>qqq-backend-module-filesystem</artifactId></dependency>
@@ -0,0 +1,40 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import java.util.Map;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.actions.scripts.ExecuteCodeAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeInput;
import com.kingsrook.qqq.backend.core.model.actions.scripts.ExecuteCodeOutput;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType;
/** Execute a fixed, side-effect-free JavaScript calculation through QQQ. */
public class CalculateOrderTotalStep implements BackendStep
{
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
Integer quantity = input.getValueInteger("quantity");
Integer unitPriceCents = input.getValueInteger("unitPriceCents");
if(quantity == null || quantity < 1 || quantity > 1000
|| unitPriceCents == null || unitPriceCents < 0 || unitPriceCents > 1_000_000)
{
throw new QException("Supply quantity 1..1000 and unitPriceCents 0..1000000");
}
ExecuteCodeOutput result = new ExecuteCodeOutput();
new ExecuteCodeAction().run(new ExecuteCodeInput()
.withCodeReference(new QCodeReference().withCodeType(QCodeType.JAVA_SCRIPT)
.withInlineCode("return quantity * unitPriceCents;"))
.withInput(Map.of("quantity", quantity, "unitPriceCents", unitPriceCents)), result);
output.addValue("totalCents", ((Number) result.getOutput()).intValue());
}
}
@@ -45,7 +45,7 @@ final class CoreRuntime implements AutoCloseable
Path brokerDirectory = dataDirectory.resolve("artemis").toAbsolutePath();
String brokerUrl = "tcp://127.0.0.1:" + brokerPort;
DemoOrderWebhookReceiver.clear();
DemoQBitData.ensureSchema(dataDirectory, new QqqAllApplication(dataDirectory, brokerUrl).defineQInstance(), port);
DemoQBitData.ensureSchema(dataDirectory, new QqqAllApplication(dataDirectory, brokerUrl, true, port, bindHost).defineQInstance(), port);
Configuration configuration = new ConfigurationImpl()
.setPersistenceEnabled(false).setSecurityEnabled(false).setJMXManagementEnabled(false)
.setBindingsDirectory(brokerDirectory.resolve("bindings").toString())
@@ -58,7 +58,7 @@ final class CoreRuntime implements AutoCloseable
try
{
QApplicationLauncher launcher = QApplicationLauncher.run(
new QqqAllApplication(dataDirectory, brokerUrl),
new QqqAllApplication(dataDirectory, brokerUrl, true, port, bindHost),
new QApplicationLauncherConfig().withRegisterShutdownHook(false)
.withServerCustomizer(server -> server.withPort(port)
.withJavalinConfigCustomizer(config -> config.jetty.host = bindHost)));
@@ -0,0 +1,82 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import java.io.Serializable;
import java.util.Map;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.module.api.actions.BaseAPIActionUtil;
/** The local catalog supports ascending ID pages and optional exact ID selection. */
public class DemoCatalogApi extends BaseAPIActionUtil
{
@Override
public String buildUrlSuffixForSingleRecordGet(Serializable primaryKey) throws QException
{
try
{
return "/" + Integer.parseInt(String.valueOf(primaryKey));
}
catch(NumberFormatException e)
{
throw new QException("Invalid catalog ID", e);
}
}
@Override
protected Integer getApiStandardLimit()
{
return 2;
}
@Override
protected String buildQueryStringForGet(QQueryFilter filter, Integer limit, Integer skip, Map<String, QFieldMetaData> fields) throws QException
{
String query = "?limit=" + (limit == null ? 2 : limit) + "&skip=" + (skip == null ? 0 : skip);
if(filter != null)
{
if(!filter.getSubFilters().isEmpty() || filter.getBooleanOperator() != QQueryFilter.BooleanOperator.AND
|| filter.getCriteria().size() > 1 || filter.getOrderBys().stream().anyMatch(order -> !"id".equals(order.getFieldName()) || !order.getIsAscending()))
{
throw new QException("Demo catalog supports ascending ID order and one exact ID filter only");
}
if(!filter.getCriteria().isEmpty())
{
var criterion = filter.getCriteria().getFirst();
if(!"id".equals(criterion.getFieldName()) || criterion.getOperator() != QCriteriaOperator.EQUALS || criterion.getValues().size() != 1)
{
throw new QException("Demo catalog supports one exact ID filter only");
}
try
{
query += "&id=" + Integer.parseInt(String.valueOf(criterion.getValues().getFirst()));
}
catch(NumberFormatException e)
{
throw new QException("Invalid catalog ID", e);
}
}
}
return query;
}
@Override
public CountOutput doCount(QTableMetaData table, CountInput input) throws QException
{
QQueryFilter filter = input.getFilter() == null ? new QQueryFilter() : input.getFilter().clone();
filter.setLimit(null);
filter.setSkip(null);
return new CountOutput().withCount(doQuery(table, new QueryInput(table.getName()).withFilter(filter)).getRecords().size());
}
}
@@ -0,0 +1,76 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.middleware.javalin.QJavalinRouteProviderInterface;
import io.javalin.config.JavalinConfig;
import static io.javalin.apibuilder.ApiBuilder.get;
/** Synthetic public supplier data, owned by the same server as the reference app. */
public class DemoCatalogRoutes implements QJavalinRouteProviderInterface
{
private static final List<Map<String, Object>> CATALOG = List.of(
Map.of("id", 101, "name", "Starter Kit", "unitPriceCents", 2500),
Map.of("id", 102, "name", "Refill Pack", "unitPriceCents", 1250),
Map.of("id", 103, "name", "Shipping Box", "unitPriceCents", 300));
@Override
public void setQInstance(QInstance instance)
{
}
@Override
public void acceptJavalinConfig(JavalinConfig config)
{
config.routes.apiBuilder(() ->
{
get("/demo/catalog/{id}", context ->
{
try
{
int id = Integer.parseInt(context.pathParam("id"));
var row = CATALOG.stream().filter(item -> Integer.valueOf(id).equals(item.get("id"))).findFirst();
if(row.isPresent())
{
context.json(row.get());
}
else
{
context.status(404);
}
}
catch(NumberFormatException e)
{
context.status(400);
}
});
get("/demo/catalog", context ->
{
try
{
int skip = Integer.parseInt(context.queryParamAsClass("skip", String.class).getOrDefault("0"));
int limit = Integer.parseInt(context.queryParamAsClass("limit", String.class).getOrDefault("2"));
Integer id = context.queryParam("id") == null ? null : Integer.valueOf(context.queryParam("id"));
if(skip < 0 || limit < 1 || limit > 1000)
{
context.status(400);
return;
}
context.json(CATALOG.stream().filter(row -> id == null || id.equals(row.get("id")))
.skip(skip).limit(limit).toList());
}
catch(NumberFormatException e)
{
context.status(400);
}
});
});
}
}
@@ -0,0 +1,90 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionOutputMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
import com.kingsrook.qqq.backend.module.api.model.AuthorizationType;
import com.kingsrook.qqq.backend.module.api.model.metadata.APIBackendMetaData;
import com.kingsrook.qqq.backend.module.api.model.metadata.APITableBackendDetails;
import com.kingsrook.qqq.middleware.javalin.QJavalinMetaData;
/** Shared, local-only demonstrations included in both profiles. */
final class DemoRuntimeFeatures
{
private DemoRuntimeFeatures()
{
}
static void add(QInstance instance, int port, String bindHost) throws QException
{
instance.addBackend(new APIBackendMetaData().withName("demoApi")
.withBaseUrl(catalogBaseUrl(bindHost, port))
.withAuthorizationType(AuthorizationType.NONE).withContentType("application/json")
.withActionUtil(new QCodeReference(DemoCatalogApi.class)));
instance.addTable(new QTableMetaData().withName("apiCatalog").withLabel("API Catalog")
.withBackendName("demoApi").withPrimaryKeyField("id")
.withBackendDetails(new APITableBackendDetails().withTablePath("/catalog"))
.withCapabilities(Capability.TABLE_QUERY, Capability.TABLE_GET, Capability.TABLE_COUNT)
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("name", QFieldType.STRING))
.withField(new QFieldMetaData("unitPriceCents", QFieldType.INTEGER)));
instance.addBackend(new QBackendMetaData().withName("transientMemory").withBackendType(MemoryBackendModule.class));
instance.addTable(new QTableMetaData().withName("demoNote").withLabel("Demo Note (Transient)")
.withBackendName("transientMemory").withPrimaryKeyField("id")
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_INSERT_EDIT_DELETE_PERMISSIONS))
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("text", QFieldType.STRING)));
instance.withRuntimeService(new QCodeReference(TransientNotesService.class));
instance.addProcess(new QProcessMetaData().withName("calculateOrderTotal").withLabel("Calculate Order Total (JavaScript)")
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION))
.withStep(new QBackendStepMetaData().withName("calculate")
.withCode(new QCodeReference(CalculateOrderTotalStep.class))
.withInputData(new QFunctionInputMetaData().withFieldList(List.of(
new QFieldMetaData("quantity", QFieldType.INTEGER),
new QFieldMetaData("unitPriceCents", QFieldType.INTEGER))))
.withOutputMetaData(new QFunctionOutputMetaData().withFieldList(List.of(
new QFieldMetaData("totalCents", QFieldType.INTEGER))))));
QJavalinMetaData.ofOrWithNew(instance).withAdditionalRouteProviderReference(new QCodeReference(DemoCatalogRoutes.class));
}
private static String catalogBaseUrl(String bindHost, int port) throws QException
{
try
{
InetAddress address = InetAddress.getByName(bindHost);
String host = address.isAnyLocalAddress()
? (address instanceof Inet6Address ? "::1" : "127.0.0.1") : bindHost;
return new URI("http", null, host, port, "/demo", null, null).toASCIIString();
}
catch(UnknownHostException | URISyntaxException e)
{
throw new QException("Could not derive demo catalog URL from HTTP bind host", e);
}
}
}
@@ -53,6 +53,7 @@ public class DemoUsers implements QAuthenticationModuleCustomizerInterface
@Override
public void customizeSession(QInstance instance, QSession session, Map<String, Object> context)
{
session.withPermissions("apiCatalog.read", "demoNote.read");
Boolean admin = ADMIN_SESSION.equals(session.getUuid());
String name = admin ? "Admin" : "Demo User";
String userId = admin ? "core:admin" : "core:demo";
@@ -76,6 +77,8 @@ public class DemoUsers implements QAuthenticationModuleCustomizerInterface
}
if(admin)
{
session.withPermissions("calculateOrderTotal.hasAccess", "demoNote.insert", "demoNote.edit", "demoNote.delete");
session.withPermissions("esbView.hasAccess", "esbOperate.hasAccess", "esbDelete.hasAccess");
for(String processName : WEBHOOK_PROCESSES)
{
session.withPermission(processName + ".hasAccess");
@@ -34,6 +34,7 @@ import com.kingsrook.qqq.backend.module.filesystem.sftp.model.metadata.SFTPTable
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBBackendMetaData;
import com.kingsrook.qqq.backend.module.mongodb.model.metadata.MongoDBTableBackendDetails;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.qqq.backend.module.postgres.model.metadata.PostgreSQLBackendMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
import com.kingsrook.qqq.esb.model.EsbDestinationType;
import com.kingsrook.qqq.esb.model.EsbInstanceMetaData;
@@ -66,7 +67,7 @@ final class FullProfileApplication extends AbstractQQQApplication
@Override
public QInstance defineQInstance() throws QException
{
QInstance instance = new QqqAllApplication(dataDirectory, settings.get("ARTEMIS_URL"), false).defineQInstance();
QInstance instance = new QqqAllApplication(dataDirectory, settings.get("ARTEMIS_URL"), false, settings.port("PORT", "8080"), settings.get("BIND_HOST")).defineQInstance();
addBackends(instance);
addTables(instance);
addAuthentication(instance);
@@ -111,13 +112,9 @@ final class FullProfileApplication extends AbstractQQQApplication
private void addBackends(QInstance instance)
{
instance.addBackend(new RDBMSBackendMetaData().withName("postgres").withVendor("postgres")
instance.addBackend(new PostgreSQLBackendMetaData().withName("postgres")
.withHostName(settings.get("POSTGRES_HOST")).withPort(settings.port("POSTGRES_PORT", "5432"))
.withDatabaseName(settings.get("POSTGRES_DATABASE"))
.withJdbcUrl("jdbc:postgresql://" + settings.get("POSTGRES_HOST") + ":"
+ settings.port("POSTGRES_PORT", "5432") + "/" + settings.get("POSTGRES_DATABASE"))
.withJdbcDriverClassName("org.postgresql.Driver")
.withActionStrategyCodeReference(new QCodeReference(PostgresActionStrategy.class))
.withUsername(settings.get("POSTGRES_USER")).withPassword(settings.get("POSTGRES_PASSWORD")));
instance.addBackend(new RDBMSBackendMetaData().withName("mysql").withVendor("mysql")
.withHostName(settings.get("MYSQL_HOST")).withPort(settings.port("MYSQL_PORT", "3306"))
@@ -1,39 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Types;
import com.kingsrook.qqq.backend.module.rdbms.strategy.BaseRDBMSActionStrategy;
/** PostgreSQL syntax for the shared QQQ RDBMS actions. */
public class PostgresActionStrategy extends BaseRDBMSActionStrategy
{
@Override
public String getIdentifierQuoteString()
{
return "\"";
}
@Override
public String getInsertDefaultValuesClause()
{
return "DEFAULT VALUES";
}
@Override
protected int bindParamObject(PreparedStatement statement, int index, Object value) throws SQLException
{
if(value == null)
{
statement.setNull(index, Types.NULL);
return 1;
}
return super.bindParamObject(statement, index, value);
}
}
@@ -14,6 +14,7 @@ import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.AuthScope;
@@ -24,6 +25,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerKeyRecordMessage;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.Cardinality;
import com.kingsrook.qqq.backend.module.filesystem.base.model.metadata.RecordFormat;
import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.FilesystemBackendMetaData;
@@ -31,6 +33,9 @@ import com.kingsrook.qqq.backend.module.filesystem.local.model.metadata.Filesyst
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
import com.kingsrook.qqq.backend.module.sqlite.model.metadata.SQLiteBackendMetaData;
import com.kingsrook.qqq.esb.envelope.EsbEvent;
import com.kingsrook.qqq.esb.api.EsbJavalinMetaDataProducer;
import com.kingsrook.qqq.esb.metadata.EsbAppMetaDataProducer;
import com.kingsrook.qqq.esb.model.EsbDestinationType;
import com.kingsrook.qqq.esb.model.EsbInstanceMetaData;
import com.kingsrook.qqq.esb.model.EsbProcessMetaData;
@@ -41,6 +46,7 @@ import com.kingsrook.qqq.esb.model.EsbTablePublication;
import com.kingsrook.qqq.esb.model.EsbTrigger;
import com.kingsrook.qqq.esb.model.QEsbDestinationMetaData;
import com.kingsrook.qqq.esb.model.QEsbProviderMetaData;
import com.kingsrook.qqq.esb.runtime.EsbTriggerHandler;
import com.kingsrook.qqq.middleware.health.JavalinHealthRouteProvider;
import com.kingsrook.qqq.middleware.health.indicators.MemoryHealthIndicator;
import com.kingsrook.qqq.middleware.health.model.metadata.HealthCheckMetaData;
@@ -52,6 +58,8 @@ public final class QqqAllApplication extends AbstractQQQApplication
private final Path dataDirectory;
private final String brokerUrl;
private final boolean includeDemoQbits;
private final int httpPort;
private final String bindHost;
public QqqAllApplication(Path dataDirectory, String brokerUrl)
{
@@ -60,6 +68,18 @@ public final class QqqAllApplication extends AbstractQQQApplication
QqqAllApplication(Path dataDirectory, String brokerUrl, boolean includeDemoQbits)
{
this(dataDirectory, brokerUrl, includeDemoQbits, 8080);
}
QqqAllApplication(Path dataDirectory, String brokerUrl, boolean includeDemoQbits, int httpPort)
{
this(dataDirectory, brokerUrl, includeDemoQbits, httpPort, "127.0.0.1");
}
QqqAllApplication(Path dataDirectory, String brokerUrl, boolean includeDemoQbits, int httpPort, String bindHost)
{
this.bindHost = bindHost;
this.httpPort = httpPort;
this.dataDirectory = dataDirectory;
this.brokerUrl = brokerUrl;
this.includeDemoQbits = includeDemoQbits;
@@ -132,6 +152,8 @@ public final class QqqAllApplication extends AbstractQQQApplication
.withType(EsbDestinationType.TOPIC).withProviderName("artemis")
.withDestinationName("qqq.all.orderEvents"));
instance.withRuntimeService(new QCodeReference(CoreEsbRuntimeService.class));
MetaDataProducerHelper.processAllMetaDataProducersInPackage(instance, EsbAppMetaDataProducer.class.getPackageName());
new EsbJavalinMetaDataProducer().produce(instance);
instance.withSupplementalMetaData(new HealthCheckMetaData().withEnabled(true)
.withEndpointPath("/health").withIndicators(List.of(new MemoryHealthIndicator().withThreshold(99))));
@@ -144,6 +166,7 @@ public final class QqqAllApplication extends AbstractQQQApplication
QJavalinMetaData.ofOrWithNew(instance)
.withAdditionalRouteProviderReference(new QCodeReference(DemoOrderWebhookReceiver.class));
}
DemoRuntimeFeatures.add(instance, httpPort, bindHost);
return instance;
}
@@ -155,6 +178,19 @@ public final class QqqAllApplication extends AbstractQQQApplication
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
if(input.getValue(EsbTriggerHandler.VALUE_ESB_MESSAGES) instanceof List<?> messages)
{
for(Object message : messages)
{
if(message instanceof EsbEvent event
&& event.getSource().endsWith("/table/order") && event.getSubject() != null)
{
Integer orderId = Integer.valueOf(event.getSubject());
input.getProcessTracer().ifPresent(tracer -> tracer.handleMessage(input,
new ProcessTracerKeyRecordMessage("order", orderId)));
}
}
}
RUN_COUNT.incrementAndGet();
}
@@ -0,0 +1,88 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QRuntimeServiceInterface;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
/** Seed and clear only the application's transient memory table. */
public class TransientNotesService implements QRuntimeServiceInterface
{
private static final QLogger LOG = QLogger.getLogger(TransientNotesService.class);
private QInstance instance;
@Override
public String getName()
{
return "transientDemoNotes";
}
@Override
public void start(QInstance instance) throws QException
{
this.instance = instance;
var previous = QContext.capture();
try
{
QContext.init(instance, new QSystemUserSession());
clearNotes();
new InsertAction().execute(new InsertInput("demoNote").withRecords(List.of(
new QRecord().withValue("id", 1).withValue("text", "Transient demo note"))));
}
catch(QException | RuntimeException failure)
{
try
{
clearNotes();
}
catch(Exception cleanup)
{
failure.addSuppressed(cleanup);
}
throw failure;
}
finally
{
QContext.init(previous);
}
}
@Override
public void stop()
{
var previous = QContext.capture();
try
{
QContext.init(instance, new QSystemUserSession());
clearNotes();
}
catch(Exception failure)
{
LOG.warn("Could not clear transient demo notes", failure);
}
finally
{
QContext.init(previous);
}
}
private void clearNotes() throws QException
{
new DeleteAction().execute(new DeleteInput("demoNote").withQueryFilter(new QQueryFilter()));
}
}
@@ -33,6 +33,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Exercises the assembled core application, including its real broker and backends. */
@@ -60,6 +61,19 @@ class CoreApplicationTest
assertEquals(200, health.statusCode());
assertTrue(health.body().contains("\"UP\""));
assertNotNull(runtime.getLauncher().getQInstance().getApp("esb"));
HttpResponse<String> adminEsb = client.send(HttpRequest.newBuilder()
.uri(URI.create("http://127.0.0.1:" + port + "/qqq/v1/esb/overview"))
.header("Cookie", "sessionId=" + DemoUsers.ADMIN_SESSION).build(),
HttpResponse.BodyHandlers.ofString());
assertEquals(200, adminEsb.statusCode(), adminEsb.body());
assertTrue(adminEsb.body().contains("\"destinations\""));
HttpResponse<String> viewerEsb = client.send(HttpRequest.newBuilder()
.uri(URI.create("http://127.0.0.1:" + port + "/qqq/v1/esb/overview"))
.header("Cookie", "sessionId=" + DemoUsers.DEMO_SESSION).build(),
HttpResponse.BodyHandlers.ofString());
assertEquals(403, viewerEsb.statusCode());
HttpResponse<String> dashboard = client.send(HttpRequest.newBuilder()
.uri(URI.create("http://localhost:" + port + "/")).build(),
HttpResponse.BodyHandlers.ofString());
@@ -108,11 +122,13 @@ class CoreApplicationTest
waitFor(() -> QEsbRuntime.getInstance().getRunner("syncOrder.orderEvents") != null
&& QEsbRuntime.getInstance().getRunner("syncOrder.orderEvents").getState() == EsbTriggerState.RUNNING);
new InsertAction().execute(new InsertInput("order").withRecords(List.of(
Integer insertedOrderId = new InsertAction().execute(new InsertInput("order").withRecords(List.of(
new QRecord().withValue("orderNo", "ORD-1002")
.withValue("customerId", 2).withValue("status", "NEW"))));
.withValue("customerId", 2).withValue("status", "NEW"))))
.getRecords().getFirst().getValueInteger("id");
waitFor(() -> QqqAllApplication.SyncOrderStep.getRunCount() > 0);
waitFor(() -> !new QueryAction().execute(new QueryInput("processTrace")).getRecords().isEmpty());
waitFor(() -> new QueryAction().execute(new QueryInput("processTrace")).getRecords().stream()
.anyMatch(trace -> insertedOrderId.equals(trace.getValueInteger("keyRecordId"))));
waitFor(() -> !new QueryAction().execute(new QueryInput("webhookEvent")).getRecords().isEmpty());
HttpResponse<String> receipts = client.send(HttpRequest.newBuilder()
@@ -16,6 +16,8 @@ import java.nio.file.Path;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.junit.jupiter.api.io.TempDir;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -39,28 +41,56 @@ class CoreJarIT
Path log = dataDirectory.resolve("app.log");
ProcessBuilder builder = new ProcessBuilder(
Path.of(System.getProperty("java.home"), "bin", "java").toString(), "-jar", jar.toString());
builder.environment().put("QQQ_ALL_PROFILE", "core");
builder.environment().put("QQQ_ALL_PORT", port.toString());
builder.environment().put("QQQ_ALL_DATA_DIR", dataDirectory.resolve("data").toString());
builder.redirectErrorStream(true).redirectOutput(log.toFile());
Process process = builder.start();
try
{
HttpClient client = HttpClient.newHttpClient();
URI healthUri = URI.create("http://127.0.0.1:" + port + "/health");
HttpResponse<String> health = awaitHealth(client, healthUri, process, log);
assertEquals(200, health.statusCode());
assertTrue(health.body().contains("\"UP\""));
assertTrue(Files.readString(log).contains("{127.0.0.1:" + port + "}"),
"Core profile must bind only to loopback");
try(HttpClient client = HttpClient.newHttpClient())
{
URI healthUri = URI.create("http://127.0.0.1:" + port + "/health");
HttpResponse<String> health = awaitHealth(client, healthUri, process, log);
assertEquals(200, health.statusCode());
assertTrue(health.body().contains("\"UP\""));
assertTrue(Files.readString(log).contains("{127.0.0.1:" + port + "}"),
"Core profile must bind only to loopback");
HttpRequest query = HttpRequest.newBuilder()
.uri(URI.create("http://127.0.0.1:" + port + "/qqq/v1/table/customer/query"))
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString("{}"))
.build();
HttpResponse<String> customers = client.send(query, HttpResponse.BodyHandlers.ofString());
assertEquals(200, customers.statusCode(), customers.body());
assertTrue(customers.body().contains("Ada Lovelace"));
HttpRequest query = HttpRequest.newBuilder()
.uri(URI.create("http://127.0.0.1:" + port + "/qqq/v1/table/customer/query"))
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString("{}"))
.build();
HttpResponse<String> customers = client.send(query, HttpResponse.BodyHandlers.ofString());
assertEquals(200, customers.statusCode(), customers.body());
assertTrue(customers.body().contains("Ada Lovelace"));
for(String table : java.util.List.of("apiCatalog", "demoNote"))
{
var result = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port
+ "/qqq/v1/table/" + table + "/query")).header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString("{}")).build(), HttpResponse.BodyHandlers.ofString());
assertEquals(200, result.statusCode(), result.body());
var records = new org.json.JSONObject(result.body()).getJSONArray("records");
assertEquals(table.equals("apiCatalog") ? 3 : 1, records.length());
}
var calculation = client.send(HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port
+ "/processes/calculateOrderTotal/run?quantity=3&unitPriceCents=1250"))
.header("Cookie", "sessionId=" + DemoUsers.ADMIN_SESSION).header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString("{}")).build(), HttpResponse.BodyHandlers.ofString());
assertEquals(200, calculation.statusCode(), calculation.body());
assertEquals(3750, new org.json.JSONObject(calculation.body()).getJSONObject("values").getInt("totalCents"));
}
try(var packaged = new java.util.jar.JarFile(jar.toFile()))
{
var service = packaged.getJarEntry("META-INF/services/javax.script.ScriptEngineFactory");
org.junit.jupiter.api.Assertions.assertNotNull(service);
try(var input = packaged.getInputStream(service))
{
assertTrue(new String(input.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)
.contains("org.openjdk.nashorn.api.scripting.NashornScriptEngineFactory"));
}
}
}
finally
{
@@ -73,6 +103,69 @@ class CoreJarIT
}
}
@ParameterizedTest
@ValueSource(strings = {"::1", "::", "0:0:0:0:0:0:0:0", "0.0.0.0"})
void packagedCatalogUsesConfiguredListener(String bindHost) throws Exception
{
String clientHost = bindHost.contains(":") ? "::1" : "127.0.0.1";
int port;
try(ServerSocket socket = new ServerSocket(0, 50, java.net.InetAddress.getByName(clientHost)))
{
port = socket.getLocalPort();
}
URI base = new URI("http", null, clientHost, port, "/", null, null);
Path log = dataDirectory.resolve("bound-app.log");
ProcessBuilder builder = new ProcessBuilder(Path.of(System.getProperty("java.home"), "bin", "java").toString(),
"-jar", System.getProperty("qqqAllJar"));
builder.environment().put("QQQ_ALL_PROFILE", "core");
builder.environment().put("QQQ_ALL_BIND_HOST", bindHost);
builder.environment().put("QQQ_ALL_PORT", Integer.toString(port));
builder.environment().put("QQQ_ALL_DATA_DIR", dataDirectory.resolve("bound-data").toString());
builder.redirectErrorStream(true).redirectOutput(log.toFile());
Process process = builder.start();
try(HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build())
{
assertEquals(200, awaitHealth(client, base.resolve("health"), process, log).statusCode());
var direct = client.send(HttpRequest.newBuilder(base.resolve("demo/catalog"))
.timeout(Duration.ofSeconds(10)).build(), HttpResponse.BodyHandlers.ofString());
assertEquals(200, direct.statusCode(), "Configured listener must serve its catalog before testing the backend");
assertEquals(2, new org.json.JSONArray(direct.body()).length());
var all = catalogQuery(client, base, "{}");
assertEquals(java.util.List.of(101, 102, 103), all.toList().stream()
.map(row -> ((java.util.Map<?, ?>) ((java.util.Map<?, ?>) row).get("values")).get("id")).toList());
for(int skip = 0; skip < 3; skip++)
{
var page = catalogQuery(client, base, "{\"filter\":{\"skip\":" + skip + ",\"limit\":1}}");
assertEquals(1, page.length());
assertEquals(java.util.List.of(101, 102, 103).get(skip), page.getJSONObject(0).getJSONObject("values").getInt("id"));
}
assertEquals(0, catalogQuery(client, base, "{\"filter\":{\"skip\":3,\"limit\":1}}").length());
var get = client.send(HttpRequest.newBuilder(base.resolve("data/apiCatalog/103"))
.timeout(Duration.ofSeconds(10)).build(), HttpResponse.BodyHandlers.ofString());
assertEquals(200, get.statusCode(), get.body());
assertEquals("Shipping Box", new org.json.JSONObject(get.body()).getJSONObject("values").getString("name"));
}
finally
{
process.destroy();
if(!process.waitFor(5, TimeUnit.SECONDS))
{
process.destroyForcibly();
assertTrue(process.waitFor(5, TimeUnit.SECONDS), "Owned packaged process must stop");
}
}
}
private static org.json.JSONArray catalogQuery(HttpClient client, URI base, String body) throws Exception
{
var response = client.send(HttpRequest.newBuilder(base.resolve("qqq/v1/table/apiCatalog/query"))
.timeout(Duration.ofSeconds(10)).header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(body)).build(), HttpResponse.BodyHandlers.ofString());
assertEquals(200, response.statusCode(), response.body());
return new org.json.JSONObject(response.body()).getJSONArray("records");
}
private static HttpResponse<String> awaitHealth(HttpClient client, URI uri, Process process, Path log) throws Exception
{
long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos();
@@ -45,6 +45,7 @@ class FullProfileTest
new QInstanceValidator().validate(instance);
RDBMSBackendMetaData postgres = (RDBMSBackendMetaData) instance.getBackend("postgres");
assertEquals("com.kingsrook.qqq.backend.module.postgres.model.metadata.PostgreSQLBackendMetaData", postgres.getClass().getName());
assertEquals("postgres.example.test", postgres.getHostName());
assertEquals("jdbc:postgresql://postgres.example.test:5432/qqq", ConnectionManager.getJdbcUrl(postgres));
assertEquals("org.postgresql.Driver", ConnectionManager.getJdbcDriverClassName(postgres));
@@ -0,0 +1,101 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import java.nio.file.Path;
import java.sql.DriverManager;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
import org.junit.jupiter.api.io.TempDir;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
/** Opt-in native PostgreSQL oracle; owns only its uniquely named table. */
@EnabledIfEnvironmentVariable(named = "QQQ_ALL_TEST_POSTGRES_PORT", matches = "[0-9]+")
@Timeout(30)
class PostgresCompatibilityTest
{
@TempDir
Path directory;
@Test
void nativeStrategyPreservesGeneratedIdsNullsAndUtcTimestamps() throws Exception
{
var environment = FullProfileTest.environment();
environment.put("QQQ_ALL_POSTGRES_HOST", "127.0.0.1");
environment.put("QQQ_ALL_POSTGRES_PORT", System.getenv("QQQ_ALL_TEST_POSTGRES_PORT"));
environment.put("QQQ_ALL_POSTGRES_DATABASE", "qqq");
environment.put("QQQ_ALL_POSTGRES_USER", "qqq");
environment.put("QQQ_ALL_POSTGRES_PASSWORD", System.getenv("QQQ_ALL_TEST_POSTGRES_PASSWORD"));
var instance = new FullProfileApplication(directory, FullProfileSettings.from(environment)).defineQInstance();
var backend = (RDBMSBackendMetaData) instance.getBackend("postgres");
String table = "compat_" + UUID.randomUUID().toString().replace("-", "");
instance.addTable(new QTableMetaData().withName("compatibility").withBackendName("postgres")
.withBackendDetails(new RDBMSTableBackendDetails().withTableName(table)).withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER))
.withField(new QFieldMetaData("value", QFieldType.STRING))
.withField(new QFieldMetaData("created", QFieldType.DATE_TIME)));
instance.addTable(new QTableMetaData().withName("compatibilityDefault").withBackendName("postgres")
.withBackendDetails(new RDBMSTableBackendDetails().withTableName(table)).withPrimaryKeyField("id")
.withField(new QFieldMetaData("id", QFieldType.INTEGER)));
var previous = QContext.capture();
try(var connection = DriverManager.getConnection(ConnectionManager.getJdbcUrl(backend), backend.getUsername(), backend.getPassword());
var statement = connection.createStatement())
{
statement.execute("CREATE TABLE " + table + " (id SERIAL PRIMARY KEY, value TEXT, created TIMESTAMP)");
try
{
QContext.init(instance, new QSystemUserSession());
var empty = new InsertAction().execute(new InsertInput("compatibilityDefault").withRecords(List.of(new QRecord()))).getRecords().getFirst();
assertNotNull(empty.getValueInteger("id"));
Instant timestamp = Instant.parse("2026-09-25T12:34:56Z");
var row = new InsertAction().execute(new InsertInput("compatibility").withRecords(List.of(
new QRecord().withValue("value", "demo").withValue("created", timestamp)))).getRecords().getFirst();
assertTrue(row.getValueInteger("id") > empty.getValueInteger("id"));
new UpdateAction().execute(new UpdateInput("compatibility").withRecords(List.of(
new QRecord().withValue("id", row.getValueInteger("id")).withValue("value", null))));
assertNull(GetAction.execute("compatibility", row.getValueInteger("id")).getValue("value"));
try(var result = statement.executeQuery("SELECT value, created FROM " + table + " WHERE id = " + row.getValueInteger("id")))
{
assertTrue(result.next());
assertNull(result.getString(1));
assertEquals(timestamp, result.getTimestamp(2, java.util.Calendar.getInstance(java.util.TimeZone.getTimeZone("UTC"))).toInstant());
assertFalse(result.next());
}
}
finally
{
statement.execute("DROP TABLE " + table);
}
}
finally
{
QContext.init(previous);
}
}
}
@@ -0,0 +1,24 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import javax.script.ScriptEngineManager;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertNotNull;
/** The reference application must ship its advertised backend and language. */
class RuntimeAssemblyTest
{
@Test
void includesApiBackendAndJavaScriptEngine()
{
assertDoesNotThrow(() -> Class.forName("com.kingsrook.qqq.backend.module.api.APIBackendModule"));
assertDoesNotThrow(() -> Class.forName("com.kingsrook.qqq.languages.javascript.QJavaScriptExecutor"));
assertNotNull(new ScriptEngineManager().getEngineByName("JavaScript"));
}
}
@@ -0,0 +1,157 @@
/*
* Copyright 2026 Kingsrook, LLC
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at https://www.apache.org/licenses/LICENSE-2.0
*/
package com.kingsrook.qqq.all;
import java.net.ServerSocket;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Path;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.esb.connection.EsbConnectionManager;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.api.io.TempDir;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
/** Exercise the assembled demo against its real local HTTP backend. */
@Timeout(45)
class RuntimeFeaturesTest
{
@TempDir
Path directory;
@AfterEach
void closeOwnedBrokerConnections()
{
EsbConnectionManager.getInstance().closeAll();
}
@Test
void apiCatalogQueriesGetsAndPaginates() throws Exception
{
try(CoreRuntime runtime = CoreRuntime.launch(directory, freePort()))
{
assertNotNull(runtime.getLauncher().getQInstance().getTable("apiCatalog"));
QContext.init(runtime.getLauncher().getQInstance(), new QSystemUserSession());
try
{
assertEquals(List.of(101, 102, 103), new QueryAction().execute(new QueryInput("apiCatalog")).getRecords()
.stream().map(record -> record.getValueInteger("id")).toList());
assertEquals(102, new QueryAction().execute(new QueryInput("apiCatalog")
.withFilter(new QQueryFilter().withSkip(1).withLimit(1))).getRecords().getFirst().getValueInteger("id"));
assertEquals("Shipping Box", GetAction.execute("apiCatalog", 103).getValueString("name"));
assertNull(GetAction.execute("apiCatalog", 999));
assertEquals(3, new CountAction().execute(new CountInput("apiCatalog")).getCount());
var exactId = new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, 103));
assertEquals(1, new CountAction().execute(new CountInput("apiCatalog").withFilter(exactId)).getCount());
assertEquals(List.of(103), new QueryAction().execute(new QueryInput("apiCatalog").withFilter(exactId)).getRecords()
.stream().map(record -> record.getValueInteger("id")).toList());
assertThrows(QException.class, () -> new QueryAction().execute(new QueryInput("apiCatalog")
.withFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "Starter Kit")))));
}
finally
{
QContext.clear();
}
}
}
@Test
void calculationUsesRealJavaScriptAndRequiresPermission() throws Exception
{
int port = freePort();
try(CoreRuntime runtime = CoreRuntime.launch(directory, port); HttpClient client = HttpClient.newHttpClient())
{
assertNotNull(runtime.getLauncher().getQInstance().getProcess("calculateOrderTotal"));
HttpRequest.Builder request = HttpRequest.newBuilder(URI.create("http://127.0.0.1:" + port
+ "/processes/calculateOrderTotal/run?quantity=3&unitPriceCents=1250"))
.header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofString("{}"));
assertEquals(403, client.send(request.header("Cookie", "sessionId=" + DemoUsers.DEMO_SESSION).build(), HttpResponse.BodyHandlers.ofString()).statusCode());
var allowed = client.send(request.setHeader("Cookie", "sessionId=" + DemoUsers.ADMIN_SESSION).build(), HttpResponse.BodyHandlers.ofString());
assertEquals(200, allowed.statusCode(), allowed.body());
assertEquals(3750, new org.json.JSONObject(allowed.body()).getJSONObject("values").getInt("totalCents"));
}
}
@Test
void calculationRejectsMissingNegativeAndExcessiveInputs()
{
for(Integer quantity : java.util.Arrays.asList(null, 0, -1, 1001))
{
var input = new RunBackendStepInput().addValue("quantity", quantity).addValue("unitPriceCents", 1250);
assertThrows(QException.class, () -> new CalculateOrderTotalStep().run(input, new RunBackendStepOutput()));
}
for(Integer price : java.util.Arrays.asList(null, -1, 1_000_001))
{
var input = new RunBackendStepInput().addValue("quantity", 3).addValue("unitPriceCents", price);
assertThrows(QException.class, () -> new CalculateOrderTotalStep().run(input, new RunBackendStepOutput()));
}
}
@Test
void memoryNotesResetAcrossRuntimeRestart() throws Exception
{
try(CoreRuntime runtime = CoreRuntime.launch(directory, freePort()))
{
assertNotNull(runtime.getLauncher().getQInstance().getTable("demoNote"));
QContext.init(runtime.getLauncher().getQInstance(), new QSystemUserSession());
try
{
assertEquals("Transient demo note", GetAction.execute("demoNote", 1).getValueString("text"));
new UpdateAction().execute(new UpdateInput("demoNote").withRecords(List.of(new QRecord().withValue("id", 1).withValue("text", "Only this run"))));
assertEquals("Only this run", GetAction.execute("demoNote", 1).getValueString("text"));
}
finally
{
QContext.clear();
}
}
closeOwnedBrokerConnections();
try(CoreRuntime runtime = CoreRuntime.launch(directory, freePort()))
{
QContext.init(runtime.getLauncher().getQInstance(), new QSystemUserSession());
try
{
assertEquals("Transient demo note", GetAction.execute("demoNote", 1).getValueString("text"));
}
finally
{
QContext.clear();
}
}
}
static int freePort() throws Exception
{
try(ServerSocket socket = new ServerSocket(0))
{
return socket.getLocalPort();
}
}
}
+16
View File
@@ -0,0 +1,16 @@
services:
app-full:
environment:
QQQ_ALL_OPENSEARCH_HOST: opensearch
rabbitmq:
ports: !override
- "127.0.0.1::15672"
artemis:
ports: !override
- "127.0.0.1::8161"
mailpit:
ports: !override
- "127.0.0.1::8025"
minio:
ports: !override
- "127.0.0.1::9001"
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env python3
"""Start the packaged core jar, smoke its HTTP API, and always stop it."""
import argparse
import os
import socket
import subprocess
import sys
import tempfile
import time
from pathlib import Path
from smoke import CORE_ADMIN_SESSION, SmokeClient, check_core
def free_port():
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
return listener.getsockname()[1]
def await_health(client, process, timeout):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if process.poll() is not None:
raise AssertionError("packaged app exited before becoming healthy")
try:
if client.json("/health").get("status") == "UP":
return
except (AssertionError, OSError, ValueError):
pass
time.sleep(0.1)
raise AssertionError("timed out waiting for packaged app health")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("jar", type=Path)
parser.add_argument("--timeout", type=int, default=90)
args = parser.parse_args()
if not args.jar.is_file():
raise AssertionError("packaged jar was not found")
with tempfile.TemporaryDirectory(prefix="qqq-all-core-smoke-") as directory:
port = free_port()
environment = os.environ.copy()
environment.update({
"QQQ_ALL_PROFILE": "core",
"QQQ_ALL_PORT": str(port),
"QQQ_ALL_DATA_DIR": str(Path(directory) / "data"),
})
with (Path(directory) / "app.log").open("wb") as output:
process = subprocess.Popen(
["java", "-jar", str(args.jar.resolve())],
env=environment, stdout=output, stderr=subprocess.STDOUT,
)
try:
client = SmokeClient(f"http://127.0.0.1:{port}", CORE_ADMIN_SESSION)
await_health(client, process, args.timeout)
check_core(client, args.timeout)
if process.poll() is not None:
raise AssertionError("packaged app exited during smoke checks")
finally:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait(timeout=10)
if __name__ == "__main__":
try:
main()
except (AssertionError, OSError, ValueError) as error:
print(f"core smoke failed: {error}", file=sys.stderr)
raise SystemExit(1) from None
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""Boot a private full Compose stack, run HTTP smoke checks, and remove it."""
import json
import os
import secrets
import socket
import subprocess
import sys
import tempfile
import urllib.parse
from pathlib import Path
from smoke import check_full
ROOT = Path(__file__).resolve().parent.parent
def free_port(exclude=()):
for _ in range(30):
port = 20000 + secrets.randbelow(10000)
if port in exclude:
continue
try:
with socket.socket() as listener:
listener.bind(("127.0.0.1", port))
return port
except OSError:
continue
raise AssertionError("could not reserve a full smoke host port")
def demo_environment():
values = {}
for line in (ROOT / ".env.example").read_text().splitlines():
if not line or line.startswith("#"):
continue
key, value = line.split("=", 1)
if "PASSWORD" in key or "SECRET" in key or key == "MONGO_KEYFILE":
value = secrets.token_urlsafe(24)
values[key] = value
app_port = free_port()
values["QQQ_ALL_PORT"] = str(app_port)
values["KEYCLOAK_PORT"] = str(free_port((app_port,)))
return values
def published_port(command, service, target, environment):
result = subprocess.run(command + ["port", service, str(target)],
cwd=ROOT, env=environment, capture_output=True, text=True,
timeout=15, check=False)
if result.returncode:
raise AssertionError(f"could not discover {service} host port")
address = result.stdout.strip()
if not address.startswith("127.0.0.1:") or not address.rsplit(":", 1)[1].isdigit():
raise AssertionError(f"unexpected {service} host port")
return int(address.rsplit(":", 1)[1])
def query_opensearch(command, environment, index_name, query):
url = ("http://127.0.0.1:9200/" + urllib.parse.quote(index_name, safe="")
+ "/_search")
result = subprocess.run(
command + ["exec", "-T", "opensearch", "curl", "--fail", "--silent",
"--show-error", "--max-time", "5", "-H", "Content-Type: application/json",
"--data-binary", json.dumps(query), url],
cwd=ROOT, env=environment, capture_output=True, text=True, timeout=15, check=False,
)
if result.returncode:
raise AssertionError("OpenSearch query failed inside Compose")
try:
return json.loads(result.stdout)
except ValueError:
raise AssertionError("OpenSearch returned invalid JSON") from None
def prepare_sftp_import_directory(command, environment, username):
path = f"/home/{username}/upload/imports"
for args in (["mkdir", "-p", path], ["chown", "1001:1001", path]):
result = subprocess.run(
command + ["exec", "-T", "sftp"] + args,
cwd=ROOT, env=environment, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=15, check=False,
)
if result.returncode:
raise AssertionError("could not prepare SFTP import directory")
def main():
if not (ROOT / "compose.yaml").is_file():
raise AssertionError("C6 Compose is not integrated into this branch")
values = demo_environment()
project = "qqqallsmoke" + secrets.token_hex(4)
environment = os.environ.copy()
environment.update(values)
os.environ.update(values)
with tempfile.TemporaryDirectory(prefix="qqq-all-full-smoke-") as directory:
env_file = Path(directory) / "compose.env"
env_file.write_text("".join(f"{key}={value}\n" for key, value in values.items()))
env_file.chmod(0o600)
command = ["docker", "compose", "-p", project, "--env-file", str(env_file),
"-f", str(ROOT / "compose.yaml"), "-f", str(ROOT / "scripts/compose.smoke.yaml"),
"--profile", "full"]
try:
started = subprocess.run(command + ["up", "-d", "--build", "--wait", "--wait-timeout", "360"],
cwd=ROOT, env=environment, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=480, check=False)
if started.returncode:
raise AssertionError("full Compose stack did not become healthy")
prepare_sftp_import_directory(command, environment, values.get("SFTP_USER", "qqq"))
check_full(
f"http://127.0.0.1:{values['QQQ_ALL_PORT']}",
f"http://keycloak.localhost:{values['KEYCLOAK_PORT']}/realms/qqq-all",
lambda index, query: query_opensearch(command, environment, index, query),
120,
)
finally:
subprocess.run(command + ["down", "--volumes", "--remove-orphans"],
cwd=ROOT, env=environment, stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL, timeout=120, check=False)
if __name__ == "__main__":
try:
main()
except (AssertionError, OSError, ValueError, subprocess.TimeoutExpired) as error:
print(f"full smoke failed: {error}", file=sys.stderr)
raise SystemExit(1) from None
+375
View File
@@ -0,0 +1,375 @@
#!/usr/bin/env python3
"""Black-box HTTP smoke checks for the qqq-all reference application."""
import argparse
import base64
import hashlib
import html.parser
import http.cookiejar
import json
import os
import re
import secrets
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
CORE_TABLES = ("customer", "order", "orderLine", "product")
CORE_SEEDS = (
("customer", "name", "Ada Lovelace"),
("order", "orderNo", "ORD-1001"),
("orderLine", "quantity", 2),
("product", "name", "QRun Starter Kit"),
)
FULL_TABLES = CORE_TABLES + (
"warehouseCustomer", "supplierOrder", "shipment", "document", "externalImportFile",
"SFTPImportSourceFileTable",
)
CORE_ADMIN_SESSION = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
class LoginForm(html.parser.HTMLParser):
def __init__(self):
super().__init__()
self.action = None
self.values = {}
self.inside = False
def handle_starttag(self, tag, attrs):
values = dict(attrs)
if tag == "form" and values.get("id") == "kc-form-login":
self.action = values.get("action")
self.inside = True
elif tag == "input" and self.inside and values.get("name"):
self.values[values["name"]] = values.get("value", "")
def handle_endtag(self, tag):
if tag == "form":
self.inside = False
class StopAtCallback(urllib.request.HTTPRedirectHandler):
def __init__(self, callback):
self.callback = callback
def redirect_request(self, request, fp, code, msg, headers, url):
if url.startswith(self.callback + "?"):
return None
return super().redirect_request(request, fp, code, msg, headers, url)
def wait_for(label, predicate, timeout):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
if predicate():
return
except (OSError, TimeoutError, AssertionError, ValueError):
pass
time.sleep(0.25)
raise AssertionError(f"timed out waiting for {label}")
class SmokeClient:
def __init__(self, base_url, session_id=None):
self.base_url = base_url.rstrip("/")
self.session_id = session_id
self.cookies = http.cookiejar.CookieJar()
self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cookies))
def request(self, path, body=None, method=None, expected_status=200):
data = None if body is None else json.dumps(body).encode()
headers = {"Content-Type": "application/json"} if data is not None else {}
if self.session_id:
headers["Cookie"] = "sessionId=" + self.session_id
request = urllib.request.Request(
self.base_url + path, data=data, method=method,
headers=headers,
)
try:
with self.opener.open(request, timeout=5) as response:
if response.status != expected_status:
raise AssertionError(f"{path}: HTTP {response.status}")
return response.read()
except urllib.error.HTTPError as error:
status = error.code
try:
if status == expected_status:
return error.read()
finally:
error.close()
raise AssertionError(f"{path}: HTTP {status}") from None
def json(self, path, body=None):
return json.loads(self.request(path, body))
def records(self, table):
result = self.json(f"/qqq/v1/table/{table}/query", {})
records = result.get("records")
if not isinstance(records, list):
raise AssertionError(f"{table}: query did not return records")
return records
def require_record(self, table, field, value):
if not any(record.get("values", {}).get(field) == value for record in self.records(table)):
raise AssertionError(f"missing expected {table} {field}")
def allow_loopback_http_session(client):
parsed = urllib.parse.urlsplit(client.base_url)
if parsed.scheme != "http" or parsed.hostname not in ("127.0.0.1", "localhost"):
raise AssertionError("OIDC HTTP cookie adjustment requires a loopback app")
cookies = [cookie for cookie in client.cookies if cookie.name == "sessionUUID"]
if len(cookies) != 1:
raise AssertionError("OIDC did not set exactly one session cookie")
if cookies[0].secure:
cookies[0].secure = False
def allow_loopback_http_cookies(cookies, url):
parsed = urllib.parse.urlsplit(url)
host = parsed.hostname or ""
if parsed.scheme != "http" or not (host in ("127.0.0.1", "localhost")
or host.endswith(".localhost")):
raise AssertionError("OIDC HTTP cookie adjustment requires a loopback host")
for cookie in cookies:
if cookie.secure and cookie.domain.lstrip(".") == host:
cookie.secure = False
def check_health_and_dashboard(client, timeout):
wait_for("health", lambda: client.json("/health").get("status") == "UP", timeout)
html = client.request("/").decode()
match = re.search(r'["\'](/_next/static/[^"\']+\.(?:js|css))["\']', html)
if not match:
raise AssertionError("dashboard did not reference a Next asset")
if not client.request(match.group(1)):
raise AssertionError("Next asset was empty")
print("health and Next asset: OK")
def check_tables(client, tables, require_seed=True):
for table in tables:
records = client.records(table)
if require_seed and not records:
raise AssertionError(f"{table}: no seeded records")
if require_seed:
for table, field, value in CORE_SEEDS:
client.require_record(table, field, value)
else:
client.require_record("customer", "name", "Ada Lovelace")
print("backend table queries: OK")
def trigger_order(client):
before = {record.get("values", {}).get("id") for record in client.records("processTrace")}
marker = "SMOKE-" + uuid.uuid4().hex[:12]
response = client.json("/data/order/", {
"orderNo": marker, "customerId": 1, "status": "NEW"
})
inserted = response.get("records", [])
order_id = inserted[0].get("values", {}).get("id") if inserted else None
if not order_id:
raise AssertionError("order insert did not return an ID")
return before, order_id, marker
def check_artemis_round_trip(client, timeout):
before, order_id, marker = trigger_order(client)
process_uuid = None
def new_sync_trace():
nonlocal process_uuid
for record in client.records("processTrace"):
values = record.get("values", {})
if (values.get("id") not in before
and record.get("recordLabel", "").startswith("Sync Order -")
and values.get("keyRecordId") == order_id
and values.get("processUUID")):
process_uuid = values["processUUID"]
return True
return False
wait_for("Artemis syncOrder trace",
new_sync_trace, timeout)
client.require_record("order", "orderNo", marker)
print("Artemis order event round trip: OK")
return process_uuid
def check_runtime_features(client, can_calculate=True):
records = client.records("apiCatalog")
if [record.get("values", {}).get("id") for record in records] != [101, 102, 103]:
raise AssertionError("API catalog did not traverse all pages")
record = client.json("/data/apiCatalog/103")
if record.get("values", {}).get("name") != "Shipping Box":
raise AssertionError("API catalog get returned incorrect record")
client.require_record("demoNote", "text", "Transient demo note")
path = "/processes/calculateOrderTotal/run?quantity=3&unitPriceCents=1250"
if can_calculate:
result = client.json(path, {})
if result.get("values", {}).get("totalCents") != 3750:
raise AssertionError("JavaScript calculation returned incorrect total")
else:
client.request(path, {}, expected_status=403)
client.request("/data/demoNote/1", {"text": "forbidden"}, method="PATCH", expected_status=403)
client.request("/data/apiCatalog/", {"id": 999, "name": "forbidden"}, expected_status=403)
print("API catalog, transient memory, JavaScript permissions: OK")
def check_core(client, timeout):
check_health_and_dashboard(client, timeout)
check_tables(client, CORE_TABLES)
check_runtime_features(client)
check_runtime_features(SmokeClient(client.base_url, "dddddddd-dddd-4ddd-8ddd-dddddddddddd"), False)
check_artemis_round_trip(client, timeout)
def oidc_login(client, keycloak_url, username, password):
"""Complete the browser authorization-code flow and establish a QQQ cookie."""
try:
discovery_url = keycloak_url.rstrip("/") + "/.well-known/openid-configuration"
with urllib.request.urlopen(discovery_url, timeout=5) as response:
authorization_endpoint = json.load(response)["authorization_endpoint"]
verifier = secrets.token_urlsafe(48)
challenge = base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest()).rstrip(b"=").decode()
state = secrets.token_urlsafe(24)
callback = client.base_url + "/"
query = urllib.parse.urlencode({
"client_id": "qqq-all", "response_type": "code", "scope": "openid email profile",
"redirect_uri": callback, "state": state, "code_challenge": challenge,
"code_challenge_method": "S256",
})
browser_cookies = http.cookiejar.CookieJar()
browser = urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(browser_cookies), StopAtCallback(callback)
)
with browser.open(authorization_endpoint + "?" + query, timeout=10) as response:
form = LoginForm()
form.feed(response.read().decode())
if not form.action:
raise AssertionError("Keycloak login form missing")
if urllib.parse.urlsplit(authorization_endpoint).scheme == "http":
allow_loopback_http_cookies(browser_cookies, authorization_endpoint)
form.values.update({"username": username, "password": password})
request = urllib.request.Request(form.action, data=urllib.parse.urlencode(form.values).encode())
try:
with browser.open(request, timeout=10) as response:
raise AssertionError(f"Keycloak did not redirect after login (HTTP {response.status})")
except urllib.error.HTTPError as response:
if response.code not in (302, 303):
status = response.code
response.close()
raise AssertionError(f"Keycloak rejected the login (HTTP {status})") from None
location = response.headers.get("Location", "")
response.close()
parameters = urllib.parse.parse_qs(urllib.parse.urlsplit(location).query)
if parameters.get("state") != [state] or not parameters.get("code"):
raise AssertionError("OIDC callback did not contain a valid code and state")
result = client.json("/manageSession", {
"code": parameters["code"][0], "redirectUri": callback, "codeVerifier": verifier
})
if not result.get("uuid") or not any(cookie.name == "sessionUUID" for cookie in client.cookies):
raise AssertionError("QQQ did not establish an OIDC session")
if urllib.parse.urlsplit(client.base_url).scheme == "http":
allow_loopback_http_session(client)
except (OSError, KeyError, ValueError, urllib.error.HTTPError) as error:
raise AssertionError("OIDC login did not complete") from None
print(f"OIDC {username} login: OK")
def http_query_index(search_url, index_name, query):
request = urllib.request.Request(
search_url.rstrip("/") + "/" + urllib.parse.quote(index_name, safe="") + "/_search",
data=json.dumps(query).encode(), headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(request, timeout=5) as response:
return json.load(response)
def check_quick_search(client, query_index, index_name, timeout):
result = client.json("/processes/quickSearchFullReindex/run", {})
if result.get("exception"):
raise AssertionError("quick-search reindex failed")
def finds_customer():
query = {"query": {"bool": {"must": {"match": {"searchableText": "Ada"}},
"filter": {"term": {"sourceTable": "customer"}}}}}
hits = query_index(index_name, query)["hits"]["hits"]
return any("Ada Lovelace" in json.dumps(hit.get("_source", {})) for hit in hits)
wait_for("quick search seeded customer", finds_customer, timeout)
print("OpenSearch customer quick search: OK")
def check_rabbitmq(client, timeout, process_uuid):
def message_arrived():
offset = 0
while True:
suffix = "" if offset == 0 else f"?offset={offset}"
page = client.json("/qqq/v1/esb/messages/orderSyncEvents" + suffix)
messages = page.get("messages", [])
for message in messages:
event = message.get("event")
if not isinstance(event, dict):
continue
data = event.get("data")
if (event.get("type") == "qqq.process.syncOrder.completed"
and isinstance(data, dict)
and data.get("processName") == "syncOrder"
and data.get("processUUID") == process_uuid):
return True
if not page.get("hasMore"):
return False
if not messages:
raise ValueError("ESB browse returned an empty page with more messages")
offset += len(messages)
wait_for("RabbitMQ syncOrder completion", message_arrived, timeout)
print("RabbitMQ completion publication: OK")
def check_full(base_url, keycloak_url, query_index, timeout):
viewer = SmokeClient(base_url)
check_health_and_dashboard(viewer, timeout)
oidc_login(viewer, keycloak_url, "demo-user", os.environ["DEMO_USER_PASSWORD"])
viewer.require_record("customer", "name", "Ada Lovelace")
check_runtime_features(viewer, False)
admin = SmokeClient(base_url)
oidc_login(admin, keycloak_url, "demo-admin", os.environ["DEMO_ADMIN_PASSWORD"])
check_tables(admin, FULL_TABLES, require_seed=False)
check_runtime_features(admin)
check_quick_search(admin, query_index, os.environ.get("QQQ_ALL_OPENSEARCH_INDEX", "qqq-all-customers"), timeout)
process_uuid = check_artemis_round_trip(admin, timeout)
check_rabbitmq(admin, timeout, process_uuid)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("profile", choices=("core", "full"))
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--timeout", type=int, default=90)
parser.add_argument("--keycloak-url", default="http://keycloak.localhost:8081/realms/qqq-all")
parser.add_argument("--opensearch-url")
args = parser.parse_args()
client = SmokeClient(args.base_url, CORE_ADMIN_SESSION if args.profile == "core" else None)
if args.profile == "core":
check_core(client, args.timeout)
else:
if not args.opensearch_url:
raise AssertionError("full profile needs --opensearch-url; use run_full_smoke.py for private Compose")
check_full(args.base_url, args.keycloak_url,
lambda index, query: http_query_index(args.opensearch_url, index, query),
args.timeout)
if __name__ == "__main__":
try:
main()
except (AssertionError, OSError, ValueError, KeyError) as error:
print(f"smoke failed: {error}", file=sys.stderr)
raise SystemExit(1) from None
+522
View File
@@ -0,0 +1,522 @@
#!/usr/bin/env python3
"""Contract tests for the HTTP smoke checks, without starting the application."""
import http.cookiejar
import json
import os
import re
import shutil
import socketserver
import subprocess
import sys
import tempfile
import threading
import unittest
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from unittest import mock
from smoke import SmokeClient, check_artemis_round_trip, check_core, check_rabbitmq, check_tables, check_runtime_features, CORE_TABLES
import run_full_smoke
from run_full_smoke import ROOT
class SmokeServer(ThreadingHTTPServer):
def server_bind(self):
socketserver.TCPServer.server_bind(self)
self.server_name = "localhost"
self.server_port = self.server_address[1]
class DemoHandler(BaseHTTPRequestHandler):
orders = 0
product_name = "QRun Starter Kit"
trace_label = "Sync Order - demo"
trace_order_id = None
inserted_marker = None
def log_message(self, *_args):
pass
def do_GET(self):
if self.path == "/data/apiCatalog/103":
return self.send_json({"values": {"id": 103, "name": "Shipping Box"}})
if self.path == "/health":
return self.send_json({"status": "UP"})
if self.path == "/":
body = b'<html><script src="/_next/static/chunks/demo.js"></script></html>'
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
if self.path == "/_next/static/chunks/demo.js":
body = b"console.log('demo');"
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
self.send_error(404)
def do_POST(self):
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
if self.path == "/data/apiCatalog/":
return self.send_error(403)
if self.path.startswith("/processes/calculateOrderTotal/run"):
if "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" not in self.headers.get("Cookie", ""):
return self.send_error(403)
return self.send_json({"values": {"totalCents": 3750}})
if self.path == "/data/order/":
assert body["orderNo"].startswith("SMOKE-")
type(self).orders += 1
type(self).trace_order_id = 100 + self.orders
type(self).inserted_marker = body["orderNo"]
return self.send_json({"records": [{"values": {"id": 100 + self.orders}}]})
if self.path.startswith("/qqq/v1/table/") and self.path.endswith("/query"):
table = self.path.split("/")[4]
if table == "apiCatalog":
return self.send_json({"records": [{"values": {"id": i}} for i in (101, 102, 103)]})
if table == "processTrace":
records = [{"recordLabel": self.trace_label,
"values": {"id": i, "processUUID": f"trace-{i}",
"keyRecordId": self.trace_order_id}}
for i in range(self.orders)]
else:
values = {
"demoNote": {"id": 1, "text": "Transient demo note"},
"customer": {"id": 1, "name": "Ada Lovelace"},
"order": {"id": 1, "orderNo": "ORD-1001"},
"orderLine": {"id": 1, "quantity": 2},
"product": {"id": 1, "name": self.product_name},
}
records = [{"values": values.get(table, {"id": 1})}]
if table == "order" and self.inserted_marker:
records.append({"values": {"id": 100 + self.orders,
"orderNo": self.inserted_marker}})
return self.send_json({"records": records})
self.send_error(404)
def do_PATCH(self):
if self.path == "/data/demoNote/1":
return self.send_error(403)
self.send_error(404)
def send_json(self, value):
data = json.dumps(value).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
class SmokeTest(unittest.TestCase):
def setUp(self):
DemoHandler.orders = 0
DemoHandler.trace_order_id = None
DemoHandler.inserted_marker = None
def test_catalog_missing_final_page_fails(self):
client = mock.Mock()
client.records.return_value = [{"values": {"id": value}} for value in (101, 102)]
with self.assertRaisesRegex(AssertionError, "all pages"):
check_runtime_features(client)
def test_wrong_javascript_result_fails(self):
client = mock.Mock()
client.records.return_value = [{"values": {"id": value}} for value in (101, 102, 103)]
client.json.side_effect = [{"values": {"name": "Shipping Box"}}, {"values": {"totalCents": 3749}}]
with self.assertRaisesRegex(AssertionError, "incorrect total"):
check_runtime_features(client)
def test_core_runner_fails_promptly_when_jar_exits(self):
with tempfile.TemporaryDirectory() as directory:
jar = Path(directory) / "broken.jar"
jar.write_bytes(b"not a jar")
result = subprocess.run(
[sys.executable, str(Path(__file__).with_name("run_core_smoke.py")),
str(jar), "--timeout", "1"], capture_output=True, text=True, timeout=5,
)
self.assertEqual(1, result.returncode)
self.assertIn("packaged app exited", result.stderr)
def test_ci_core_smokes_maven_final_artifact(self):
pom = ET.parse(ROOT / "qqq-all-app" / "pom.xml")
final_name = pom.findtext(".//{http://maven.apache.org/POM/4.0.0}build/"
"{http://maven.apache.org/POM/4.0.0}finalName")
workflow = (ROOT / ".github" / "workflows" / "smoke.yml").read_text()
command = re.search(r"run: python3 scripts/run_core_smoke.py (\S+)", workflow)
self.assertIsNotNone(command)
self.assertEqual(f"qqq-all-app/target/{final_name}.jar", command.group(1))
def test_standalone_full_smoke_requires_explicit_search_endpoint(self):
result = subprocess.run(
[sys.executable, str(Path(__file__).with_name("smoke.py")), "full",
"--base-url", "http://127.0.0.1:1"],
capture_output=True, text=True, timeout=5,
)
self.assertEqual(1, result.returncode)
self.assertIn("run_full_smoke.py", result.stderr)
def test_full_runner_reserves_distinct_non_ephemeral_app_ports(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / ".env.example").write_text(
"QQQ_ALL_PORT=8080\nKEYCLOAK_PORT=8081\nPOSTGRES_PASSWORD=example\n")
with mock.patch.object(run_full_smoke, "ROOT", root):
values = run_full_smoke.demo_environment()
app_port = int(values["QQQ_ALL_PORT"])
keycloak_port = int(values["KEYCLOAK_PORT"])
self.assertTrue(20000 <= app_port < 30000)
self.assertTrue(20000 <= keycloak_port < 30000)
self.assertNotEqual(app_port, keycloak_port)
self.assertNotEqual("example", values["POSTGRES_PASSWORD"])
def test_full_runner_supplies_generated_credentials_to_http_checks(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / "compose.yaml").write_text("services: {}\n")
values = {"QQQ_ALL_PORT": "20001", "KEYCLOAK_PORT": "20002",
"DEMO_USER_PASSWORD": "generated-user",
"DEMO_ADMIN_PASSWORD": "generated-admin",
"RABBITMQ_PASSWORD": "generated-rabbit"}
def verify_credentials(*_args):
for key in ("DEMO_USER_PASSWORD", "DEMO_ADMIN_PASSWORD", "RABBITMQ_PASSWORD"):
self.assertEqual(values[key], os.environ[key])
with (mock.patch.object(run_full_smoke, "ROOT", root),
mock.patch.object(run_full_smoke, "demo_environment", return_value=values),
mock.patch.object(run_full_smoke, "published_port", return_value=20003),
mock.patch.object(run_full_smoke.subprocess, "run",
return_value=subprocess.CompletedProcess([], 0)),
mock.patch.object(run_full_smoke, "check_full", side_effect=verify_credentials) as check,
mock.patch.dict(os.environ, {}, clear=False)):
run_full_smoke.main()
check.assert_called_once()
def test_full_runner_queries_private_opensearch_service(self):
command = ["docker", "compose", "-p", "smoke"]
with mock.patch.object(run_full_smoke.subprocess, "run",
return_value=subprocess.CompletedProcess([], 0,
stdout='{"hits":{"hits":[]}}')) as invoke:
result = run_full_smoke.query_opensearch(command, {}, "customers", {"query": {}})
self.assertEqual([], result["hits"]["hits"])
args = invoke.call_args.args[0]
self.assertEqual(["exec", "-T", "opensearch", "curl"], args[len(command):len(command) + 4])
self.assertIn("http://127.0.0.1:9200/customers/_search", args)
def test_full_runner_fails_when_private_opensearch_query_fails(self):
with mock.patch.object(run_full_smoke.subprocess, "run",
return_value=subprocess.CompletedProcess([], 22, stdout="")):
with self.assertRaisesRegex(AssertionError, "OpenSearch query failed"):
run_full_smoke.query_opensearch(["docker", "compose"], {}, "customers", {"query": {}})
def test_full_runner_prepares_sftp_import_directory(self):
command = ["docker", "compose", "-p", "smoke"]
with mock.patch.object(run_full_smoke.subprocess, "run",
return_value=subprocess.CompletedProcess([], 0)) as invoke:
run_full_smoke.prepare_sftp_import_directory(command, {}, "qqq")
self.assertEqual([
command + ["exec", "-T", "sftp", "mkdir", "-p", "/home/qqq/upload/imports"],
command + ["exec", "-T", "sftp", "chown", "1001:1001", "/home/qqq/upload/imports"],
], [call.args[0] for call in invoke.call_args_list])
def test_full_runner_fails_when_sftp_fixture_cannot_be_prepared(self):
with mock.patch.object(run_full_smoke.subprocess, "run", side_effect=[
subprocess.CompletedProcess([], 0), subprocess.CompletedProcess([], 1)]):
with self.assertRaisesRegex(AssertionError, "SFTP import directory"):
run_full_smoke.prepare_sftp_import_directory(["docker", "compose"], {}, "qqq")
def test_oidc_cookie_is_sent_only_to_loopback_http_app(self):
from smoke import allow_loopback_http_session
class SecureSession(DemoHandler):
def do_GET(self):
if self.path in ("/login", "/login-plain"):
data = b"ok"
self.send_response(200)
secure = "; Secure" if self.path == "/login" else ""
self.send_header("Set-Cookie", "sessionUUID=test-session; Path=/; HttpOnly" + secure)
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
return
if self.path == "/protected":
if "sessionUUID=test-session" not in self.headers.get("Cookie", ""):
return self.send_error(403)
return self.send_json({"ok": True})
return super().do_GET()
server = SmokeServer(("127.0.0.1", 0), SecureSession)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
client.request("/login")
with self.assertRaisesRegex(AssertionError, "HTTP 403"):
client.request("/protected")
allow_loopback_http_session(client)
self.assertTrue(client.json("/protected")["ok"])
plain = SmokeClient(f"http://127.0.0.1:{server.server_port}")
plain.request("/login-plain")
allow_loopback_http_session(plain)
self.assertTrue(plain.json("/protected")["ok"])
remote = SmokeClient("http://example.com")
with self.assertRaisesRegex(AssertionError, "loopback"):
allow_loopback_http_session(remote)
finally:
server.shutdown()
server.server_close()
thread.join()
def test_keycloak_secure_auth_cookie_is_sent_on_loopback_http(self):
from smoke import allow_loopback_http_cookies
class SecureKeycloak(DemoHandler):
def do_GET(self):
if self.path == "/auth":
data = b"login"
self.send_response(200)
self.send_header("Set-Cookie", "AUTH_SESSION_ID=test-auth; Path=/; Secure; HttpOnly")
self.send_header("Content-Length", str(len(data)))
self.end_headers()
self.wfile.write(data)
return
return super().do_GET()
def do_POST(self):
if self.path == "/login":
if "AUTH_SESSION_ID=test-auth" not in self.headers.get("Cookie", ""):
return self.send_error(400)
return self.send_json({"ok": True})
return super().do_POST()
server = SmokeServer(("127.0.0.1", 0), SecureKeycloak)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
base = f"http://127.0.0.1:{server.server_port}"
cookies = http.cookiejar.CookieJar()
browser = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookies))
browser.open(base + "/auth", timeout=2).close()
with self.assertRaises(urllib.error.HTTPError) as denied:
browser.open(urllib.request.Request(base + "/login", data=b"x"), timeout=2)
denied.exception.close()
allow_loopback_http_cookies(cookies, base)
with browser.open(urllib.request.Request(base + "/login", data=b"x"), timeout=2) as response:
self.assertEqual({"ok": True}, json.load(response))
with self.assertRaisesRegex(AssertionError, "loopback"):
allow_loopback_http_cookies(cookies, "http://example.com")
finally:
server.shutdown()
server.server_close()
thread.join()
def test_core_checks_public_http_behavior(self):
server = SmokeServer(("127.0.0.1", 0), DemoHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
client.session_id = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
check_core(client, timeout=2)
finally:
server.shutdown()
server.server_close()
thread.join()
def test_missing_backend_records_fail(self):
server = SmokeServer(("127.0.0.1", 0), DemoHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
with self.assertRaisesRegex(AssertionError, "missing expected customer"):
client.require_record("customer", "name", "Grace Hopper")
finally:
server.shutdown()
server.server_close()
thread.join()
def test_core_rejects_wrong_filesystem_seed(self):
class WrongProduct(DemoHandler):
product_name = "Unrelated Product"
server = SmokeServer(("127.0.0.1", 0), WrongProduct)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
with self.assertRaisesRegex(AssertionError, "missing expected product"):
check_tables(client, CORE_TABLES)
finally:
server.shutdown()
server.server_close()
thread.join()
def test_artemis_rejects_unrelated_trace(self):
class WrongTrace(DemoHandler):
trace_label = "Unrelated Process - demo"
server = SmokeServer(("127.0.0.1", 0), WrongTrace)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
with self.assertRaisesRegex(AssertionError, "Artemis.*syncOrder"):
check_artemis_round_trip(client, timeout=0.3)
finally:
server.shutdown()
server.server_close()
thread.join()
def test_artemis_returns_the_new_trace_uuid(self):
server = SmokeServer(("127.0.0.1", 0), DemoHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
self.assertEqual("trace-0", check_artemis_round_trip(client, timeout=2))
finally:
server.shutdown()
server.server_close()
thread.join()
def test_artemis_rejects_trace_for_another_order(self):
class WrongOrderTrace(DemoHandler):
trace_order_id = 999
def do_POST(self):
if self.path == "/data/order/":
body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
assert body["orderNo"].startswith("SMOKE-")
type(self).orders += 1
return self.send_json({"records": [{"values": {"id": 101}}]})
return super().do_POST()
server = SmokeServer(("127.0.0.1", 0), WrongOrderTrace)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
with self.assertRaisesRegex(AssertionError, "Artemis.*syncOrder"):
check_artemis_round_trip(client, timeout=0.3)
finally:
server.shutdown()
server.server_close()
thread.join()
def test_rabbitmq_rejects_unrelated_completion(self):
class WrongCompletion(DemoHandler):
def do_GET(self):
if self.path == "/qqq/v1/esb/messages/orderSyncEvents":
event = {"type": "qqq.process.other.completed",
"data": {"processName": "other", "processUUID": "trace-1"}}
return self.send_json({"messages": [{"event": event}], "hasMore": False})
return super().do_GET()
server = SmokeServer(("127.0.0.1", 0), WrongCompletion)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
with self.assertRaisesRegex(AssertionError, "RabbitMQ syncOrder completion"):
check_rabbitmq(client, 0.3, "trace-1")
finally:
server.shutdown()
server.server_close()
thread.join()
def test_rabbitmq_rejects_wrong_process_uuid(self):
class WrongRun(DemoHandler):
def do_GET(self):
if self.path == "/qqq/v1/esb/messages/orderSyncEvents":
event = {"type": "qqq.process.syncOrder.completed",
"data": {"processName": "syncOrder", "processUUID": "other-run"}}
return self.send_json({"messages": [{"event": event}], "hasMore": False})
return super().do_GET()
server = SmokeServer(("127.0.0.1", 0), WrongRun)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
with self.assertRaisesRegex(AssertionError, "RabbitMQ syncOrder completion"):
check_rabbitmq(client, 0.3, "trace-0")
finally:
server.shutdown()
server.server_close()
thread.join()
def test_rabbitmq_accepts_matching_process_uuid(self):
class MatchingRun(DemoHandler):
def do_GET(self):
if self.path == "/qqq/v1/esb/messages/orderSyncEvents":
event = {"type": "qqq.process.syncOrder.completed",
"data": {"processName": "syncOrder", "processUUID": "trace-0"}}
return self.send_json({"messages": [{"event": event}], "hasMore": False})
return super().do_GET()
server = SmokeServer(("127.0.0.1", 0), MatchingRun)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
check_rabbitmq(client, 2, "trace-0")
finally:
server.shutdown()
server.server_close()
thread.join()
def test_rabbitmq_finds_completion_after_first_browse_page(self):
class LaterPage(DemoHandler):
def do_GET(self):
if self.path == "/qqq/v1/esb/messages/orderSyncEvents":
old = {"type": "qqq.process.syncOrder.completed",
"data": {"processName": "syncOrder", "processUUID": "old-run"}}
return self.send_json({"messages": [{"event": old}] * 50, "hasMore": True})
if self.path == "/qqq/v1/esb/messages/orderSyncEvents?offset=50":
event = {"type": "qqq.process.syncOrder.completed",
"data": {"processName": "syncOrder", "processUUID": "trace-0"}}
return self.send_json({"messages": [{"event": event}], "hasMore": False})
return super().do_GET()
server = SmokeServer(("127.0.0.1", 0), LaterPage)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
client = SmokeClient(f"http://127.0.0.1:{server.server_port}")
check_rabbitmq(client, 2, "trace-0")
finally:
server.shutdown()
server.server_close()
thread.join()
@unittest.skipUnless((ROOT / "compose.yaml").is_file() and shutil.which("docker"),
"C6 Compose integration is not available")
def test_full_smoke_keeps_opensearch_private_and_uses_ephemeral_service_ports(self):
command = ["docker", "compose", "--env-file", str(ROOT / ".env.example"),
"-f", str(ROOT / "compose.yaml"),
"-f", str(ROOT / "scripts/compose.smoke.yaml"),
"--profile", "full", "config", "--format", "json"]
result = subprocess.run(command, capture_output=True, text=True, timeout=15)
self.assertEqual(0, result.returncode, "full smoke Compose config failed")
services = json.loads(result.stdout)["services"]
self.assertNotIn("ports", services["opensearch"])
for service, target in (("rabbitmq", 15672),
("artemis", 8161), ("mailpit", 8025), ("minio", 9001)):
with self.subTest(service=service):
ports = services[service]["ports"]
self.assertEqual(1, len(ports))
self.assertEqual(target, ports[0]["target"])
self.assertEqual("127.0.0.1", ports[0]["host_ip"])
self.assertFalse(ports[0].get("published"))
if __name__ == "__main__":
unittest.main()