commit f0d695ec976cb0ed474b9958e76e65eebdf20793 Author: James Maes Date: Wed Nov 4 10:45:28 2026 -0600 chore: preserve paused QBit work and document remaining issues Save the current implementation and README, status, and known-issue documentation at the user-requested stop. No completed release or deployment is claimed. Co-Authored-By: Claude Co-Authored-By: GPT-6 Codex diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..719e132 --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +target/ +.local/ +log/ +logs/ +dist/ +.env +*.log +*.mv.db +*.trace.db +.idea/ +.vscode/ +*.iml +.DS_Store +Thumbs.db + +# generated by flatten-maven-plugin (publication POM) +.flattened-pom.xml + +# Local handoff/build evidence: not product source. +.local/ +_release-handoff/ +node_modules/ +HANDOFF.md +STATE.md +COLLABORATION-HANDOFF.md +CORE-DEPENDENCY.md +PROVENANCE.tsv +PROVENANCE.md +SOURCE-MANIFEST.tsv +SOURCE-MANIFEST.sha256 +*.private.* +*.log +# Prepared caller retained as an inert documentation reference while CI/CD is on hold. +.gitea/ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..c0bcafe --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,3 @@ +wrapperVersion=3.3.4 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..68da819 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,55 @@ +# AGENTS.md + +Guidance for coding agents working in this repository. + +## Project Overview + +**QBit Scripting** (`qbit-scripting`): shared script projects, immutable strict-TypeScript builds, capability broker and record lifecycle hooks for QQQ. Current state and the production gate: `docs/STATUS.md`. + +This QBit never executes script code and never compiles it in the JVM. Compilation and execution belong to an external, isolated worker behind `WorkerProvider`; with no qualified provider every operation fails closed. + +## Build Commands + +```bash +./mvnw compile # always use the wrapper: machine Maven 3.10 cannot read the parent POM +./mvnw verify # Checkstyle (validate phase, tests included) + unit tests +``` + +## Architecture + +Entry point: `ScriptingQBitProducer` (`QBitProducer`, same style as qbit-printing) with `ScriptingQBitConfig`. Packages under `com.kingsrook.qbits.scripting`: + +- `source` - `SourceFile`, `SourceSnapshot` (hashed, immutable), `GitCommitRef` + optional `GitSourceImporter` interface. +- `build` - `BuildCoordinator` (the only signer/verifier of artifacts), `ArtifactSubmission`, `BuildOutcome`. +- `worker` - `WorkerProvider` seam, `WorkerMessages` + `WorkerProtocol` (bounded JSON), `UnconfiguredWorkerProvider`. +- `capability` - `CapabilityDeclaration`, `CapabilityBroker`, `RecordReader`/`QqqRecordReader`. +- `hooks` - `HookEvent`, `HookResult`, `ScriptHookCustomizer` (a `TableCustomizerInterface`). +- `service` - `ScriptingService` (draft/check/test/publish/bind), `ScriptExecutor`, `ScriptSourceRegistry`, `ScriptingStore`, `Actor`. + +Dependency rule: nothing imports document-generation, record-files, mobile or printing. Core only (`qqq-backend-core`). + +## Invariants + +- Draft sources live in the existing core `script` / `scriptRevision` / `scriptRevisionFile` tables; this QBit appends immutable revisions. Only publication advances `script.currentScriptRevisionId`, through reviewed core isolation authority in the same transaction; drafts never activate. No second registry. +- Tables `scripting_*` are readable with `scripting.view` and writable only by the system session the service switches to after authorizing a named `Actor` (`ScriptingTablePermission`). +- Lifecycle permissions are separate and never imply each other: `scripting.draft|check|test|publish|bind`. +- An artifact is accepted only if `BuildCoordinator` verified source hash, pinned toolchain, bundle hash and its HMAC attestation; this is re-checked before every execution. +- A binding names a publication; later drafts/builds/publications never change what it runs. +- Native hooks support PRE_INSERT/PRE_UPDATE patches only in authorized early phases, PRE_DELETE veto, and read-only POST_INSERT/UPDATE/DELETE observations. Pre failures veto; post failures warn because the backend action already occurred. Never call a post callback "after commit". Native validation owns field constraints; do not duplicate it. +- No Nashorn, Node `vm`, shell eval, embedded JS engine or regex "type checking" anywhere, and no tests that execute hostile code. Tests use the protocol-level `FakeWorker`. +- Worker messages carry no credentials, sessions or Java objects; capability tokens are opaque and scoped; reads are read-only. + +## Naming + +- Tables: lowercase snake_case, prefixed `scripting_`. Permissions: `scripting.`. + +## Code Style + +- 3-space indentation, opening braces on the next line, `return (x);`. +- Javadoc flower-box banners on every class, record, enum, constructor and method, including private ones and tests (Checkstyle enforces this and the license header). +- Explicit imports only, grouped java.*, third party, static; no wildcard imports. +- Fluent setters (`withX()`) for configuration. No mutable global singletons. + +## Dependencies + +- Java 21. Parent `com.kingsrook:qbit-build-parent:2.0.0`, QQQ `4.1.0-RC.21` modules (public). NOTE: the source needs core APIs that RC.21 does not contain; see `docs/STATUS.md`. Tests: JUnit 6.0.1. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4bb040e --- /dev/null +++ b/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, source + code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 Kingsrook, LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..6fbe46b --- /dev/null +++ b/NOTICE @@ -0,0 +1,6 @@ +QBit Scripting +Copyright 2026 Kingsrook, LLC + +This product includes software developed at Kingsrook, LLC (https://kingsrook.com/). + +Licensed under the Apache License, Version 2.0. diff --git a/README.md b/README.md new file mode 100644 index 0000000..79a82df --- /dev/null +++ b/README.md @@ -0,0 +1,97 @@ +# QBit Scripting + +> **Paused, unreleased work in progress — 2026-11-04.** Implementation has stopped. This is a source checkpoint, not a production-ready release or a complete installable package set. + +Start with [current status and build prerequisites](docs/RELEASE-STATUS.md) and [known issues](docs/KNOWN-ISSUES.md). These documents supersede older release-readiness statements below. + +`base/`, `rdbms/`, `middleware/` and `github/` are the Java reactor modules. `worker/`, `wasm-runtime/`, `supervisor/` and `deploy/` contain separate runtime components and deployment material. A Java package build does not build or qualify every runtime image. + + +Shared, reusable scripting for QQQ: script projects, immutable strict-TypeScript builds with provenance, capability-limited execution and typed record lifecycle hooks. Other QBits consume published, pinned scripts; this one knows nothing about documents, retained files, mobile or printing. + +> **Status: foundation, not a production runtime.** This repository implements and tests the control plane. Running or compiling script code requires an external hardened worker deployment that has not yet been qualified. With no worker configured, every build and execution fails closed. See [docs/STATUS.md](docs/STATUS.md) for the standalone release status and remaining production gates. + +## What it gives you + +| Concern | Implementation | +|---|---| +| Source registry | Reuses core `script`, `scriptRevision`, `scriptRevisionFile`. Each save appends a new revision; nothing here updates one. | +| Separate steps | `saveDraft` / `check` / `test` / `publish` / `bind`, each with its own permission (`scripting.draft`, `.check`, `.test`, `.publish`, `.bind`) and a named `Actor`. | +| Strict builds | `BuildCoordinator` sends a hashed snapshot to a `WorkerProvider`, requires strict mode and its pinned toolchain id, re-hashes the bundle, and signs (HMAC-SHA256) the provenance. Failed builds store diagnostics and no artifact. | +| Optional warm preparation | Explicit tests/publication/binding may prepare verified immutable code using `WarmWorkerPreparation`; no hook or hot execution fallback. Current authority and pins are rechecked. [Contract and full173 verification](docs/AUTHORING-PREPARATION.md). | +| Native warm lifecycle | Optional per-node startup/reconnect reconciliation under a declared metadata principal, bounded exact signed pins, idle authenticated heartbeat and current aggregate readiness. No hot-path preparation. [Contract and full192 verification](docs/WARM-LIFECYCLE.md). | +| Immutable artifacts | `scripting_artifact` rows hold bundle bytes, SHA-256, source hash, toolchain, coordinator id, attestation and a per-project version. Verified again before every execution. | +| Pinning | `scripting_publication` numbers published versions. A `scripting_binding` names one publication; new drafts or publications do not change it. | +| Capabilities | Read-only `CapabilityDeclaration` (table, fields, max rows, max calls). Builds declare, bindings grant a subset. `CapabilityBroker` issues opaque scoped tokens; every read is re-checked (table, fields, filter fields, budgets, size) and then runs through QQQ `QueryAction` under the current actor. | +| Hooks | `ScriptHooks.install` composes native insert/update/delete hooks with existing Java callbacks. Early pre hooks may patch or veto; delete may veto; post callbacks are observations, never commit notifications. See [native hook contract](docs/NATIVE-RECORD-HOOKS.md). | +| Context scripts | `validatePublishedContext` freezes publication id, artifact id, bundle hash and signed capabilities. `executePublishedContext` rechecks that pin, current actor/project permission and narrowed grants. JSON in/out; no dependency on consumers. | +| History | Independently committed append-only STARTED and terminal `scripting_execution` events share an operation id. A rejected record write cannot roll back its execution history. A crash leaves an unresolved STARTED event. | +| Git (optional) | `GitSourceImporter` port for an exact 40-hex commit. The base ships no importer; the optional `github` module (`qbit-scripting-github`, `GitHubSourceImporter`) implements it without coupling to another QBit. | + +## Existing core Script model: assessment + +Decision: **reuse, do not replace, and do not migrate data now.** + +- `script` is the project, `scriptRevision` + `scriptRevisionFile` are draft source revisions. They already have an author, sequence number, per-file contents and the existing script UI. +- Fit: file names up to 100 chars (our path limit follows), contents unbounded in the schema (we bound them at 256 KiB each, 2 MiB total). Reviewed core isolation guards enforce append-only source revisions; builds also record a source hash and `publish` refuses a revision whose files changed after the build. +- Core isolation: project creation uses `ScriptExecutionIsolation.insertIsolatedScript`; drafts never activate. Tested publication advances `script.currentScriptRevisionId` through `activateIsolatedRevision` within the publication transaction and verifies the persisted pointer. The monotonic ISOLATED marker prevents legacy execution. Runtime consumers still require an immutable publication ID and signed artifact, never the pointer alone. Source lifecycle guards preserve parent associations and reject edits/deletes/reparenting of isolated revisions/files. +- Core `Script`/`ScriptRevision` have no build, artifact, capability or publication concepts and no place for them; those live in the five `scripting_*` tables keyed by the core ids. +- Open migration items (see [docs/STATUS.md](docs/STATUS.md)): legacy Java-Script scripts are not converted or imported; a `ScriptType` for strict-TypeScript hook projects is a host decision; legacy projects keep their existing dialog; isolated projects use the semantic source editor and central core guards refuse legacy source mutation/execution. + +## Worker contract (what a provider must be) + +`WorkerProvider` exchanges JSON text only (`WorkerProtocol`, version `qbit-scripting-worker/1`). Bounds: compile messages 4 MiB, execute 2 MiB, results 128 KiB, broker messages 64 KiB, bundle 1 MiB, payload 256 KiB, JSON depth 12, 100 diagnostics, 500-char messages. Inbound messages are size-checked before parsing and unknown keys are rejected. + +- Compile: files + pinned toolchain id + entry point in; strict flag, toolchain id, diagnostics, bundle and its SHA-256 out. The host re-hashes the bundle and compares the toolchain identifier. Actual compiler enforcement depends on the independently qualified worker; a matching string is not isolation evidence. +- Execute: artifact id, artifact SHA-256, bundle, entry point, event, `{proposed, prior}` hook payload or `{context: ...}` CONTEXT payload, **opaque capability token**, limits (wall, CPU, memory, output, broker calls, clamped to absolute maxima). The worker must verify the hash before running. There is no credential, session or connection field. +- Broker: the worker sends `query` requests carrying the token; the host answers or refuses. No other operation exists. +- `qualificationId()` must equal the evidence id the host configured (`ScriptingRuntime.withRequiredQualificationId`). Blank or mismatched means refuse both compilation and execution; the default is blank. + +The host signs the exact capability manifest separately from the artifact. Stored manifest or project/source associations are checked before capabilities are used. scripting_build.manifest_attestation is a required schema addition; old unsigned builds are refused and must be rebuilt. + +Broker callbacks must be synchronous on the initiating QQQ thread and session. The host independently checks total calls and result size. Tokens expire within the trusted provider's transport budget (default 15 seconds, absolute maximum 120 seconds including container setup), are closed after execution, and per-table budget accounting is serialized. This does not extend the script's wall/CPU budget. Hooks omit hidden, password, binary and field-restricted values; patch permission is rechecked against current metadata and field policy. + +`CoordinatorWorkerProvider` is the concrete trusted Java transport to `worker/dist/coordinator-cli.js`. Operator configuration supplies absolute executable/CLI paths, a digest-pinned image, matching external evidence id, stable owner id, private durable POSIX state directory and controlled executable search path. It uses no shell, clears inherited environment, keeps broker callbacks on the caller thread, bounds UTF-8 frames and discarded diagnostics, enforces a deadline with graceful then forced cleanup, and accepts a matching final result only after exit 0. The Node coordinator verifies runsc and runtime provenance and journals container ownership; its reaper must be deployed. See `worker/README.md`. This adapter and its protocol-only process tests do not qualify the actual sandbox. + +## Hooks and durability + +`ScriptHooks.install` composes existing trusted Java customizers for all six native record events, preserving host phases and invoking each once. Early pre-insert/update patches pass through native validation; late host phases support veto only. Post events run after the backend action and before the caller transaction commits; they cannot promise durable after-commit effects. See [native hook semantics](docs/NATIVE-RECORD-HOOKS.md). Separate `ScriptAfterHooks.install(instance, table, AFTER_...)` stages required same-transaction Jobs intents for committed observations; INSERT/UPDATE read current authorized state, DELETE carries metadata only. See [durable AFTER semantics and current qualification](docs/DURABLE-AFTER-HOOKS.md). + +Draft/revision/file saves, build/artifact storage, publication and binding changes use real QQQ backend transactions. Participating tables must share a transactional backend; authoring rejects an ambient transaction to avoid returning an uncommitted publication. Only the explicit memory backend is allowed as a volatile test fixture. Database unique keys resolve publication/version and enabled-binding races; a conflict fails rather than replacing a publication. Supported targets are native MySQL 8.4 (`src/main/resources/db/mysql/`, with the optional RDBMS adapter's real-provider tests) and synchronous file H2; deployment qualification of those schemas, immutable guards and backup/restore remains a host gate (see `docs/TODO.md`). The PostgreSQL scripts under `src/main/resources/db/postgresql/` are prospective only and are not a supported runtime promise. Hosts must also enforce unique `(scriptId, sequenceNo)` source revisions in the existing core schema. H2 tests exercise rollback, independent connections and concurrent publication. + +Execution STARTED and terminal events commit on independent transactions, temporarily suspending the caller transaction. Missing audit storage prevents worker invocation; failure to record a terminal event prevents using its result. The pool needs capacity for the caller and independent audit connection. An unresolved STARTED event is an incomplete attempt, never success or exactly-once execution. + +## Wiring + +```java +ScriptingRuntime runtime = new ScriptingRuntime() + .withWorker(myQualifiedProvider) // omit => fail closed + .withCoordinator(new BuildCoordinator("coord-1", key32Bytes, "typescript-6.0.3-node-24")) + .withRequiredQualificationId("worker-qual-2026-11-03"); +new ScriptingQBitProducer() + .withConfig(new ScriptingQBitConfig().withBackendName("rdbms").withRuntime(runtime)) + .withTableEnricher(table -> table.withBackendDetails(...)) // RDBMS hosts + .produce(qInstance); // core script tables must exist first +ScriptHooks.install(table, HookEvent.PRE_INSERT, HookEvent.PRE_UPDATE, HookEvent.PRE_DELETE); +``` + +Hosts grant `scripting.view|draft|check|test|publish|bind|execute` through their normal permission mechanism. Every service Actor must match the active identified QQQ session and both must hold the action permission; external authentication must establish that session first. Source authoring and lifecycle operations also require access to the owning core Script project. Generic scripting and source views are filtered by that project; ordinary generic source totals/aggregates and joined source reads are denied because they cannot apply that row policy. Source metadata hides count/statistics capabilities; authorized bounded project history remains available. The signing key is host configuration and must come from the host's secret store, never from a script project or the worker. + +## Build + +The optional [middleware artifact](middleware/README.md) registers native authenticated authoring routes and generated OpenAPI. Its source-first frontend integration uses the shared Script registry, with separate draft/build/context-test/publication actions, selected-record hook dry runs and explicit binding activation/state controls. The reviewed local core/RDBMS isolation artifact is now a required selective dependency; ordinary RC20 core is not sufficient. Native HTTP creation/save/build/test/publication succeeds in tests with a protocol-only worker. The frontend has a bounded TypeScript6 semantic worker, scoped declarations and explicit exact-commit GitHub imports. Live sandbox/host credentials and debugger qualification remain outstanding. + +```bash +./mvnw verify +``` + +Wrapper 3.9.11 (`.mvn/wrapper/maven-wrapper.properties`, `mvnw`, `mvnw.cmd` are copied from qbit-printing). + +## Local verification + +Wrapper install runs all verify gates: real QQQ memory/JDBC actions and protocol-only process fixtures, with Checkstyle including tests and JaCoCo minimum 85% instruction/65% branch coverage. Test counts are not recorded here; run the gate. No Java test executes hostile script text or qualifies the actual runtime/production database. Maven still reports the shared published parent-resolution warning; do not suppress it or edit the local cache. + + +## Maven publication + +Published POMs are self-contained: the estate's `flatten-maven-plugin` (`ossrh` mode) resolves `${revision}` and the `qqq.*` version properties at build time, so a consumer needs no `-Drevision`/`-D` override and sees the exact pinned core. Set `qqq.core.version`, `qqq.version` (and for Record Files `qqq.snapshot.version`) once in the release build. The inherited `qbit-build-parent` POM still warns until its owner publishes a literal version. This is a private functional release candidate, not GA and not published. diff --git a/base/LICENSE b/base/LICENSE new file mode 100644 index 0000000..4bb040e --- /dev/null +++ b/base/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, source + code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 Kingsrook, LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/base/NOTICE b/base/NOTICE new file mode 100644 index 0000000..6fbe46b --- /dev/null +++ b/base/NOTICE @@ -0,0 +1,6 @@ +QBit Scripting +Copyright 2026 Kingsrook, LLC + +This product includes software developed at Kingsrook, LLC (https://kingsrook.com/). + +Licensed under the Apache License, Version 2.0. diff --git a/base/pom.xml b/base/pom.xml new file mode 100644 index 0000000..ad85a70 --- /dev/null +++ b/base/pom.xml @@ -0,0 +1,80 @@ + + + 4.0.0 + + com.kingsrookqbit-build-parent2.0.0 + com.kingsrook.qbits + qbit-scripting + ${revision} + QQQ Scripting + QQQ QBit: shared compiled-TypeScript script projects, immutable builds, capability broker and record hooks. Execution requires an external isolated worker. + Apache License, Version 2.0https://www.apache.org/licenses/LICENSE-2.0 + + 0.1.0-SNAPSHOT + 21 + UTF-8 + 4.1.0-RC.214.1.0-RC.21 + none + + + com.kingsrook.qqqqqq-backend-core${qqq.core.version} + com.kingsrook.qqqqqq-backend-module-rdbms${qqq.core.version} + + com.google.guavaguava33.7.2-jre + com.kingsrook.qqqqqq-bom-pom${qqq.version}pomimport + org.junitjunit-bom6.0.1pomimport + + + + com.kingsrook.qqqqqq-backend-core${qqq.core.version} + org.junit.jupiterjunit-jupiter6.0.1test + com.kingsrook.qqqqqq-backend-module-rdbms${qqq.core.version}test + com.h2databaseh22.5.252test + + + + src/main/resourcesfalse + + + + + org.apache.maven.pluginsmaven-resources-plugin3.3.1 + package-license-noticesprocess-resourcescopy-resources + ${project.build.outputDirectory}/META-INF + ${project.basedir}falseLICENSENOTICE + + + + org.jacocojacoco-maven-plugin0.8.15 + + prepare-agent + reportverifyreport + coverageverifycheck + BUNDLE + INSTRUCTIONCOVEREDRATIO0.85 + BRANCHCOVEREDRATIO0.65 + + + + + org.apache.maven.pluginsmaven-checkstyle-plugin3.3.1 + com.puppycrawl.toolscheckstyle10.12.6 + ${project.basedir}/../checkstyle/config.xmlcheckstyle.reactor.root=${maven.multiModuleProjectDirectory}truetruewarningtrue + validatecheck + + org.apache.maven.pluginsmaven-compiler-plugin3.14.1 + org.apache.maven.pluginsmaven-surefire-plugin3.5.4 + org.apache.maven.pluginsmaven-jar-plugin3.4.2${project.version} + test-jar + + + + org.codehaus.mojoflatten-maven-plugin1.7.3 + trueossrh + + flattenprocess-resourcesflatten + flatten.cleancleanclean + + + + diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingAuthoringProcesses.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingAuthoringProcesses.java new file mode 100644 index 0000000..56d425f --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingAuthoringProcesses.java @@ -0,0 +1,128 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.scripts.Script; + +/*************************************************************************** + ** Native project creation for every QQQ process client. Source editing, build, + ** test and publication remain separate service-authorized lifecycle actions. + ***************************************************************************/ +public final class ScriptingAuthoringProcesses +{ + public static final String CREATE_PROJECT = "scripting_create_project"; + + /*************************************************************************** + ** Static metadata registration. + ***************************************************************************/ + private ScriptingAuthoringProcesses() {} + + /*************************************************************************** + ** Use existing form/link contracts, never a client-specific creation API. + ***************************************************************************/ + public static void register(QInstance instance) throws QException + { + if(instance.getProcess(CREATE_PROJECT) != null) + { + throw new QException("Scripting project creation process name is already in use."); + } + instance.addProcess(new QProcessMetaData().withName(CREATE_PROJECT).withLabel("New script").withTableName(Script.TABLE_NAME) + .withMinInputRecords(0).withMaxInputRecords(0) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED).withCustomPermissionChecker(new QCodeReference(Permission.class))) + .withStep(new QFrontendStepMetaData().withName("details").withLabel("New script") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM)) + .withFormField(new QFieldMetaData("name", QFieldType.STRING).withLabel("Script name").withMaxLength(200).withIsRequired(true))) + .withStep(new QBackendStepMetaData().withName("create").withCode(new QCodeReference(Create.class))) + .withStep(new QFrontendStepMetaData().withName("result").withLabel("Script created") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)) + .withViewField(new QFieldMetaData("scriptId", QFieldType.INTEGER).withLabel("Open script") + .withFieldAdornment(new FieldAdornment(AdornmentType.LINK).withValue(AdornmentType.LinkValues.TO_RECORD_FROM_TABLE, Script.TABLE_NAME))) + .withViewField(new QFieldMetaData("nextStep", QFieldType.STRING).withLabel("Next step")))); + ScriptingLifecycleProcesses.register(instance); + } + + /*************************************************************************** + ** Discovery checks the current named grant and native table write ceiling. + ***************************************************************************/ + public static class Permission implements CustomPermissionChecker + { + /*************************************************************************** + ** Rendering an action never grants the later service mutation. + ***************************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException + { + try + { + Actor.fromSession(QContext.getQSession()).require(ScriptingPermissions.DRAFT); + PermissionsHelper.checkTablePermissionThrowing(new InsertInput(Script.TABLE_NAME), TablePermissionSubType.INSERT); + } + catch(ScriptingException denied) + { + throw new QPermissionDeniedException("Script draft creation requires current authoring permission."); + } + } + } + + /*************************************************************************** + ** Create an isolated project through the same public semantic service as API. + ***************************************************************************/ + public static class Create implements BackendStep + { + /*************************************************************************** + ** No input supplies identity, execution mode, source privilege or runtime. + ***************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + new Permission().checkPermissionsThrowing(input, input.getProcess()); + Integer id = ScriptingQBitConfig.serviceFor(QContext.getQInstance()).createProject(Actor.fromSession(QContext.getQSession()), input.getValueString("name")); + output.addValue("scriptId", id); + output.addValue("nextStep", "Open the script to save TypeScript files. Build, test and publish are separate authorized actions."); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingException.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingException.java new file mode 100644 index 0000000..d57015b --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingException.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** The only failure type scripting services raise on purpose. The kind lets + ** callers distinguish a denial from a fail-closed runtime without parsing text. + ** Messages are written to be safe to show a user: never source, stack traces, + ** worker output or secrets. + *******************************************************************************/ +public class ScriptingException extends QException +{ + /******************************************************************************* + ** Why the operation was refused. + *******************************************************************************/ + public enum Kind + { + DENIED, + INVALID, + NOT_FOUND, + UNAVAILABLE, + INTEGRITY, + LIMIT + } + + + + private final Kind kind; + + + + /*************************************************************************** + ** Create an exception of the given kind with a user-safe message. + ***************************************************************************/ + public ScriptingException(Kind kind, String message) + { + super(message); + this.kind = kind; + } + + + + /*************************************************************************** + ** The failure category. + ***************************************************************************/ + public Kind getKind() + { + return (kind); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingLifecycleProcesses.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingLifecycleProcesses.java new file mode 100644 index 0000000..b5a0da2 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingLifecycleProcesses.java @@ -0,0 +1,251 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + +import java.util.List; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.FieldAdornment; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QRecordListMetaData; + +/*************************************************************************** + ** Native form fallback for clients without the enhanced source widget. Only + ** standard forms, code fields, per-request choices and record links are used. + ***************************************************************************/ +public final class ScriptingLifecycleProcesses +{ + /*************************************************************************** + ** Fixed operations map to existing semantic service permissions. + ***************************************************************************/ + public enum Operation + { + ADD_FILE("add_file", "Add source file", ScriptingPermissions.DRAFT), + EDIT_FILE("edit_file", "Edit source file", ScriptingPermissions.DRAFT), + RENAME_FILE("rename_file", "Rename source file", ScriptingPermissions.DRAFT), + REMOVE_FILE("remove_file", "Remove source file", ScriptingPermissions.DRAFT), + IMPORT_COMMIT("import_commit", "Import exact Git commit", ScriptingPermissions.DRAFT), + CHECK("check_draft", "Check latest draft", ScriptingPermissions.CHECK), + TEST_CONTEXT("test_context", "Test context build", ScriptingPermissions.TEST), + PUBLISH("publish_build", "Publish tested build", ScriptingPermissions.PUBLISH); + + final String suffix; + final String label; + final String permission; + + /*************************************************************************** + ** Immutable native process identity. + ***************************************************************************/ + Operation(String suffix, String label, String permission) + { + this.suffix = suffix; + this.label = label; + this.permission = permission; + } + } + + /*************************************************************************** + ** Metadata utility. + ***************************************************************************/ + private ScriptingLifecycleProcesses() {} + + /*************************************************************************** + ** Stable process names are also discoverable through ordinary QQQ metadata. + ***************************************************************************/ + public static String name(Operation operation) + { + return ("scripting_" + operation.suffix); + } + + /*************************************************************************** + ** Reject caller-invented operation names before touching project data. + ***************************************************************************/ + static Operation operation(String name) throws ScriptingException + { + for(Operation operation : Operation.values()) + { + if(name(operation).equals(name)) + { + return (operation); + } + } + throw new ScriptingException(ScriptingException.Kind.INVALID, "Unknown script authoring process"); + } + + /*************************************************************************** + ** Selected script identity is loaded by a backend step before any source form. + ***************************************************************************/ + public static void register(QInstance instance) throws QException + { + for(Operation operation : Operation.values()) + { + if(instance.getProcess(name(operation)) != null) + { + throw new QException("Script lifecycle process name is already in use: " + name(operation)); + } + var process = new QProcessMetaData().withName(name(operation)).withLabel(operation.label).withTableName("script") + .withMinInputRecords(1).withMaxInputRecords(1) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED).withCustomPermissionChecker(new QCodeReference(Permission.class))) + .withStep(new QBackendStepMetaData().withName("prepare").withCode(new QCodeReference(ScriptingLifecycleSteps.Prepare.class)) + .withInputData(new QFunctionInputMetaData().withRecordListMetaData(new QRecordListMetaData().withTableName("script")))); + if(operation == Operation.EDIT_FILE || operation == Operation.RENAME_FILE || operation == Operation.REMOVE_FILE) + { + process.withStep(form("choose", "Choose source file").withFormField(text("selectedPath", "Source file", SourceFile.MAX_PATH_LENGTH))) + .withStep(new QBackendStepMetaData().withName("load").withCode(new QCodeReference(ScriptingLifecycleSteps.LoadFile.class))); + } + var form = form("details", operation.label); + form.withFormField(new QFieldMetaData("scriptId", QFieldType.INTEGER).withLabel("Script project").withIsEditable(false)); + if(operation == Operation.ADD_FILE || operation == Operation.EDIT_FILE) + { + form.withFormField(text("path", "Relative TypeScript path", SourceFile.MAX_PATH_LENGTH).withIsEditable(operation == Operation.ADD_FILE)) + .withFormField(code("contents", "TypeScript source", "typescript", SourceFile.MAX_CONTENT_BYTES).withIsRequired(false)) + .withFormField(text("message", "Revision note", 500).withIsRequired(false)); + } + else if(operation == Operation.RENAME_FILE || operation == Operation.REMOVE_FILE) + { + form.withFormField(text("path", "Selected source file", SourceFile.MAX_PATH_LENGTH).withIsEditable(false)); + if(operation == Operation.RENAME_FILE) + { + form.withFormField(text("newPath", "New relative TypeScript path", SourceFile.MAX_PATH_LENGTH)); + } + form.withFormField(text("sourceActionNotice", "Draft change", 400).withIsEditable(false).withIsRequired(false)) + .withFormField(text("message", "Revision note", 500).withIsRequired(false)); + } + else if(operation == Operation.IMPORT_COMMIT) + { + form.withFormField(text("repository", "GitHub repository (owner/name)", 201)) + .withFormField(text("commitSha", "Full lowercase commit SHA (40 characters)", 40)) + .withFormField(text("subdirectory", "TypeScript source directory (optional)", 200).withIsRequired(false)) + .withFormField(text("importNotice", "Import behavior", 300).withIsEditable(false).withIsRequired(false)); + } + else if(operation == Operation.CHECK) + { + form.withFormField(new QFieldMetaData("expectedRevisionId", QFieldType.INTEGER).withLabel("Pinned source revision").withIsEditable(false)) + .withFormField(text("entryPoint", "Entrypoint file", SourceFile.MAX_PATH_LENGTH)) + .withFormField(code("capabilitiesJson", "Declared READ capabilities (JSON array)", "json", WorkerProtocol.MAX_PAYLOAD_BYTES)); + } + else + { + form.withFormField(new QFieldMetaData("buildId", QFieldType.INTEGER).withLabel("Exact successful build").withIsRequired(true) + .withIsEditable(operation == Operation.PUBLISH)); + if(operation == Operation.TEST_CONTEXT) + { + form.withFormField(code("inputJson", "Context test input (JSON object)", "json", WorkerProtocol.MAX_PAYLOAD_BYTES)) + .withFormField(code("capabilitiesJson", "Granted READ capabilities (within build declaration)", "json", WorkerProtocol.MAX_PAYLOAD_BYTES)); + } + else + { + form.withFormField(text("note", "Publication note", 500).withIsRequired(false)); + } + } + process.withStep(form).withStep(new QBackendStepMetaData().withName("apply").withCode(new QCodeReference(ScriptingLifecycleSteps.Apply.class))) + .withStep(result()); + instance.addProcess(process); + } + } + + /*************************************************************************** + ** Standard editable text; server validators remain authoritative. + ***************************************************************************/ + private static QFieldMetaData text(String name, String label, int maximum) + { + return (new QFieldMetaData(name, QFieldType.STRING).withLabel(label).withMaxLength(maximum).withIsRequired(true)); + } + + /*************************************************************************** + ** Native code-editor adornment is optional presentation, never a parser. + ***************************************************************************/ + private static QFieldMetaData code(String name, String label, String language, int maximum) + { + return (new QFieldMetaData(name, QFieldType.TEXT).withLabel(label).withMaxLength(maximum).withIsRequired(true) + .withFieldAdornment(new FieldAdornment(AdornmentType.CODE_EDITOR).withValue("languageMode", language))); + } + + /*************************************************************************** + ** Every supported client can render an ordinary edit form. + ***************************************************************************/ + private static QFrontendStepMetaData form(String name, String label) + { + return (new QFrontendStepMetaData().withName(name).withLabel(label) + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))); + } + + /*************************************************************************** + ** Native links expose exact saved identities without asking users to guess IDs. + ***************************************************************************/ + private static QFrontendStepMetaData result() + { + var step = new QFrontendStepMetaData().withName("result").withLabel("Script action result") + .withComponent(new QFrontendComponentMetaData().withType(QComponentType.VIEW_FORM)); + List fields = List.of("scriptId", "revisionId", "buildId", "executionId", "publicationId"); + List tables = List.of("script", "scriptRevision", ScriptingTables.BUILD, ScriptingTables.EXECUTION, ScriptingTables.PUBLICATION); + for(int i = 0; i < fields.size(); i++) + { + step.withViewField(new QFieldMetaData(fields.get(i), QFieldType.INTEGER).withLabel(tables.get(i)) + .withFieldAdornment(new FieldAdornment(AdornmentType.LINK).withValue(AdornmentType.LinkValues.TO_RECORD_FROM_TABLE, tables.get(i)))); + } + return (step.withViewField(text("resultStatus", "Status", 100)) + .withViewField(code("resultJson", "Diagnostics, test result or import provenance", "json", WorkerProtocol.MAX_PAYLOAD_BYTES))); + } + + /*************************************************************************** + ** Named permission discovery never substitutes for project checks on submit. + ***************************************************************************/ + public static class Permission implements CustomPermissionChecker + { + /*************************************************************************** + ** Current user identity and read/action permissions are always required. + ***************************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException + { + try + { + Actor actor = Actor.fromSession(QContext.getQSession()); + actor.require(ScriptingPermissions.VIEW); + actor.require(operation(metadata.getName()).permission); + } + catch(ScriptingException denied) + { + throw new QPermissionDeniedException("Script action is unavailable for the current user."); + } + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingLifecycleSteps.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingLifecycleSteps.java new file mode 100644 index 0000000..9d88532 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingLifecycleSteps.java @@ -0,0 +1,415 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import com.kingsrook.qbits.scripting.ScriptingLifecycleProcesses.Operation; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.service.ScriptingAuthoring; +import com.kingsrook.qbits.scripting.service.ScriptingService; +import com.kingsrook.qbits.scripting.source.GitCommitRef; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessMetaDataAdjustment; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import org.json.JSONArray; +import org.json.JSONObject; + +/*************************************************************************** + ** Native steps only coordinate authorized public lifecycle calls. Process values + ** and read-only form fields are data, never identity or authorization tokens. + ***************************************************************************/ +public final class ScriptingLifecycleSteps +{ + /*************************************************************************** + ** Namespace for native backend steps. + ***************************************************************************/ + private ScriptingLifecycleSteps() {} + + /*************************************************************************** + ** Always resolve the instance-owned service, with no cross-QBit dependencies. + ***************************************************************************/ + private static ScriptingService service() throws ScriptingException + { + return (ScriptingQBitConfig.serviceFor(QContext.getQInstance())); + } + + /*************************************************************************** + ** Current permission and project isolation are rechecked on every step. + ***************************************************************************/ + private static ScriptingAuthoring.Project project(RunBackendStepInput input, Actor actor) throws QException + { + new ScriptingLifecycleProcesses.Permission().checkPermissionsThrowing(input, input.getProcess()); + Integer id = input.getValueInteger("scriptId"); + if(id == null || id < 1) + { + throw invalid("Select an isolated script project"); + } + service().authoring().requireIsolated(actor, id); + var project = service().authoring().project(actor, id); + actor.require(ScriptingPermissions.VIEW); + actor.require(ScriptingLifecycleProcesses.operation(input.getProcessName()).permission); + return (project); + } + + /*************************************************************************** + ** Detect changed drafts before reading or dispatching a pinned native form. + ***************************************************************************/ + private static void requireCurrentRevision(RunBackendStepInput input, ScriptingAuthoring.Project project) throws ScriptingException + { + if(!Objects.equals(input.getValueInteger("expectedRevisionId"), project.latestRevisionId())) + { + throw invalid("Source changed; reopen this action before continuing"); + } + } + + /*************************************************************************** + ** Immutable source reads remain project-scoped and preserve every other file. + ***************************************************************************/ + private static List files(Actor actor, Integer project, Integer revision) throws ScriptingException + { + var files = revision == null ? List.of() : service().authoring().draft(actor, project, revision).files(); + actor.require(ScriptingPermissions.VIEW); + return (files); + } + + /*************************************************************************** + ** Per-request enum metadata never mutates the global process definition. + ***************************************************************************/ + private static void choices(RunBackendStepInput input, RunBackendStepOutput output, String step, String field, QFieldType type, List> values) + { + var metadata = input.getProcess().getFrontendStep(step).getFormFields().stream().filter(candidate -> field.equals(candidate.getName())).findFirst().orElseThrow().clone(); + metadata.setInlinePossibleValueSource(new QPossibleValueSource().withName("scripting_process_" + field).withType(QPossibleValueSourceType.ENUM) + .withIdType(type).withEnumValues(values)); + output.setProcessMetaDataAdjustment(new ProcessMetaDataAdjustment().withUpdatedField(metadata)); + } + + /*************************************************************************** + ** A generic host-owned error carries no source or record content. + ***************************************************************************/ + private static ScriptingException invalid(String message) + { + return (new ScriptingException(ScriptingException.Kind.INVALID, message)); + } + + /*************************************************************************** + ** Prepare exact immutable identities and bounded form choices from current access. + ***************************************************************************/ + public static class Prepare implements BackendStep + { + /*************************************************************************** + ** Only the selected primary key is read; selected record bodies are ignored. + ***************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + if(input.getRecords() == null || input.getRecords().size() != 1) + { + throw invalid("Select one script project"); + } + Integer id = input.getRecords().getFirst().getValueInteger("id"); + input.addValue("scriptId", id); + Actor actor = Actor.fromSession(QContext.getQSession()); + var project = project(input, actor); + Operation operation = ScriptingLifecycleProcesses.operation(input.getProcessName()); + output.addValue("scriptId", id); + output.addValue("expectedRevisionId", project.latestRevisionId()); + output.addValue("capabilitiesJson", "[]"); + output.addValue("inputJson", "{}"); + if(operation == Operation.IMPORT_COMMIT) + { + if(!project.importAvailable()) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "No Git importer is configured"); + } + output.addValue("importNotice", "Import replaces the current draft file set with a new immutable revision. Earlier revisions remain available. Review, check, test and publish separately."); + return; + } + if(operation == Operation.ADD_FILE) + { + output.addValue("path", project.latestRevisionId() == null ? "main.ts" : "new-file.ts"); + output.addValue("contents", "export async function main(context: unknown): Promise> { return {}; }\n"); + return; + } + if(operation == Operation.EDIT_FILE || operation == Operation.RENAME_FILE || operation == Operation.REMOVE_FILE || operation == Operation.CHECK) + { + var files = files(actor, id, project.latestRevisionId()); + if(files.isEmpty()) + { + throw invalid("Add a source file first"); + } + if(operation == Operation.REMOVE_FILE && files.size() == 1) + { + throw invalid("A source project must retain at least one file"); + } + List> paths = new ArrayList<>(); + files.forEach(file -> paths.add(new QPossibleValue<>(file.path(), file.path()))); + String field = operation == Operation.CHECK ? "entryPoint" : "selectedPath"; + choices(input, output, operation == Operation.CHECK ? "details" : "choose", field, QFieldType.STRING, paths); + output.addValue(field, files.getFirst().path()); + return; + } + var builds = project.builds().stream().filter(build -> "SUCCEEDED".equals(build.status())).toList(); + if(builds.isEmpty()) + { + throw invalid("Check a source revision successfully first"); + } + output.addValue("buildId", builds.getFirst().id()); + if(operation == Operation.PUBLISH) + { + List> values = new ArrayList<>(); + builds.forEach(build -> values.add(new QPossibleValue<>(build.id(), "Source revision " + build.revisionId() + " / build " + build.id()))); + choices(input, output, "details", "buildId", QFieldType.INTEGER, values); + } + } + } + + /*************************************************************************** + ** Existing-file actions pin the exact selected path after the file chooser. + ***************************************************************************/ + public static class LoadFile implements BackendStep + { + /*************************************************************************** + ** A forged choice cannot load another project's file or a stale base. + ***************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + Actor actor = Actor.fromSession(QContext.getQSession()); + var project = project(input, actor); + requireCurrentRevision(input, project); + var file = files(actor, project.scriptId(), project.latestRevisionId()).stream().filter(candidate -> candidate.path().equals(input.getValueString("selectedPath"))) + .findFirst().orElseThrow(() -> invalid("Choose a file in this source revision")); + Operation operation = ScriptingLifecycleProcesses.operation(input.getProcessName()); + if(operation == Operation.EDIT_FILE) + { + output.addValue("contents", file.content()); + } + else if(operation == Operation.RENAME_FILE || operation == Operation.REMOVE_FILE) + { + output.addValue("sourceActionNotice", "This saves a new draft revision only. Earlier revisions remain unchanged. Imports and entrypoints are not rewritten; review and check the draft before testing or publishing."); + } + else + { + throw invalid("This action does not select an existing source file"); + } + output.addValue("path", file.path()); + } + } + + /*************************************************************************** + ** Apply one action using the same public services as semantic API callers. + ***************************************************************************/ + public static class Apply implements BackendStep + { + /*************************************************************************** + ** Pins are reauthorized, and saved files use the public optimistic overload. + ***************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + Actor actor = Actor.fromSession(QContext.getQSession()); + var project = project(input, actor); + Operation operation = ScriptingLifecycleProcesses.operation(input.getProcessName()); + output.addValue("scriptId", project.scriptId()); + output.addValue("resultJson", "{}"); + if(operation == Operation.ADD_FILE || operation == Operation.EDIT_FILE) + { + saveFile(input, output, project, actor, operation); + } + else if(operation == Operation.RENAME_FILE || operation == Operation.REMOVE_FILE) + { + changeFilePath(input, output, project, actor, operation); + } + else if(operation == Operation.IMPORT_COMMIT) + { + requireCurrentRevision(input, project); + GitCommitRef reference; + try + { + String directory = input.getValueString("subdirectory"); + reference = new GitCommitRef(input.getValueString("repository"), input.getValueString("commitSha"), + directory == null || directory.isEmpty() ? null : directory); + } + catch(IllegalArgumentException failure) + { + throw invalid("Import requires a repository, exact lowercase commit SHA and safe optional directory"); + } + Integer revision = service().importDraft(actor, project.scriptId(), input.getValueInteger("expectedRevisionId"), reference); + var imported = service().authoring().draft(actor, project.scriptId(), revision); + var provenance = imported.provenance(); + output.addValue("revisionId", revision); + output.addValue("resultStatus", "DRAFT_IMPORTED"); + output.addValue("resultJson", new JSONObject().put("repository", provenance.repository()).put("commitSha", provenance.commitSha()) + .put("subdirectory", provenance.subdirectory()).put("sourceSha256", provenance.sourceSha256()).put("importedBy", provenance.importedBy()).toString()); + } + else if(operation == Operation.CHECK) + { + requireCurrentRevision(input, project); + if(project.latestRevisionId() == null) + { + throw invalid("Add a source file first"); + } + var built = service().check(actor, project.latestRevisionId(), input.getValueString("entryPoint"), ScriptingProcessInputs.capabilities(input.getValueString("capabilitiesJson"))); + output.addValue("revisionId", project.latestRevisionId()); + output.addValue("buildId", built.buildId()); + output.addValue("resultStatus", built.success() ? "CHECK_PASSED" : "CHECK_FAILED"); + JSONArray diagnostics = new JSONArray(); + built.diagnostics().forEach(diagnostic -> diagnostics.put(new JSONObject().put("file", diagnostic.file()).put("line", diagnostic.line()) + .put("column", diagnostic.column()).put("code", diagnostic.code()).put("message", diagnostic.message()))); + output.addValue("resultJson", diagnostics.toString()); + } + else + { + Integer build = input.getValueInteger("buildId"); + if(build == null || build < 1) + { + throw invalid("Choose a successful build"); + } + service().authoring().requireBuild(actor, project.scriptId(), build); + if(!service().authoring().build(actor, project.scriptId(), build).success()) + { + throw invalid("Choose a successful build"); + } + output.addValue("buildId", build); + if(operation == Operation.TEST_CONTEXT) + { + var result = service().testContext(actor, build, WorkerProtocol.checkPayload(input.getValueString("inputJson")), + ScriptingProcessInputs.capabilities(input.getValueString("capabilitiesJson"))); + output.addValue("executionId", result.executionId()); + output.addValue("resultStatus", result.status()); + output.addValue("resultJson", result.contextJson() == null ? new JSONObject().put("message", result.message()).toString() : result.contextJson()); + } + else + { + Integer publication = service().publish(actor, build, input.getValueString("note")); + output.addValue("publicationId", publication); + output.addValue("resultStatus", "PUBLISHED"); + } + } + } + + /*************************************************************************** + ** Rename/remove only within the pinned snapshot. Existing source validators + ** and optimistic save own path uniqueness, bounds and immutable persistence. + ***************************************************************************/ + private static void changeFilePath(RunBackendStepInput input, RunBackendStepOutput output, ScriptingAuthoring.Project project, Actor actor, Operation operation) throws ScriptingException + { + requireCurrentRevision(input, project); + List files = new ArrayList<>(files(actor, project.scriptId(), project.latestRevisionId())); + String selected = input.getValueString("path"); + int index = -1; + for(int i = 0; i < files.size(); i++) + { + if(files.get(i).path().equals(selected)) + { + index = i; + } + } + if(index < 0) + { + throw invalid("Choose an existing source file"); + } + if(operation == Operation.REMOVE_FILE) + { + if(files.size() == 1) + { + throw invalid("A source project must retain at least one file"); + } + files.remove(index); + } + else + { + String replacement = input.getValueString("newPath"); + if(selected.equals(replacement)) + { + throw invalid("Choose a different source path"); + } + try + { + files.set(index, new SourceFile(replacement, files.get(index).content())); + } + catch(IllegalArgumentException failure) + { + throw invalid("Enter a valid bounded relative TypeScript file"); + } + } + Integer revision = service().saveDraft(actor, project.scriptId(), input.getValueInteger("expectedRevisionId"), input.getValueString("message"), files); + output.addValue("revisionId", revision); + output.addValue("resultStatus", operation == Operation.RENAME_FILE ? "DRAFT_FILE_RENAMED" : "DRAFT_FILE_REMOVED"); + } + + /*************************************************************************** + ** Replace or append exactly one validated file; preserve the rest verbatim. + ***************************************************************************/ + private static void saveFile(RunBackendStepInput input, RunBackendStepOutput output, ScriptingAuthoring.Project project, Actor actor, Operation operation) throws ScriptingException + { + requireCurrentRevision(input, project); + List files = new ArrayList<>(files(actor, project.scriptId(), project.latestRevisionId())); + SourceFile replacement; + try + { + replacement = new SourceFile(input.getValueString("path"), input.getValueString("contents")); + } + catch(IllegalArgumentException invalid) + { + throw invalid("Enter a valid bounded relative TypeScript file"); + } + int match = -1; + for(int i = 0; i < files.size(); i++) + { + if(files.get(i).path().equalsIgnoreCase(replacement.path())) + { + match = i; + } + } + if(operation == Operation.ADD_FILE) + { + if(match >= 0) + { + throw invalid("This file already exists; use Edit source file"); + } + files.add(replacement); + } + else + { + if(match < 0 || !files.get(match).path().equals(replacement.path())) + { + throw invalid("Choose an existing source file"); + } + files.set(match, replacement); + } + Integer revision = service().saveDraft(actor, project.scriptId(), input.getValueInteger("expectedRevisionId"), input.getValueString("message"), files); + output.addValue("revisionId", revision); + output.addValue("resultStatus", "DRAFT_SAVED"); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingNavigation.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingNavigation.java new file mode 100644 index 0000000..36ec940 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingNavigation.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.DefaultWidgetRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier; +import com.kingsrook.qqq.backend.core.model.scripts.Script; + +/*************************************************************************** + ** Existing QQQ app/form/widget contracts expose isolated projects without a + ** client-specific route. Host-defined widgets and sections are preserved. + ***************************************************************************/ +public final class ScriptingNavigation +{ + public static final String VIEWER = "scriptViewer"; + + /*************************************************************************** + ** Metadata utility. + ***************************************************************************/ + private ScriptingNavigation() {} + + /*************************************************************************** + ** Supply only missing standard presentation metadata, never legacy actions. + ***************************************************************************/ + public static void register(QInstance instance) throws QException + { + if(instance.getWidget(VIEWER) == null) + { + instance.addWidget(new QWidgetMetaData().withName(VIEWER).withLabel("Contents").withIsCard(true) + .withType(WidgetType.SCRIPT_VIEWER.getType()).withCodeReference(new QCodeReference(DefaultWidgetRenderer.class))); + } + var script = instance.getTable(Script.TABLE_NAME); + if(script.getSections() == null || script.getSections().isEmpty()) + { + script.withSection(new QFieldSection("identity", new QIcon("badge"), Tier.T1, List.copyOf(script.getFields().keySet()))); + } + if(script.getSections() == null || script.getSections().stream().noneMatch(section -> VIEWER.equals(section.getWidgetName()))) + { + if(script.getSections() != null && script.getSections().stream().anyMatch(section -> "scriptSource".equals(section.getName()))) + { + throw new QException("Script source section name is already in use."); + } + script.withSection(new QFieldSection("scriptSource", new QIcon("data_object"), Tier.T2).withWidgetName(VIEWER)); + } + if(instance.getApp("scripting") == null) + { + instance.addApp(new QAppMetaData().withName("scripting").withLabel("Scripts").withIcon(new QIcon("code")) + .withSection(new QAppSection().withName("projects").withLabel("Scripts").withTables(List.of(Script.TABLE_NAME)) + .withProcesses(List.of(ScriptingAuthoringProcesses.CREATE_PROJECT)))); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingPermissions.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingPermissions.java new file mode 100644 index 0000000..eb87d41 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingPermissions.java @@ -0,0 +1,48 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +/******************************************************************************* + ** One permission per separate lifecycle action, named scripting.<action> + ** like the other QBits. Holding one never implies another: a drafter cannot + ** check, a checker cannot publish, a publisher cannot bind. + *******************************************************************************/ +public final class ScriptingPermissions +{ + public static final String WARM = "scripting.warm"; + public static final String VIEW = "scripting.view"; + public static final String DRAFT = "scripting.draft"; + public static final String CHECK = "scripting.check"; + public static final String TEST = "scripting.test"; + public static final String PUBLISH = "scripting.publish"; + public static final String BIND = "scripting.bind"; + public static final String EXECUTE = "scripting.execute"; + + + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private ScriptingPermissions() + { + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingProcessInputs.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingProcessInputs.java new file mode 100644 index 0000000..8239208 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingProcessInputs.java @@ -0,0 +1,101 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +/*************************************************************************** + ** Bounded native-form capability decoding; semantic limits remain in the shared + ** capability constructors and the signed-manifest execution service. + ***************************************************************************/ +final class ScriptingProcessInputs +{ + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private ScriptingProcessInputs() {} + + /*************************************************************************** + ** Reject unknown keys and coercible strings before the shared stored codec. + ***************************************************************************/ + static List capabilities(String json) throws ScriptingException + { + if(json == null || json.length() > WorkerProtocol.MAX_PAYLOAD_BYTES) + { + throw invalid(); + } + String bounded = WorkerProtocol.checkPayload("{\"capabilities\":" + json + "}"); + try + { + JSONObject envelope = new JSONObject(bounded); + if(!envelope.keySet().equals(Set.of("capabilities"))) + { + throw invalid(); + } + JSONArray array = envelope.getJSONArray("capabilities"); + if(array.length() > CapabilityDeclaration.MAX_DECLARATIONS) + { + throw invalid(); + } + for(int i = 0; i < array.length(); i++) + { + JSONObject entry = array.getJSONObject(i); + if(!entry.keySet().equals(Set.of("operation", "table", "fields", "maxRows", "maxCalls")) + || !"READ".equals(entry.get("operation")) || !(entry.get("table") instanceof String) + || !(entry.get("maxRows") instanceof Integer) || !(entry.get("maxCalls") instanceof Integer)) + { + throw invalid(); + } + JSONArray fields = entry.getJSONArray("fields"); + if(fields.length() > CapabilityDeclaration.MAX_FIELDS) + { + throw invalid(); + } + for(int j = 0; j < fields.length(); j++) + { + if(!(fields.get(j) instanceof String)) + { + throw invalid(); + } + } + } + return (CapabilityDeclaration.listFromJson(array.toString())); + } + catch(JSONException malformed) + { + throw invalid(); + } + } + + /*************************************************************************** + ** Never reflect untrusted JSON into diagnostic messages. + ***************************************************************************/ + private static ScriptingException invalid() + { + return (new ScriptingException(ScriptingException.Kind.INVALID, "Enter a bounded array of exact typed READ capabilities")); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingQBitConfig.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingQBitConfig.java new file mode 100644 index 0000000..0cbed2f --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingQBitConfig.java @@ -0,0 +1,163 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.List; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qbits.scripting.service.ScriptingService; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; +import com.kingsrook.qqq.backend.core.model.scripts.Script; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile; + + +/******************************************************************************* + ** Host configuration: the backend that holds the scripting tables, and the + ** runtime collaborators (worker, coordinator, optional importer). The host must + ** already define the core script, scriptRevision and scriptRevisionFile tables + ** (for example through core's ScriptsMetaDataProvider); this QBit builds on them + ** and does not create a second registry. + *******************************************************************************/ +public class ScriptingQBitConfig implements QBitConfig +{ + private String backendName; + private ScriptingRuntime runtime = new ScriptingRuntime(); + private transient ScriptingService service; + + + + /*************************************************************************** + ** The (existing) backend for the scripting tables. + ***************************************************************************/ + public ScriptingQBitConfig withBackendName(String value) + { + backendName = value; + return (this); + } + + + + /*************************************************************************** + ** The runtime collaborators; defaults are inert and fail closed. + ***************************************************************************/ + public ScriptingQBitConfig withRuntime(ScriptingRuntime value) + { + runtime = value; + return (this); + } + + + + /*************************************************************************** + ** The backend name. + ***************************************************************************/ + public String getBackendName() + { + return (backendName); + } + + + + /*************************************************************************** + ** The runtime. + ***************************************************************************/ + public ScriptingRuntime getRuntime() + { + return (runtime); + } + + + + /*************************************************************************** + ** The service created for this instance at production time. + ***************************************************************************/ + public ScriptingService getService() + { + return (service); + } + + + + /*************************************************************************** + ** Set by the producer once the instance's tables are in place. + ***************************************************************************/ + void setService(ScriptingService value) + { + service = value; + } + + + + /*************************************************************************** + ** Fail production early when the host skipped a prerequisite. + ***************************************************************************/ + @Override + public void validate(QInstance qInstance, List errors) + { + assertCondition(backendName != null && !backendName.isBlank(), "Scripting requires a backend name", errors); + assertCondition(runtime != null, "Scripting requires a runtime", errors); + for(String table : List.of(Script.TABLE_NAME, ScriptRevision.TABLE_NAME, ScriptRevisionFile.TABLE_NAME)) + { + assertCondition(qInstance.getTable(table) != null, "Scripting requires the core table [" + table + "] to be defined first", errors); + } + } + + + + /*************************************************************************** + ** Find the scripting service configured in an instance, or null when this QBit + ** is not installed (callers must then fail closed). + ***************************************************************************/ + public static ScriptingService serviceFor(QInstance instance) + { + for(QBitMetaData metadata : instance.getQBits().values()) + { + if(metadata.getConfig() instanceof ScriptingQBitConfig config && config.getService() != null) + { + return (config.getService()); + } + } + return (null); + } + + + + /*************************************************************************** + ** Exact instance configuration for native runtime and health lifecycle hooks. + ***************************************************************************/ + public static ScriptingQBitConfig configFor(QInstance instance) + { + if(instance != null) + { + for(QBitMetaData metadata : instance.getQBits().values()) + { + if(metadata.getConfig() instanceof ScriptingQBitConfig config) + { + return (config); + } + } + } + return (null); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingQBitProducer.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingQBitProducer.java new file mode 100644 index 0000000..c69c20d --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingQBitProducer.java @@ -0,0 +1,315 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.List; +import java.util.function.Consumer; +import com.kingsrook.qbits.scripting.service.IsolatedSourceGuard; +import com.kingsrook.qbits.scripting.service.ScriptingService; +import com.kingsrook.qbits.scripting.service.WarmRuntimeService; +import com.kingsrook.qbits.scripting.worker.WarmWorkerMaintenance; +import com.kingsrook.qqq.backend.core.actions.customizers.MultiCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.scripts.ScriptTableCustomizer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; + + +/******************************************************************************* + ** Registers the scripting-owned tables (builds, artifacts, publications, + ** bindings, execution history) and creates the instance's ScriptingService. The + ** project and draft-source tables are the existing core script tables and must + ** already be defined. Every scripting table is read-protected by scripting.view + ** and writable only by the service itself (see ScriptingTablePermission). + *******************************************************************************/ +public class ScriptingQBitProducer implements QBitProducer +{ + private ScriptingQBitConfig config; + private Consumer tableEnricher = table -> + { + // default: nothing; RDBMS hosts add their backend details here + }; + + + + /*************************************************************************** + ** Supply the host configuration. + ***************************************************************************/ + public ScriptingQBitProducer withConfig(ScriptingQBitConfig value) + { + config = value; + return (this); + } + + + + /*************************************************************************** + ** Let the host add backend-specific table details, for example RDBMS table + ** names, to each scripting table. + ***************************************************************************/ + public ScriptingQBitProducer withTableEnricher(Consumer value) + { + tableEnricher = value; + return (this); + } + + + + /*************************************************************************** + ** Validate prerequisites, add the tables, and install the service. + ***************************************************************************/ + @Override + public void produce(QInstance instance, String namespace) throws QException + { + if(config == null) + { + throw new QException("Scripting requires a configuration"); + } + if(namespace != null && !namespace.isBlank() && !"scripting".equals(namespace)) + { + throw new QException("The scripting QBit supports the scripting namespace only"); + } + config.validate(instance); + if(!instance.getTable("script").getFields().containsKey("executionMode")) + { + throw new QException("Scripting requires the reviewed core isolation marker metadata"); + } + requireSystemReadableProjectLocks(instance); + for(TableCustomizers hook : List.of(TableCustomizers.PRE_INSERT_RECORD, TableCustomizers.PRE_UPDATE_RECORD, TableCustomizers.PRE_DELETE_RECORD)) + { + appendGuard(instance.getTable("script"), hook, new QCodeReference(ScriptTableCustomizer.class)); + appendGuard(instance.getTable("scriptRevision"), hook, new QCodeReference(IsolatedSourceGuard.class)); + appendGuard(instance.getTable("scriptRevisionFile"), hook, new QCodeReference(IsolatedSourceGuard.class)); + } + for(String name : List.of("scriptRevision", "scriptRevisionFile")) + { + QTableMetaData source = instance.getTable(name); + QPermissionRules rules = PermissionsHelper.getEffectivePermissionRules(source, instance).clone(); + source.setPermissionRules(rules.withCustomPermissionChecker(SourceAggregatePermission.wrap(rules.getCustomPermissionChecker()))); + source.withoutCapabilities(Capability.TABLE_COUNT, Capability.QUERY_STATS); + } + appendGuard(instance.getTable("scriptRevision"), TableCustomizers.POST_QUERY_RECORD, new QCodeReference(IsolatedSourceGuard.class)); + appendGuard(instance.getTable("scriptRevisionFile"), TableCustomizers.POST_QUERY_RECORD, new QCodeReference(IsolatedSourceGuard.class)); + + instance.addQBit(new QBitMetaData().withGroupId("com.kingsrook.qbits").withArtifactId("qbit-scripting").withVersion(version()).withNamespace("scripting").withConfig(config)); + + instance.addTable(table(ScriptingTables.BUILD, "Script builds", + integer("script_id"), integer("script_revision_id"), string("source_sha256"), text("manifest_json"), string("manifest_attestation"), string("toolchain_id"), string("status"), + text("diagnostics_json"), string("requested_by"), dateTime("created_at"))); + instance.addTable(table(ScriptingTables.ARTIFACT, "Script artifacts", + integer("build_id"), integer("script_id"), integer("version"), string("source_sha256"), string("toolchain_id"), string("entry_point"), string("coordinator_id"), + text("bundle"), string("bundle_sha256"), integer("size_bytes"), text("source_map"), string("attestation"), dateTime("created_at")) + .withUniqueKey(new UniqueKey("build_id")).withUniqueKey(new UniqueKey("script_id", "version"))); + instance.addTable(table(ScriptingTables.PUBLICATION, "Script publications", + integer("script_id"), integer("version"), integer("build_id"), integer("artifact_id"), string("published_by"), dateTime("published_at"), string("note")) + .withUniqueKey(new UniqueKey("script_id", "version"))); + instance.addTable(table(ScriptingTables.BINDING, "Script hook bindings", + string("name"), string("table_name"), string("event"), integer("publication_id"), text("grants_json"), text("patchable_fields"), integer("enabled"), string("active_slot"), + string("created_by"), dateTime("created_at")) + .withUniqueKey(new UniqueKey("active_slot"))); + instance.addTable(table(ScriptingTables.EXECUTION, "Script executions", + string("operation_id").withMaxLength(64), string("kind"), integer("build_id"), integer("binding_id"), integer("publication_id"), integer("artifact_id"), string("actor_id"), string("event"), + string("status"), integer("duration_ms"), string("message"), string("result_action"), dateTime("created_at"))); + instance.addTable(table(ScriptingTables.IMPORT, "Script source imports", + integer("script_id"), integer("script_revision_id"), string("repository"), string("commit_sha"), string("subdirectory"), + string("source_sha256"), string("imported_by"), dateTime("created_at")) + .withUniqueKey(new UniqueKey("script_revision_id"))); + + ScriptingAuthoringProcesses.register(instance); + ScriptingNavigation.register(instance); + config.setService(new ScriptingService(config.getRuntime())); + if(config.getRuntime().getWorker() instanceof WarmWorkerMaintenance) + { + instance.withRuntimeService(new QCodeReference(WarmRuntimeService.class)); + HealthChecksMetaData.ofOrWithNew(instance).withCheck(new HealthCheckMetaData().withName("scripting.warm") + .withLabel("Scripting warm code").withComponent("scripting").withIncludeInReadiness(true) + .withFailureThreshold(1).withIntervalSeconds(10).withTimeoutMs(5000).withProbeCache(false) + .withCheck(new QCodeReference(WarmRuntimeService.class))); + } + } + + + + /*************************************************************************** + ** The QBit alone reads and writes its project tables as the QQQ system session, + ** after the caller has been authorized. A host may lock those tables with + ** native record security (for example by tenant); QQQ lets the system session + ** through only for a key type that defines an all-access key. Without it every + ** internal read would silently see no rows and even the owner's authoring would + ** fail as "not found", so refuse the configuration with the exact remedy. + ***************************************************************************/ + private static void requireSystemReadableProjectLocks(QInstance instance) throws QException + { + for(String tableName : List.of("script", "scriptRevision", "scriptRevisionFile")) + { + for(var lock : flatten(instance.getTable(tableName).getRecordSecurityLocks())) + { + var keyType = lock.getSecurityKeyType() == null ? null : instance.getSecurityKeyType(lock.getSecurityKeyType()); + if(keyType != null && (keyType.getAllAccessKeyName() == null || keyType.getAllAccessKeyName().isBlank())) + { + throw new QException("Scripting reads its project tables as the QQQ system session, but the record-security lock on table " + tableName + + " uses key type " + keyType.getName() + " which defines no all-access key. Configure QSecurityKeyType.withAllAccessKeyName for it."); + } + } + } + } + + + + /*************************************************************************** + ** Leaf locks of a possibly nested lock list. + ***************************************************************************/ + private static List flatten( + List locks) + { + List leaves = new java.util.ArrayList<>(); + for(var lock : locks == null ? List.of() : locks) + { + if(lock instanceof com.kingsrook.qqq.backend.core.model.metadata.security.MultiRecordSecurityLock multi) + { + leaves.addAll(flatten(multi.getLocks())); + } + else + { + leaves.add(lock); + } + } + return (leaves); + } + + + + /*************************************************************************** + ** Keep host behavior and apply integrity checks to its final record values. + ***************************************************************************/ + private static void appendGuard(QTableMetaData table, TableCustomizers hook, QCodeReference guard) + { + QCodeReference prior = table.getCustomizer(hook.getRole()).orElse(null); + if(prior == null) + { + table.withCustomizer(hook, guard); + } + else + { + table.getCustomizers().put(hook.getRole(), MultiCustomizer.of(prior, guard)); + } + } + + + + /*************************************************************************** + ** The build-produced JAR version; IDE/test class directories say so honestly. + ***************************************************************************/ + private static String version() + { + String version = ScriptingQBitProducer.class.getPackage().getImplementationVersion(); + return (version == null || version.isBlank() ? "development" : version); + } + + + + /*************************************************************************** + ** One scripting table: integer id, the given fields, all non-editable in the + ** UI, protected by the shared table permission rule. + ***************************************************************************/ + private QTableMetaData table(String name, String label, QFieldMetaData... fields) + { + QTableMetaData table = new QTableMetaData() + .withName(name) + .withLabel(label) + .withBackendName(config.getBackendName()) + .withPrimaryKeyField("id") + .withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(ScriptingTablePermission.class)) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withCustomPermissionChecker(new QCodeReference(ScriptingTablePermission.class))) + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withLabel("Id").withIsEditable(false)); + for(QFieldMetaData field : fields) + { + table.withField(field); + } + tableEnricher.accept(table); + return (table); + } + + + + /*************************************************************************** + ** Read-only string field with a readable label. + ***************************************************************************/ + private static QFieldMetaData string(String name) + { + return (field(name, QFieldType.STRING)); + } + + + + /*************************************************************************** + ** Read-only long text field. + ***************************************************************************/ + private static QFieldMetaData text(String name) + { + return (field(name, QFieldType.TEXT)); + } + + + + /*************************************************************************** + ** Read-only integer field. + ***************************************************************************/ + private static QFieldMetaData integer(String name) + { + return (field(name, QFieldType.INTEGER)); + } + + + + /*************************************************************************** + ** Read-only timestamp field. + ***************************************************************************/ + private static QFieldMetaData dateTime(String name) + { + return (field(name, QFieldType.DATE_TIME)); + } + + + + /*************************************************************************** + ** Build a field whose label is derived from its snake_case name. + ***************************************************************************/ + private static QFieldMetaData field(String name, QFieldType type) + { + String label = name.substring(0, 1).toUpperCase(java.util.Locale.ROOT) + name.substring(1).replace('_', ' '); + return (new QFieldMetaData(name, type).withLabel(label).withIsEditable(false)); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingTablePermission.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingTablePermission.java new file mode 100644 index 0000000..28f3785 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingTablePermission.java @@ -0,0 +1,156 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; + + +/******************************************************************************* + ** Permission rule for the scripting-owned tables. Reads need scripting.view. + ** Writes are accepted only from the system session ScriptingService switches to + ** after it has authorized the human actor, so no user (administrator or not) can + ** forge a build, artifact, publication or execution row through generic CRUD. + *******************************************************************************/ +public class ScriptingTablePermission extends UseOtherPermissionNameCustomPermissionChecker implements TableCustomizerInterface +{ + /*************************************************************************** + ** Expose the read permission in QQQ's available-permission catalog. + ***************************************************************************/ + public ScriptingTablePermission() + { + withPermissionName(ScriptingPermissions.VIEW); + } + + + + /*************************************************************************** + ** Enforce on every action, including direct backend-step invocation. + ***************************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException + { + boolean isWrite = input instanceof InsertInput || input instanceof UpdateInput || input instanceof DeleteInput; + if(isWrite) + { + if(!(QContext.getQSession() instanceof QSystemUserSession)) + { + throw new QPermissionDeniedException("Scripting records are written by the scripting service only"); + } + return; + } + + if(QContext.getQSession() == null || !QContext.getQSession().hasPermission(ScriptingPermissions.VIEW)) + { + throw new QPermissionDeniedException("Permission " + ScriptingPermissions.VIEW + " is required"); + } + if(!(QContext.getQSession() instanceof QSystemUserSession) && (input instanceof CountInput || input instanceof AggregateInput)) + { + throw new QPermissionDeniedException("Project-scoped scripting counts and aggregates require a dedicated authorized endpoint"); + } + } + + + + /*************************************************************************** + ** Generic views cannot reveal another project's sources, diagnostics or + ** history. Internal storage runs as the trusted system session. Queries + ** omitting association keys fail closed; pagination is filtered afterwards. + ***************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface input, List records) throws QException + { + if(QContext.getQSession() instanceof QSystemUserSession) + { + return (records); + } + List visible = new ArrayList<>(); + for(QRecord row : records) + { + try + { + Integer project = row.getValueInteger("script_id"); + if(ScriptingTables.BINDING.equals(input.getTableName())) + { + project = accessible(ScriptingTables.PUBLICATION, row.getValueInteger("publication_id")).getValueInteger("script_id"); + } + else if(ScriptingTables.EXECUTION.equals(input.getTableName())) + { + project = accessible(ScriptingTables.BUILD, row.getValueInteger("build_id")).getValueInteger("script_id"); + } + accessible("script", project); + visible.add(row); + } + catch(QException e) + { + if(input instanceof GetInput) + { + throw new QNotFoundException("Scripting record was not found"); + } + } + } + return (visible); + } + + + + /*************************************************************************** + ** Fetch an associated row under the current caller with enforced ACLs. + ***************************************************************************/ + private static QRecord accessible(String table, Integer id) throws QException + { + if(id == null) + { + throw new QPermissionDeniedException("Scripting association is not accessible"); + } + GetInput input = new GetInput(); + input.setTableName(table); + input.setPrimaryKey(id); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + QRecord record = new GetAction().execute(input).getRecord(); + if(record == null) + { + throw new QPermissionDeniedException("Scripting association is not accessible"); + } + return (record); + } +} + diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingTables.java b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingTables.java new file mode 100644 index 0000000..51b42db --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/ScriptingTables.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +/******************************************************************************* + ** Stable table names. Projects and draft sources are NOT here: they live in the + ** existing core script, scriptRevision and scriptRevisionFile tables, which the + ** host must already define (see ScriptingQBitProducer). + *******************************************************************************/ +public final class ScriptingTables +{ + public static final String BUILD = "scripting_build"; + public static final String ARTIFACT = "scripting_artifact"; + public static final String PUBLICATION = "scripting_publication"; + public static final String BINDING = "scripting_binding"; + public static final String EXECUTION = "scripting_execution"; + public static final String IMPORT = "scripting_import"; + + + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private ScriptingTables() + { + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/SourceAggregatePermission.java b/base/src/main/java/com/kingsrook/qbits/scripting/SourceAggregatePermission.java new file mode 100644 index 0000000..5a24bd5 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/SourceAggregatePermission.java @@ -0,0 +1,161 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.Map; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.permissions.AvailablePermission; +import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionSubType; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.metadata.code.InitializableViaCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceWithProperties; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; + + +/******************************************************************************* + ** Opted in by the scripting producer on reused source tables. SQL aggregate + ** results cannot be filtered by the parent ACL/post-query policy, so ordinary + ** callers must use bounded authorized project history instead of generic totals. + ** Trusted internal system work and the existing host checker remain intact. + *******************************************************************************/ +public class SourceAggregatePermission implements CustomPermissionChecker, InitializableViaCodeReference +{ + private CustomPermissionChecker delegate; + + + + /*************************************************************************** + ** Preserve an existing host checker without a mutable global registry. + ***************************************************************************/ + public static QCodeReference wrap(QCodeReference previous) + { + return (new QCodeReferenceWithProperties(SourceAggregatePermission.class, previous == null ? Map.of() : Map.of("delegate", previous))); + } + + + + /*************************************************************************** + ** Instantiate only the trusted metadata-supplied host permission checker. + ***************************************************************************/ + @Override + public void initialize(QCodeReference reference) + { + if(reference instanceof QCodeReferenceWithProperties properties && properties.getProperties().get("delegate") instanceof QCodeReference previous) + { + delegate = QCodeLoader.getAdHoc(CustomPermissionChecker.class, previous); + } + } + + + + /*************************************************************************** + ** Aggregate denial is independent of requested filters and table read rights. + ** Preserve exact ordinary read/write permission subtypes for other actions. + ***************************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException + { + if((input instanceof CountInput || input instanceof AggregateInput) && !(QContext.getQSession() instanceof QSystemUserSession)) + { + throw new QPermissionDeniedException("Source totals require an authorized project history endpoint"); + } + if(input instanceof AbstractTableActionInput tableInput && !metadata.getName().equals(tableInput.getTableName()) + && !(QContext.getQSession() instanceof QSystemUserSession)) + { + throw new QPermissionDeniedException("Joined source reads cannot apply the owning project visibility policy"); + } + if(input instanceof QueryOrGetInputInterface query && query.getQueryJoins() != null && !query.getQueryJoins().isEmpty() + && !(QContext.getQSession() instanceof QSystemUserSession)) + { + throw new QPermissionDeniedException("Source reads cannot join rows outside their parent visibility policy"); + } + if(delegate != null) + { + delegate.checkPermissionsThrowing(input, metadata); + return; + } + QPermissionRules rules = metadata.getPermissionRules(); + if(!(input instanceof AbstractTableActionInput)) + { + QTableMetaData view = new QTableMetaData().withName(metadata.getName()).withPermissionRules(rules.clone().withCustomPermissionChecker(null)); + if(PermissionsHelper.getPermissionCheckResult(input, view) != PermissionCheckResult.ALLOW) + { + throw new QPermissionDeniedException("Source permission denied"); + } + return; + } + TablePermissionSubType type = input instanceof InsertInput ? TablePermissionSubType.INSERT + : input instanceof UpdateInput ? TablePermissionSubType.EDIT : input instanceof DeleteInput ? TablePermissionSubType.DELETE : TablePermissionSubType.READ; + String suffix = switch(rules.getLevel()) + { + case NOT_PROTECTED -> null; + case HAS_ACCESS_PERMISSION -> "hasAccess"; + case READ_WRITE_PERMISSIONS -> type == TablePermissionSubType.READ ? "read" : "write"; + case READ_INSERT_EDIT_DELETE_PERMISSIONS -> type.getPermissionSuffix(); + }; + String base = PermissionsHelper.getEffectivePermissionBaseName(rules, metadata.getName()); + if(suffix != null && (QContext.getQSession() == null || !QContext.getQSession().hasPermission(base + "." + suffix))) + { + throw new QPermissionDeniedException("Source permission denied"); + } + } + + + + /*************************************************************************** + ** Preserve specialized host permission catalog entries. + ***************************************************************************/ + @Override + public boolean handlesBuildAvailablePermission() + { + return (delegate != null && delegate.handlesBuildAvailablePermission()); + } + + + + /*************************************************************************** + ** Catalog delegation does not introduce new authorization names. + ***************************************************************************/ + @Override + public AvailablePermission buildAvailablePermission(QPermissionRules rules, PermissionSubType type, String base, MetaDataWithName metadata, String objectType) + { + return (delegate == null ? null : delegate.buildAvailablePermission(rules, type, base, metadata, objectType)); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/build/ArtifactSubmission.java b/base/src/main/java/com/kingsrook/qbits/scripting/build/ArtifactSubmission.java new file mode 100644 index 0000000..04785e5 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/build/ArtifactSubmission.java @@ -0,0 +1,32 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.build; + + +/******************************************************************************* + ** A compiled bundle plus the provenance the trusted coordinator vouches for. The + ** attestation is an HMAC by the coordinator's key over the source hash, + ** toolchain, entry point and bundle hash, so JavaScript supplied by anyone else + ** (a client, an IDE, a database editor) cannot be accepted as "compiled". + *******************************************************************************/ +public record ArtifactSubmission(String coordinatorId, String sourceSha256, String toolchainId, String entryPoint, String bundle, String bundleSha256, String sourceMap, String attestation) +{ +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/build/BuildCoordinator.java b/base/src/main/java/com/kingsrook/qbits/scripting/build/BuildCoordinator.java new file mode 100644 index 0000000..fdd0cde --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/build/BuildCoordinator.java @@ -0,0 +1,241 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.build; + + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.GeneralSecurityException; +import java.security.MessageDigest; +import java.util.List; +import java.util.UUID; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.CompileRequest; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.CompileResponse; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Diagnostic; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import com.kingsrook.qbits.scripting.worker.WorkerProvider; + + +/******************************************************************************* + ** The configured trusted build coordinator. It is the only thing that turns a + ** worker's compile answer into an artifact submission, and the only thing that + ** accepts one. It does not trust the worker: it re-hashes the returned bundle, + ** compares the toolchain to its own pinned identity, requires strict mode, and + ** then signs. Verification before storing AND before every execution makes a + ** tampered stored bundle or a forged submission fail closed. + ** + ** The signing key is host configuration held only in this object; it is never + ** sent to a worker. Known limit: it lives in the application JVM, so it defends + ** against forged data and database edits, not against a compromised host process + ** (an out-of-process signer is a hardening item in docs/TODO.md). + *******************************************************************************/ +public class BuildCoordinator +{ + public static final int MIN_KEY_BYTES = 32; + + private final String coordinatorId; + private final byte[] key; + private final String toolchainId; + + + + /*************************************************************************** + ** Create a coordinator with a pinned toolchain identity and signing key. + ***************************************************************************/ + public BuildCoordinator(String coordinatorId, byte[] key, String toolchainId) + { + if(coordinatorId == null || coordinatorId.isBlank() || toolchainId == null || toolchainId.isBlank() || key == null || key.length < MIN_KEY_BYTES) + { + throw new IllegalArgumentException("Coordinator id, toolchain id and a key of at least " + MIN_KEY_BYTES + " bytes are required"); + } + this.coordinatorId = coordinatorId; + this.key = key.clone(); + this.toolchainId = toolchainId; + } + + + + /*************************************************************************** + ** The pinned toolchain identity every artifact must carry. + ***************************************************************************/ + public String getToolchainId() + { + return (toolchainId); + } + + + + /*************************************************************************** + ** Compile a verified snapshot through the worker and return the verdict. + ** Worker unavailability propagates as UNAVAILABLE; it is never a "success". + ***************************************************************************/ + public BuildOutcome build(WorkerProvider worker, SourceSnapshot snapshot, String entryPoint) throws ScriptingException + { + if(!snapshot.verify()) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Source snapshot does not match its hash"); + } + if(entryPoint == null || !snapshot.hasFile(entryPoint)) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Entry point is not a file of the snapshot"); + } + + String requestId = "b-" + UUID.randomUUID(); + String requestJson = WorkerProtocol.encodeCompile(new CompileRequest(requestId, toolchainId, entryPoint, snapshot.sha256(), snapshot.files())); + CompileResponse response = WorkerProtocol.decodeCompileResponse(worker.compile(requestJson)); + + if(!requestId.equals(response.requestId())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Worker answered a different request"); + } + if(!toolchainId.equals(response.toolchainId())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Worker toolchain does not match the pinned toolchain"); + } + if(!response.strict()) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Worker did not compile in strict mode"); + } + + if(!response.success()) + { + List diagnostics = response.diagnostics().isEmpty() + ? List.of(new Diagnostic("", 0, 0, "BUILD", "The build failed without diagnostics")) + : response.diagnostics(); + return (new BuildOutcome(false, diagnostics, null)); + } + + String actualSha = sha256Hex(response.bundle()); + if(!actualSha.equals(response.bundleSha256())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Bundle hash returned by the worker does not match the bundle"); + } + + ArtifactSubmission submission = new ArtifactSubmission(coordinatorId, snapshot.sha256(), toolchainId, entryPoint, response.bundle(), actualSha, response.sourceMap(), attest(snapshot.sha256(), toolchainId, entryPoint, actualSha)); + return (new BuildOutcome(true, response.diagnostics(), submission)); + } + + + + /*************************************************************************** + ** Verify a submission or a re-assembled stored artifact: right coordinator, + ** right source (when given), pinned toolchain, bundle hash and attestation. + ***************************************************************************/ + public void verify(ArtifactSubmission submission, String expectedSourceSha256) throws ScriptingException + { + if(submission == null || !coordinatorId.equals(submission.coordinatorId())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Artifact was not produced by the configured build coordinator"); + } + if(expectedSourceSha256 != null && !expectedSourceSha256.equals(submission.sourceSha256())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Artifact does not match the source it claims"); + } + if(!toolchainId.equals(submission.toolchainId())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Artifact toolchain does not match the pinned toolchain"); + } + if(submission.bundle() == null || !sha256Hex(submission.bundle()).equals(submission.bundleSha256())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Artifact bytes do not match their hash"); + } + byte[] expected = attest(submission.sourceSha256(), submission.toolchainId(), submission.entryPoint(), submission.bundleSha256()).getBytes(StandardCharsets.UTF_8); + byte[] actual = String.valueOf(submission.attestation()).getBytes(StandardCharsets.UTF_8); + if(!MessageDigest.isEqual(expected, actual)) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Artifact attestation is invalid"); + } + } + + + + /*************************************************************************** + ** HMAC-SHA256 over a canonical, newline-delimited statement of provenance. + ***************************************************************************/ + private String attest(String sourceSha256, String toolchain, String entryPoint, String bundleSha256) + { + return (sign("qbit-scripting-artifact/1\n" + coordinatorId + "\n" + sourceSha256 + "\n" + toolchain + "\n" + entryPoint + "\n" + bundleSha256)); + } + + + + /*************************************************************************** + ** Seal the exact host-created manifest, including project identity and + ** capability declarations. This signature is never supplied by a worker. + ***************************************************************************/ + public String attestManifest(String manifest) + { + return (sign("qbit-scripting-manifest/1\n" + coordinatorId + "\n" + manifest)); + } + + + + /*************************************************************************** + ** Verify the stored manifest before interpreting its authority. + ***************************************************************************/ + public void verifyManifest(String manifest, String attestation) throws ScriptingException + { + if(manifest == null || attestation == null || !MessageDigest.isEqual(attestManifest(manifest).getBytes(StandardCharsets.UTF_8), attestation.getBytes(StandardCharsets.UTF_8))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Build manifest attestation is invalid"); + } + } + + + + /*************************************************************************** + ** HMAC a domain-separated host statement. + ***************************************************************************/ + private String sign(String statement) + { + try + { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return (SourceSnapshot.hex(mac.doFinal(statement.getBytes(StandardCharsets.UTF_8)))); + } + catch(GeneralSecurityException e) + { + throw new IllegalStateException("HmacSHA256 is required by the Java platform", e); + } + } + + + + /*************************************************************************** + ** SHA-256 of the UTF-8 bytes, lower-case hex. + ***************************************************************************/ + public static String sha256Hex(String text) + { + try + { + return (SourceSnapshot.hex(MessageDigest.getInstance("SHA-256").digest(text.getBytes(StandardCharsets.UTF_8)))); + } + catch(GeneralSecurityException e) + { + throw new IllegalStateException("SHA-256 is required by the Java platform", e); + } + } +} + diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/build/BuildOutcome.java b/base/src/main/java/com/kingsrook/qbits/scripting/build/BuildOutcome.java new file mode 100644 index 0000000..4fa6cfd --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/build/BuildOutcome.java @@ -0,0 +1,46 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.build; + + +import java.util.List; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Diagnostic; + + +/******************************************************************************* + ** The coordinator's verdict. A submission exists only when the compile + ** succeeded in strict mode with zero errors; a failed build has none and so can + ** never be stored as an artifact or published. + *******************************************************************************/ +public record BuildOutcome(boolean success, List diagnostics, ArtifactSubmission submission) +{ + /*************************************************************************** + ** Freeze the diagnostics. + ***************************************************************************/ + public BuildOutcome + { + diagnostics = List.copyOf(diagnostics); + if(success != (submission != null)) + { + throw new IllegalArgumentException("A submission must exist exactly when the build succeeded"); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/capability/BrokerRequest.java b/base/src/main/java/com/kingsrook/qbits/scripting/capability/BrokerRequest.java new file mode 100644 index 0000000..a566807 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/capability/BrokerRequest.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.capability; + + +import java.util.List; +import java.util.Set; + + +/******************************************************************************* + ** A worker's read request, already parsed and shape-checked by WorkerProtocol. + ** It is only a request: the broker still checks the capability token, the + ** grant, every field and the budgets, and QQQ still checks the actor's access. + *******************************************************************************/ +public record BrokerRequest(String token, String table, List fields, List criteria, int limit) +{ + public static final Set OPERATORS = Set.of("EQUALS", "NOT_EQUALS", "IN", "LESS_THAN", "LESS_THAN_OR_EQUALS", "GREATER_THAN", "GREATER_THAN_OR_EQUALS"); + public static final int MAX_CRITERIA = 10; + public static final int MAX_VALUES = 50; + + + + /*************************************************************************** + ** Freeze the lists. + ***************************************************************************/ + public BrokerRequest + { + fields = List.copyOf(fields); + criteria = List.copyOf(criteria); + } + + + + /******************************************************************************* + ** One filter condition. Values are strings, numbers or booleans only. + *******************************************************************************/ + public record Criterion(String field, String operator, List values) + { + /*************************************************************************** + ** Freeze the values. + ***************************************************************************/ + public Criterion + { + values = List.copyOf(values); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/capability/CapabilityBroker.java b/base/src/main/java/com/kingsrook/qbits/scripting/capability/CapabilityBroker.java new file mode 100644 index 0000000..1299628 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/capability/CapabilityBroker.java @@ -0,0 +1,253 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.capability; + + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; + + +/******************************************************************************* + ** Issues opaque, short-lived, scoped capability tokens and answers worker read + ** requests. A token is random bytes carrying no data; the broker keeps only its + ** hash and the grants behind it, so a worker can neither read nor widen its own + ** authority. Each read is checked here (token, table, fields, criteria fields, + ** call and row budgets, response size) and then again by the RecordReader under + ** the current actor's QQQ permissions. + ** + ** Default is read-only: the only operation is a bounded query. State is + ** per-broker-instance (no static registry); a token dies on close or expiry. + *******************************************************************************/ +public class CapabilityBroker +{ + public static final int MAX_RESPONSE_BYTES = 256 * 1024; + public static final int MAX_OPEN_LEASES = 1000; + + private final RecordReader reader; + private final Clock clock; + private final SecureRandom random = new SecureRandom(); + private final Map leases = new ConcurrentHashMap<>(); + + + + /******************************************************************************* + ** One open token: its grants, budgets and expiry. + *******************************************************************************/ + private record Lease(Instant expiresAt, Map grants) + { + } + + + + /******************************************************************************* + ** Remaining budget for one granted table. + *******************************************************************************/ + private record GrantState(CapabilityDeclaration declaration, AtomicInteger callsLeft, AtomicInteger rowsLeft) + { + } + + + + /*************************************************************************** + ** Create a broker over a host-side reader. + ***************************************************************************/ + public CapabilityBroker(RecordReader reader, Clock clock) + { + this.reader = reader; + this.clock = clock; + } + + + + /*************************************************************************** + ** Open a lease for exactly these grants and return the opaque token. Total + ** rows a lease can return is maxRows times maxCalls per table. + ***************************************************************************/ + public synchronized String open(List grants, Duration ttl) throws ScriptingException + { + if(ttl == null || ttl.isZero() || ttl.isNegative() || ttl.compareTo(Duration.ofMinutes(2)) > 0) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Capability lease lifetime is outside policy"); + } + Instant now = clock.instant(); + leases.values().removeIf(lease -> lease.expiresAt().isBefore(now)); + if(leases.size() >= MAX_OPEN_LEASES) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Too many open capability leases"); + } + + Map states = new HashMap<>(); + for(CapabilityDeclaration grant : grants) + { + states.put(grant.tableName(), new GrantState(grant, new AtomicInteger(grant.maxCalls()), new AtomicInteger(grant.maxRows() * grant.maxCalls()))); + } + + byte[] bytes = new byte[32]; + random.nextBytes(bytes); + String token = "qct_" + Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + leases.put(hash(token), new Lease(now.plus(ttl), Map.copyOf(states))); + return (token); + } + + + + /*************************************************************************** + ** Revoke a token immediately. + ***************************************************************************/ + public void close(String token) + { + leases.remove(hash(token)); + } + + + + /*************************************************************************** + ** Number of open leases, for tests and diagnostics. + ***************************************************************************/ + public int openLeaseCount() + { + return (leases.size()); + } + + + + /*************************************************************************** + ** Answer one worker request. Never throws: every refusal is a bounded, + ** generic error response. + ***************************************************************************/ + public String handle(String requestJson) + { + try + { + BrokerRequest request = WorkerProtocol.decodeBrokerRequest(requestJson); + Lease lease = leases.get(hash(request.token())); + if(lease == null || !lease.expiresAt().isAfter(clock.instant())) + { + return (WorkerProtocol.encodeBrokerError("DENIED", "Capability token is not valid")); + } + GrantState state = lease.grants().get(request.table()); + if(state == null) + { + return (WorkerProtocol.encodeBrokerError("DENIED", "Table is not granted")); + } + synchronized(state) + { + return (read(request, state)); + } + } + catch(ScriptingException e) + { + return (WorkerProtocol.encodeBrokerError(e.getKind().name(), e.getMessage())); + } + catch(RuntimeException e) + { + return (WorkerProtocol.encodeBrokerError("UNAVAILABLE", "Read could not be completed")); + } + } + + + + /*************************************************************************** + ** Apply the grant to one request and read. + ***************************************************************************/ + private String read(BrokerRequest request, GrantState state) throws ScriptingException + { + CapabilityDeclaration grant = state.declaration(); + if(!grant.fields().containsAll(request.fields())) + { + return (WorkerProtocol.encodeBrokerError("DENIED", "A requested field is not granted")); + } + for(BrokerRequest.Criterion criterion : request.criteria()) + { + if(!grant.fields().contains(criterion.field())) + { + return (WorkerProtocol.encodeBrokerError("DENIED", "A filtered field is not granted")); + } + } + if(request.limit() < 1) + { + return (WorkerProtocol.encodeBrokerError("INVALID", "Limit must be at least 1")); + } + + if(state.callsLeft().decrementAndGet() < 0) + { + return (WorkerProtocol.encodeBrokerError("LIMIT", "Call budget is exhausted")); + } + int limit = Math.min(request.limit(), Math.min(grant.maxRows(), state.rowsLeft().get())); + if(limit < 1) + { + return (WorkerProtocol.encodeBrokerError("LIMIT", "Row budget is exhausted")); + } + + List> rows = reader.read(request.table(), request.fields(), request.criteria(), limit); + if(rows.size() > limit) + { + rows = rows.subList(0, limit); + } + + boolean truncated = false; + List> kept = new ArrayList<>(); + for(Map row : rows) + { + kept.add(row); + if(WorkerProtocol.encodeBrokerResponse(kept, false).getBytes(StandardCharsets.UTF_8).length > MAX_RESPONSE_BYTES) + { + kept.remove(kept.size() - 1); + truncated = true; + break; + } + } + state.rowsLeft().addAndGet(-kept.size()); + return (WorkerProtocol.encodeBrokerResponse(kept, truncated)); + } + + + + /*************************************************************************** + ** Only the hash of a token is ever stored. + ***************************************************************************/ + private static String hash(String token) + { + try + { + return (SourceSnapshot.hex(MessageDigest.getInstance("SHA-256").digest(String.valueOf(token).getBytes(StandardCharsets.UTF_8)))); + } + catch(NoSuchAlgorithmException e) + { + throw new IllegalStateException("SHA-256 is required by the Java platform", e); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/capability/CapabilityDeclaration.java b/base/src/main/java/com/kingsrook/qbits/scripting/capability/CapabilityDeclaration.java new file mode 100644 index 0000000..f1fd030 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/capability/CapabilityDeclaration.java @@ -0,0 +1,170 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.capability; + + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Pattern; +import com.kingsrook.qbits.scripting.ScriptingException; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + + +/******************************************************************************* + ** A read-only grant on one table: which fields may be read and how much. The same + ** type is used twice. A project's build manifest DECLARES what it may need; a + ** binding GRANTS some or all of that, never more. Writes are not representable: + ** a future write capability needs its own explicit type, audit and review. + *******************************************************************************/ +public record CapabilityDeclaration(String tableName, Set fields, int maxRows, int maxCalls) +{ + public static final int ABSOLUTE_MAX_ROWS = 500; + public static final int ABSOLUTE_MAX_CALLS = 20; + public static final int MAX_FIELDS = 50; + public static final int MAX_DECLARATIONS = 20; + + private static final Pattern NAME = Pattern.compile("[A-Za-z][A-Za-z0-9_]{0,99}"); + + + + /*************************************************************************** + ** Validate and freeze (sorted) so JSON and hashes are deterministic. + ***************************************************************************/ + public CapabilityDeclaration + { + if(tableName == null || !NAME.matcher(tableName).matches()) + { + throw new IllegalArgumentException("Invalid table name in capability"); + } + if(fields == null || fields.isEmpty() || fields.size() > MAX_FIELDS) + { + throw new IllegalArgumentException("A capability needs between 1 and " + MAX_FIELDS + " fields"); + } + for(String field : fields) + { + if(field == null || !NAME.matcher(field).matches()) + { + throw new IllegalArgumentException("Invalid field name in capability"); + } + } + if(maxRows < 1 || maxRows > ABSOLUTE_MAX_ROWS || maxCalls < 1 || maxCalls > ABSOLUTE_MAX_CALLS) + { + throw new IllegalArgumentException("Capability budgets must be within 1.." + ABSOLUTE_MAX_ROWS + " rows and 1.." + ABSOLUTE_MAX_CALLS + " calls"); + } + fields = Collections.unmodifiableSet(new TreeSet<>(fields)); + } + + + + /*************************************************************************** + ** True when this grant asks for nothing beyond what the declaration allows. + ***************************************************************************/ + public boolean isWithin(CapabilityDeclaration declared) + { + return (tableName.equals(declared.tableName) + && declared.fields.containsAll(fields) + && maxRows <= declared.maxRows + && maxCalls <= declared.maxCalls); + } + + + + /*************************************************************************** + ** True when every grant in the list is within some declaration. + ***************************************************************************/ + public static boolean allWithin(List grants, List declared) + { + for(CapabilityDeclaration grant : grants) + { + if(declared.stream().noneMatch(grant::isWithin)) + { + return (false); + } + } + return (true); + } + + + + /*************************************************************************** + ** Canonical JSON object. + ***************************************************************************/ + public JSONObject toJson() + { + return (new JSONObject().put("table", tableName).put("fields", new JSONArray(fields)).put("maxRows", maxRows).put("maxCalls", maxCalls).put("operation", "READ")); + } + + + + /*************************************************************************** + ** Serialize a list; order is preserved. + ***************************************************************************/ + public static String listToJson(List list) + { + JSONArray array = new JSONArray(); + list.forEach(d -> array.put(d.toJson())); + return (array.toString()); + } + + + + /*************************************************************************** + ** Parse a stored list. Any shape problem, or any operation other than READ, + ** is rejected. + ***************************************************************************/ + public static List listFromJson(String json) throws ScriptingException + { + try + { + JSONArray array = new JSONArray(json); + if(array.length() > MAX_DECLARATIONS) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Too many capability declarations"); + } + List list = new ArrayList<>(); + for(int i = 0; i < array.length(); i++) + { + JSONObject object = array.getJSONObject(i); + if(!"READ".equals(object.optString("operation"))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Only READ capabilities are supported"); + } + Set fields = new TreeSet<>(); + JSONArray fieldArray = object.getJSONArray("fields"); + for(int j = 0; j < fieldArray.length(); j++) + { + fields.add(fieldArray.getString(j)); + } + list.add(new CapabilityDeclaration(object.getString("table"), fields, object.getInt("maxRows"), object.getInt("maxCalls"))); + } + return (List.copyOf(list)); + } + catch(JSONException | IllegalArgumentException e) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Capability declarations are malformed"); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/capability/QqqRecordReader.java b/base/src/main/java/com/kingsrook/qbits/scripting/capability/QqqRecordReader.java new file mode 100644 index 0000000..b46bc09 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/capability/QqqRecordReader.java @@ -0,0 +1,168 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.capability; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Reads through the ordinary QQQ QueryAction under the CURRENT session, so table + ** permission and record security locks are enforced by QQQ for every call, not + ** cached at grant time. On top of that the host verifies that the table and each + ** field exist, are visible, and are not password or binary fields, and returns + ** only the projected values converted to JSON-safe primitives. + ** + ** Field protection is layered: the declared allowlist and metadata checks here, + ** then core's field-access policy, which QueryAction applies for selected fields, + ** filter criteria and returned values (a refusal surfaces as DENIED; masked values + ** are passed on as returned). Real core memory tests cover masking, hidden + ** fields and denied filters. Do not declare a + ** sensitive field a script does not need to read. + *******************************************************************************/ +public class QqqRecordReader implements RecordReader +{ + /*************************************************************************** + ** Query as the current session and project the result. + ***************************************************************************/ + @Override + public List> read(String table, List fields, List criteria, int limit) throws ScriptingException + { + QTableMetaData metaData = QContext.getQInstance().getTable(table); + if(metaData == null) + { + throw new ScriptingException(ScriptingException.Kind.NOT_FOUND, "Unknown table"); + } + + Set queryFields = new LinkedHashSet<>(fields); + for(String field : fields) + { + requireReadable(metaData, field); + } + + QQueryFilter filter = new QQueryFilter(); + for(BrokerRequest.Criterion criterion : criteria) + { + requireReadable(metaData, criterion.field()); + queryFields.add(criterion.field()); + List values = new ArrayList<>(); + for(Object value : criterion.values()) + { + values.add((Serializable) value); + } + filter.addCriteria(new QFilterCriteria(criterion.field(), QCriteriaOperator.valueOf(criterion.operator()), values)); + } + filter.setLimit(limit); + + try + { + QueryInput input = new QueryInput().withTableName(table).withFilter(filter).withFieldNamesToInclude(queryFields); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + List> rows = new ArrayList<>(); + for(QRecord record : new QueryAction().execute(input).getRecords()) + { + Map row = new LinkedHashMap<>(); + for(String field : fields) + { + row.put(field, jsonSafe(record.getValue(field))); + } + rows.add(row); + if(rows.size() >= limit) + { + break; + } + } + return (rows); + } + catch(QPermissionDeniedException e) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Read was not permitted"); + } + catch(QException e) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Read could not be completed"); + } + } + + + + /*************************************************************************** + ** The field must exist and be an ordinary visible value. + ***************************************************************************/ + private static void requireReadable(QTableMetaData table, String fieldName) throws ScriptingException + { + QFieldMetaData field = table.getFields().get(fieldName); + if(field == null || Boolean.TRUE.equals(field.getIsHidden()) || field.getType() == QFieldType.PASSWORD || field.getType() == QFieldType.BLOB) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Field is not readable"); + } + } + + + + /*************************************************************************** + ** Only strings, numbers, booleans and null cross to a worker. + ***************************************************************************/ + public static Object jsonSafe(Serializable value) + { + if(value == null || value instanceof String || value instanceof Boolean || value instanceof Byte || value instanceof Short + || value instanceof Integer || value instanceof Long || value instanceof BigDecimal || value instanceof BigInteger) + { + return (value); + } + if((value instanceof Double d && Double.isFinite(d)) || (value instanceof Float f && Float.isFinite(f))) + { + return (value); + } + if(value instanceof Instant || value instanceof LocalDate || value instanceof LocalDateTime || value instanceof LocalTime || value instanceof OffsetDateTime) + { + return (value.toString()); + } + throw new IllegalArgumentException("Unsupported script data value"); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/capability/RecordReader.java b/base/src/main/java/com/kingsrook/qbits/scripting/capability/RecordReader.java new file mode 100644 index 0000000..f5ab70c --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/capability/RecordReader.java @@ -0,0 +1,41 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.capability; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.scripting.ScriptingException; + + +/******************************************************************************* + ** The host side of a read: returns projected, JSON-safe rows (String, Number, + ** Boolean or null values only). No QRecord, session, entity or other Java object + ** may cross this interface towards a worker. The default implementation is + ** QqqRecordReader; tests substitute a fixed reader. + *******************************************************************************/ +public interface RecordReader +{ + /*************************************************************************** + ** Read at most limit rows, as the current actor, with only the named fields. + ***************************************************************************/ + List> read(String table, List fields, List criteria, int limit) throws ScriptingException; +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/hooks/DeleteHookSnapshots.java b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/DeleteHookSnapshots.java new file mode 100644 index 0000000..088b575 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/DeleteHookSnapshots.java @@ -0,0 +1,175 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.hooks; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.json.JSONObject; + + +/******************************************************************************* + ** Bounded invocation-local pre-delete read receipts. Never persisted or exposed. + ** Failed actions can leave receipts only until consumption or QContext teardown; + ** aggregate bounds prevent an unbounded accumulation in a long-running context. + *******************************************************************************/ +final class DeleteHookSnapshots +{ + private static final String KEY = DeleteHookSnapshots.class.getName(); + private static final int MAX_PENDING = 8; + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private DeleteHookSnapshots() + { + } + + /*************************************************************************** + ** A retry or preview on the same input can never reuse an earlier receipt. + ***************************************************************************/ + static void discard(DeleteInput input) + { + State state = QContext.getObject(KEY, State.class).orElse(null); + if(state != null && state.pending != null) + { + Capture removed = state.pending.remove(input); + if(removed != null) + { + state.bytes -= removed.bytes(); + } + if(state.pending.isEmpty()) + { + QContext.removeObject(KEY); + } + } + } + + /*************************************************************************** + ** Capture only already-authorized bounded JSON; identity is exact, not an id + ** supplied by a caller. The thread-local container is not a global registry. + ***************************************************************************/ + static void save(DeleteInput input, Map snapshots) throws ScriptingException + { + if(snapshots.isEmpty()) + { + return; + } + State state = QContext.getObject(KEY, State.class).orElseGet(State::new); + int bytes = 0; + for(Map.Entry entry : snapshots.entrySet()) + { + bytes = addBytes(bytes, entry.getKey(), entry.getValue()); + } + if(state.pending == null || state.pending.size() >= MAX_PENDING || bytes > WorkerProtocol.MAX_PAYLOAD_BYTES - state.bytes) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Pending delete hook context budget exceeded"); + } + state.pending.put(input, new Capture(input.getTableName(), HookRecordAccess.transaction(input), QContext.getQInstance(), QContext.getQSession(), Thread.currentThread(), + Actor.fromSession(QContext.getQSession()).id(), Map.copyOf(snapshots), bytes)); + state.bytes += bytes; + QContext.setObject(KEY, state); + } + + /*************************************************************************** + ** Bound both the temporary capture and the aggregate pending receipt set. + ***************************************************************************/ + static int addBytes(int total, Serializable key, String json) throws ScriptingException + { + if(key == null || !Set.of(String.class, Integer.class, Long.class, Short.class, Byte.class, Boolean.class, + BigInteger.class, BigDecimal.class, Float.class, Double.class).contains(key.getClass()) + || (key instanceof Double value && !Double.isFinite(value)) || (key instanceof Float value && !Float.isFinite(value))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Delete hook requires a scalar finite primary key"); + } + int keyBytes = JSONObject.valueToString(key).getBytes(StandardCharsets.UTF_8).length; + if(keyBytes > 1024) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Delete hook primary key is too large"); + } + WorkerProtocol.checkPayload(json); + int bytes = json.getBytes(StandardCharsets.UTF_8).length + keyBytes; + if(bytes > WorkerProtocol.MAX_PAYLOAD_BYTES - total) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Delete hook context budget exceeded"); + } + return (total + bytes); + } + + /*************************************************************************** + ** Consume once before processing native successful-delete rows. No receipt + ** can transfer between inputs, principals, instances, threads, or retries. + ***************************************************************************/ + static Map take(DeleteInput input) throws ScriptingException + { + State state = QContext.getObject(KEY, State.class).orElse(null); + Capture capture = state == null || state.pending == null ? null : state.pending.get(input); + discard(input); + if(capture == null) + { + return (Map.of()); + } + if(!capture.table().equals(input.getTableName()) || capture.transaction() != HookRecordAccess.transaction(input) || capture.instance() != QContext.getQInstance() || capture.session() != QContext.getQSession() || capture.thread() != Thread.currentThread()) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Delete hook context identity changed"); + } + Actor.requireCurrentIdentity(capture.actor()); + Map records = new HashMap<>(); + for(Map.Entry entry : capture.snapshots().entrySet()) + { + QRecord record = new QRecord(); + new JSONObject(entry.getValue()).toMap().forEach((name, value) -> record.setValue(name, (Serializable) value)); + records.put(entry.getKey(), record); + } + return (records); + } + + /*************************************************************************** + ** Transient state cannot survive serialization of an asynchronous context. + ***************************************************************************/ + private static final class State implements Serializable + { + private final transient Map pending = new IdentityHashMap<>(); + private int bytes; + } + + /*************************************************************************** + ** Authority is captured at the actual native pre-delete read, not inferred + ** later from a row that no longer exists. + ***************************************************************************/ + private record Capture(String table, Object transaction, QInstance instance, QSession session, Thread thread, String actor, Map snapshots, int bytes) + { + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookEvent.java b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookEvent.java new file mode 100644 index 0000000..1e67a98 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookEvent.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.hooks; + + +/******************************************************************************* + ** Native action phases. POST means after backend action, never after commit. + *******************************************************************************/ +public enum HookEvent +{ + PRE_INSERT, + PRE_UPDATE, + PRE_DELETE, + POST_INSERT, + POST_UPDATE, + POST_DELETE, + AFTER_INSERT, + AFTER_UPDATE, + AFTER_DELETE; + + /*************************************************************************** + ** Durable observations are separate from pre-commit backend callbacks. + ***************************************************************************/ + public boolean isAfterCommit() + { + return (this == AFTER_INSERT || this == AFTER_UPDATE || this == AFTER_DELETE); + } + + /*************************************************************************** + ** Only proposed insert/update values may be patched. + ***************************************************************************/ + public boolean permitsPatch() + { + return (this == PRE_INSERT || this == PRE_UPDATE); + } + + /*************************************************************************** + ** Post callbacks observe a backend action whose transaction may roll back. + ***************************************************************************/ + public boolean isPostAction() + { + return (this == POST_INSERT || this == POST_UPDATE || this == POST_DELETE); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookRecordAccess.java b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookRecordAccess.java new file mode 100644 index 0000000..c835897 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookRecordAccess.java @@ -0,0 +1,170 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.hooks; + + +import java.io.Serializable; +import java.util.Map; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.capability.QqqRecordReader; +import com.kingsrook.qbits.scripting.observability.ScriptTiming; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessHelper; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.json.JSONObject; + + +/******************************************************************************* + ** Genuine native read authorization; never substitute a write-action input. + *******************************************************************************/ +public final class HookRecordAccess +{ + private static final ThreadLocal READING = new ThreadLocal<>(); + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private HookRecordAccess() + { + } + + /*************************************************************************** + ** Read in the owning action transaction, honoring host read customizers and + ** row/field/credential/channel policies. Heavy content is not implicit context. + ***************************************************************************/ + public static QRecord read(AbstractTableActionInput action, QRecord selected) throws QException + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.AUTHORIZED_RECORD_READ)) + { + try + { + if(Boolean.TRUE.equals(READING.get())) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Recursive hook context read is forbidden"); + } + READING.set(true); + try + { + var instance = QContext.getQInstance(); + var session = QContext.getQSession(); + String actor = Actor.fromSession(session).id(); + GetInput input = input(action); + input.setPrimaryKey(selected.getValue(action.getTable().getPrimaryKeyField())); + if(input.getPrimaryKey() == null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Hook context is not readable"); + } + QRecord record = new GetAction().execute(input).getRecord(); + if(QContext.getQInstance() != instance || QContext.getQSession() != session) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Hook context identity changed during read"); + } + Actor.requireCurrentIdentity(actor); + if(record == null || !record.getErrors().isEmpty()) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Hook context is not readable"); + } + return (record); + } + finally + { + READING.remove(); + } + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + } + + /*************************************************************************** + ** Read customizers cannot recursively invoke an isolated record hook. + ***************************************************************************/ + static boolean isReading() + { + return (Boolean.TRUE.equals(READING.get())); + } + + /*************************************************************************** + ** Use a read-action input even for a table-only recheck after deletion. + ***************************************************************************/ + static GetInput input(AbstractTableActionInput action) + { + GetInput input = new GetInput(action.getTableName()).withTransaction(transaction(action)) + .withShouldGenerateDisplayValues(false).withShouldFetchHeavyFields(false).withInputSource(action.getInputSource()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (input); + } + + /*************************************************************************** + ** Bind capture to exactly the same effective transaction as its native read. + ***************************************************************************/ + static QBackendTransaction transaction(AbstractTableActionInput action) + { + QBackendTransaction explicit = switch(action) + { + case InsertInput insert -> insert.getTransaction(); + case UpdateInput update -> update.getTransaction(); + case DeleteInput delete -> delete.getTransaction(); + default -> null; + }; + return (explicit == null ? QContext.getQBackendTransaction() : explicit); + } + + /*************************************************************************** + ** Proposed or prior record values as JSON; password and binary fields are + ** never sent, and only ordinary JSON scalars are. + ***************************************************************************/ + public static String project(QTableMetaData table, QRecord record) + { + JSONObject object = new JSONObject(); + for(Map.Entry entry : record.getValues().entrySet()) + { + QFieldMetaData field = table.getFields().get(entry.getKey()); + if(field == null || Boolean.TRUE.equals(field.getIsHidden()) || field.getType() == QFieldType.PASSWORD || field.getType() == QFieldType.BLOB + || !FieldAccessHelper.getTier(table, entry.getKey()).canReadValue()) + { + continue; + } + Object value = QqqRecordReader.jsonSafe(entry.getValue()); + object.put(entry.getKey(), value == null ? JSONObject.NULL : value); + } + return (object.toString()); + } + + + +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookResult.java b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookResult.java new file mode 100644 index 0000000..52ac876 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/HookResult.java @@ -0,0 +1,111 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.hooks; + + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + + +/******************************************************************************* + ** What a hook script decided: let the write proceed, propose field values, or + ** reject it. A patch is only a proposal; the host applies it to bound, editable, + ** non-security fields and type-checks every value (see ScriptHookCustomizer). + *******************************************************************************/ +public record HookResult(Action action, Map patch, String message) +{ + public static final int MAX_MESSAGE_CHARS = 500; + public static final int MAX_PATCH_FIELDS = 50; + + + + /******************************************************************************* + ** The three possible decisions. + *******************************************************************************/ + public enum Action + { + ALLOW, + PATCH, + REJECT + } + + + + /*************************************************************************** + ** Normalize: patch only with PATCH, message only bounded text. + ***************************************************************************/ + public HookResult + { + if(action == null) + { + throw new IllegalArgumentException("A hook result needs an action"); + } + Map copy = new LinkedHashMap<>(); + if(patch != null) + { + copy.putAll(patch); + } + if(action != Action.PATCH && !copy.isEmpty()) + { + throw new IllegalArgumentException("Only a PATCH result may carry a patch"); + } + if(action == Action.PATCH && (copy.isEmpty() || copy.size() > MAX_PATCH_FIELDS)) + { + throw new IllegalArgumentException("A PATCH result needs between 1 and " + MAX_PATCH_FIELDS + " fields"); + } + patch = Collections.unmodifiableMap(copy); + if(message != null && message.length() > MAX_MESSAGE_CHARS) + { + message = message.substring(0, MAX_MESSAGE_CHARS); + } + } + + + + /*************************************************************************** + ** Let the write proceed unchanged. + ***************************************************************************/ + public static HookResult allow() + { + return (new HookResult(Action.ALLOW, null, null)); + } + + + + /*************************************************************************** + ** Reject the write with a user-facing reason. + ***************************************************************************/ + public static HookResult reject(String message) + { + return (new HookResult(Action.REJECT, null, message)); + } + + + + /*************************************************************************** + ** Propose field values. + ***************************************************************************/ + public static HookResult patch(Map values) + { + return (new HookResult(Action.PATCH, values, null)); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptAfterHooks.java b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptAfterHooks.java new file mode 100644 index 0000000..4a75dce --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptAfterHooks.java @@ -0,0 +1,219 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.hooks; + + +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingPermissions; +import com.kingsrook.qbits.scripting.ScriptingQBitConfig; +import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerInterface; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.TransactionalRecordChangeMetaData; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.jobs.DurableProcessMetaData; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Explicit host opt-in to committed observations through the existing Jobs owner. + ** Existing synchronous customizers are unchanged; conflicting required listeners + ** are refused, never replaced or composed under an unproved transaction contract. + *******************************************************************************/ +public final class ScriptAfterHooks +{ + public static final String PROCESS = "scripting.deliverAfterHook"; + public static final String REVISION = "scripting-after-1"; + + /*************************************************************************** + ** Metadata utility. + ***************************************************************************/ + private ScriptAfterHooks() {} + + /*************************************************************************** + ** Register fixed native dispatch and selected required source event phases. + ***************************************************************************/ + public static QTableMetaData install(QInstance instance, QTableMetaData table, HookEvent... events) throws QException + { + if(instance == null || table == null || events == null || events.length == 0 || events.length > 3 + || table.getName().startsWith("scripting_") || Set.of("script", "scriptRevision", "scriptRevisionFile").contains(table.getName())) + { + throw new QException("Invalid durable scripting target"); + } + EnumSet selected = EnumSet.noneOf(RecordChangeType.class); + for(HookEvent event : events) + { + if(event == null || !event.isAfterCommit()) + { + throw new QException("Durable scripting only installs AFTER phases"); + } + selected.add(RecordChangeType.valueOf(event.name().substring(6))); + } + TransactionalRecordChangeMetaData prior = TransactionalRecordChangeMetaData.of(table); + if(prior != null) + { + if(!ourListener(prior)) + { + throw new QException("The table already has another required transactional listener"); + } + for(RecordChangeType type : RecordChangeType.values()) + { + if(prior.requires(type)) + { + selected.add(type); + } + } + } + QProcessMetaData existing = instance.getProcess(PROCESS); + if(existing != null) + { + DurableProcessMetaData durable = DurableProcessMetaData.of(existing); + if(durable == null || !REVISION.equals(durable.getTransactionalRevision()) || existing.getStepList() == null || existing.getStepList().size() != 1 + || !(existing.getStepList().getFirst() instanceof QBackendStepMetaData step) || step.getCode() == null + || !Dispatch.class.getName().equals(step.getCode().getName()) || step.getCode().getCodeType() != QCodeType.JAVA || step.getCode().getInlineCode() != null) + { + throw new QException("Durable scripting process name is already in use"); + } + } + else + { + instance.addProcess(new QProcessMetaData().withName(PROCESS).withLabel("Deliver committed script event") + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED).withCustomPermissionChecker(new QCodeReference(Permission.class))) + .withSupplementalMetaData(new DurableProcessMetaData().withEligible(true).withTransactionalRevision(REVISION)) + .withStepList(List.of(new QBackendStepMetaData().withName("deliver").withCode(new QCodeReference(Dispatch.class))))); + } + table.withSupplementalMetaData(new TransactionalRecordChangeMetaData(new QCodeReference(Listener.class), selected)); + return (table); + } + + /*************************************************************************** + ** Authoring advertises only explicitly installed durable source phases. + ***************************************************************************/ + public static boolean installed(QTableMetaData table, HookEvent event) throws QException + { + TransactionalRecordChangeMetaData metadata = TransactionalRecordChangeMetaData.of(table); + return (event.isAfterCommit() && metadata != null && ourListener(metadata) + && metadata.requires(RecordChangeType.valueOf(event.name().substring(6)))); + } + + /*************************************************************************** + ** Only this fixed Java listener defines the supported consumer contract. + ***************************************************************************/ + private static boolean ourListener(TransactionalRecordChangeMetaData metadata) + { + QCodeReference reference = metadata.getListener(); + return (reference != null && reference.getCodeType() == QCodeType.JAVA && reference.getInlineCode() == null + && Listener.class.getName().equals(reference.getName())); + } + + /*************************************************************************** + ** Named execution access is checked again by the owner service at delivery. + ***************************************************************************/ + public static class Permission extends UseOtherPermissionNameCustomPermissionChecker + { + /*********************************************************************** + ** Register the existing shared execution grant. + ***********************************************************************/ + public Permission() + { + withPermissionName(ScriptingPermissions.EXECUTE); + } + } + + /*************************************************************************** + ** Required native callback cannot be suppressed by notification flags. + ***************************************************************************/ + public static class Listener implements RecordChangeListenerInterface + { + /*********************************************************************** + ** Same-transaction enlistment failure must fail the source mutation. + ***********************************************************************/ + @Override + public boolean isRequired() + { + return (true); + } + + /*********************************************************************** + ** Scope is determined solely by the host's immutable table declaration. + ***********************************************************************/ + @Override + public boolean appliesTo(String table, RecordChangeType type) + { + try + { + return (installed(QContext.getQInstance().getTable(table), HookEvent.valueOf("AFTER_" + type.name()))); + } + catch(QException e) + { + throw new IllegalStateException("Invalid durable scripting metadata", e); + } + } + + /*********************************************************************** + ** Persist only script pins and event locators; never DML snapshot values. + ***********************************************************************/ + @Override + public void onRecordsChanged(RecordChangeEvent event) throws QException + { + var service = ScriptingQBitConfig.serviceFor(QContext.getQInstance()); + if(service == null || HookRecordAccess.isReading()) + { + throw new QException("Durable scripting is unavailable or recursively invoked"); + } + service.afterHooks().enlist(event); + } + } + + /*************************************************************************** + ** Requests carry no authority: delivery reads the active core-verified intent. + ***************************************************************************/ + public static class Dispatch implements BackendStep + { + /*********************************************************************** + ** Direct process calls and ordinary jobs cannot fabricate enlisted origin. + ***********************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + var service = ScriptingQBitConfig.serviceFor(QContext.getQInstance()); + if(service == null) + { + throw new QException("Scripting is unavailable"); + } + service.afterHooks().deliver(); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptHookCustomizer.java b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptHookCustomizer.java new file mode 100644 index 0000000..81dcd53 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptHookCustomizer.java @@ -0,0 +1,520 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.hooks; + + +import java.io.Serializable; +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingQBitConfig; +import com.kingsrook.qbits.scripting.observability.ScriptTiming; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.service.ScriptingResults.ExecutionSummary; +import com.kingsrook.qbits.scripting.service.ScriptingResults.HookBindingView; +import com.kingsrook.qbits.scripting.service.ScriptingService; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessHelper; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage; +import com.kingsrook.qqq.backend.core.model.statusmessages.QWarningMessage; + + +/******************************************************************************* + ** Pinned, bounded native record hooks. Pre actions may veto; only insert/update + ** may patch. Post actions are read-only observations and failures are warnings. + ** Execution audit is independent; it never attests that the record committed. + ** Install with ScriptHooks.install to preserve existing Java customizer order. + *******************************************************************************/ +public class ScriptHookCustomizer implements TableCustomizerInterface +{ + /*************************************************************************** + ** Run the insert hook. Preview runs are checked too: scripts are read-only. + ***************************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + return (apply(insertInput, HookEvent.PRE_INSERT, records, null, permitsInsertPatch(insertInput, isPreview))); + } + + + + /*************************************************************************** + ** Run before required-field checks so patched defaults are validated. + ***************************************************************************/ + @Override + public WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return (WhenToRun.BEFORE_ALL_VALIDATIONS); + } + + + + /*************************************************************************** + ** Run the update hook with prior values when the backend supplies them. + ***************************************************************************/ + @Override + public List preUpdate(UpdateInput updateInput, List records, boolean isPreview, Optional> oldRecordList) throws QException + { + return (apply(updateInput, HookEvent.PRE_UPDATE, records, null, permitsUpdatePatch(updateInput, isPreview))); + } + + + + /*************************************************************************** + ** Shared implementation for both events. + ***************************************************************************/ + private List apply(AbstractTableActionInput input, HookEvent event, List records, Map capturedDeletes, boolean permitsPatch) + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.HOOK_CALLBACK)) + { + try + { + if(HookRecordAccess.isReading()) + { + failAll(event, records, "Recursive hook context read is forbidden"); + return (records); + } + String tableName = input.getTableName(); + ScriptingService service = ScriptingQBitConfig.serviceFor(QContext.getQInstance()); + if(service == null) + { + failAll(event, records, "Scripting is not installed for this table hook"); + return (records); + } + + HookBindingView binding; + try + { + Optional found = service.findBinding(tableName, event); + if(found.isEmpty()) + { + return (records); + } + binding = found.get(); + } + catch(ScriptingException e) + { + failAll(event, records, "The script hook could not be looked up"); + return (records); + } + + if(records.size() > service.getMaxRecordsPerHook()) + { + refuseContext(service, binding, actorId(QContext.getQSession()), event, records); + return (records); + } + + QTableMetaData table = QContext.getQInstance().getTable(tableName); + String actorId = actorId(QContext.getQSession()); + for(QRecord record : records) + { + if(!record.getErrors().isEmpty()) + { + continue; + } + boolean checkingContext = true; + try + { + checkContextPermissions(input, event, record); + QRecord proposed = record; + QRecord prior = null; + if(event == HookEvent.PRE_UPDATE) + { + prior = HookRecordAccess.read(input, record); + } + else if(event == HookEvent.POST_DELETE) + { + proposed = capturedDeletes == null ? null : capturedDeletes.get(record.getValue(table.getPrimaryKeyField())); + if(proposed == null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Missing authorized delete snapshot"); + } + } + else if(event != HookEvent.PRE_INSERT) + { + proposed = HookRecordAccess.read(input, record); + } + var session = QContext.getQSession(); + var instance = QContext.getQInstance(); + checkingContext = false; + ExecutionSummary summary = service.runHook(binding, actorId, HookRecordAccess.project(table, proposed), prior == null ? null : HookRecordAccess.project(table, prior)); + checkingContext = true; + if(QContext.getQSession() != session || QContext.getQInstance() != instance) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Hook changed the initiating context"); + } + Actor.requireCurrentIdentity(actorId); + checkContextPermissions(input, event, record); + if(event != HookEvent.PRE_INSERT && event != HookEvent.POST_DELETE) + { + HookRecordAccess.read(input, record); + } + if(!summary.succeeded()) + { + report(event, record, "The script hook could not be completed (execution " + summary.executionId() + ")"); + continue; + } + if(summary.hook().action() == HookResult.Action.PATCH && !permitsPatch) + { + report(event, record, "This hook phase does not support patches"); + continue; + } + applyResult(table, binding, record, summary); + } + catch(QException e) + { + if(checkingContext) + { + refuseContext(service, binding, actorId, event, List.of(record)); + } + else + { + report(event, record, "The script hook or its execution audit could not be completed"); + } + } + } + return (records); + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + } + + + + /*************************************************************************** + ** A context refusal is a durable admission outcome, not a worker execution. + ***************************************************************************/ + private static void refuseContext(ScriptingService service, HookBindingView binding, String actor, HookEvent event, List records) + { + try + { + service.recordHookRefusal(binding, actor); + failAll(event, records, "The script hook context was refused"); + } + catch(QException e) + { + failAll(event, records, "The script hook context was refused and its refusal audit could not be stored"); + } + } + + + + /*************************************************************************** + ** Apply a decision to the record. + ***************************************************************************/ + private void applyResult(QTableMetaData table, HookBindingView binding, QRecord record, ExecutionSummary summary) + { + HookResult result = summary.hook(); + switch(result.action()) + { + case ALLOW: + return; + case REJECT: + String reason = result.message() == null || result.message().isBlank() ? "Rejected by a script" : result.message().replaceAll("\\p{Cntrl}", " "); + record.addError(new BadInputStatusMessage(reason)); + return; + case PATCH: + Map checked = new HashMap<>(); + for(Map.Entry entry : result.patch().entrySet()) + { + QFieldMetaData field = table.getFields().get(entry.getKey()); + if(!binding.patchableFields().contains(entry.getKey()) || field == null || !field.getIsEditable() + + || Boolean.TRUE.equals(field.getIsHidden()) || !FieldAccessHelper.getTier(table, entry.getKey()).canWrite() + || entry.getKey().equals(table.getPrimaryKeyField()) + || (table.getRecordSecurityLocks() != null && table.getRecordSecurityLocks().stream().anyMatch(lock -> entry.getKey().equals(lock.getFieldName())))) + { + record.addError(new BadInputStatusMessage("A script proposed a change to a field it may not change (execution " + summary.executionId() + ")")); + return; + } + try + { + checked.put(entry.getKey(), coerce(field, entry.getValue())); + } + catch(IllegalArgumentException e) + { + record.addError(new BadInputStatusMessage("A script proposed an invalid value (execution " + summary.executionId() + ")")); + return; + } + } + checked.forEach((name, value) -> record.setValue(name, value)); + return; + default: + record.addError(new BadInputStatusMessage("The script hook returned an unknown decision")); + return; + } + } + + + + /*************************************************************************** + ** Convert a JSON scalar to the field's type or refuse. + ***************************************************************************/ + private static Serializable coerce(QFieldMetaData field, Object value) + { + if(value == null) + { + return (null); + } + QFieldType type = field.getType(); + if((type == QFieldType.STRING || type == QFieldType.TEXT) && value instanceof String s) + { + return (s); + } + if(type == QFieldType.BOOLEAN && value instanceof Boolean b) + { + return (b); + } + if(value instanceof Number number) + { + BigDecimal decimal = new BigDecimal(number.toString()); + try + { + if(type == QFieldType.DECIMAL) + { + return (decimal); + } + if(type == QFieldType.INTEGER) + { + return (decimal.intValueExact()); + } + if(type == QFieldType.LONG) + { + return (decimal.longValueExact()); + } + } + catch(ArithmeticException e) + { + throw new IllegalArgumentException("Number is not a whole value in range", e); + } + } + throw new IllegalArgumentException("Value does not match the field type"); + } + + + + /*************************************************************************** + ** Identity of whoever is writing, for execution history. + ***************************************************************************/ + private static String actorId(QSession session) + { + try + { + return (Actor.fromSession(session).id()); + } + catch(ScriptingException e) + { + return (null); + } + } + + + + /*************************************************************************** + ** Refuse every record with the same generic reason. + ***************************************************************************/ + private static void failAll(HookEvent event, List records, String message) + { + for(QRecord record : records) + { + report(event, record, message); + } + } + + /*************************************************************************** + ** Preserve phase semantics: late insert phases cannot introduce new values. + ***************************************************************************/ + public boolean permitsInsertPatch(InsertInput input, boolean preview) + { + WhenToRun phase = whenToRunPreInsert(input, preview); + return (phase == WhenToRun.BEFORE_ALL_VALIDATIONS); + } + + /*************************************************************************** + ** Delete hooks can veto but never rewrite the selected identity. + ***************************************************************************/ + @Override + public List preDelete(DeleteInput input, List records, boolean preview) throws QException + { + DeleteHookSnapshots.discard(input); + HookBindingView postBinding = null; + ScriptingService service = ScriptingQBitConfig.serviceFor(QContext.getQInstance()); + try + { + postBinding = preview || service == null ? null : service.findBinding(input.getTableName(), HookEvent.POST_DELETE).orElse(null); + if(postBinding != null) + { + if(records.size() > service.getMaxRecordsPerHook()) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Too many delete snapshots"); + } + Map snapshots = new HashMap<>(); + int bytes = 0; + for(QRecord record : records) + { + if(record.getErrors().isEmpty()) + { + String json = HookRecordAccess.project(input.getTable(), HookRecordAccess.read(input, record)); + bytes = DeleteHookSnapshots.addBytes(bytes, record.getValue(input.getTable().getPrimaryKeyField()), json); + snapshots.put(record.getValue(input.getTable().getPrimaryKeyField()), json); + } + } + DeleteHookSnapshots.save(input, snapshots); + } + } + catch(QException e) + { + // POST_DELETE is observational. Missing context is refused and audited + // only after a successful native deletion; an actual PRE_DELETE binding + // still runs its own checks below and can veto independently. + DeleteHookSnapshots.discard(input); + } + List result = apply(input, HookEvent.PRE_DELETE, records, null, false); + if(result.stream().allMatch(record -> !record.getErrors().isEmpty())) + { + DeleteHookSnapshots.discard(input); + } + return (result); + } + + /*************************************************************************** + ** Observe backend insert; this does not attest transaction commit. + ***************************************************************************/ + @Override + public List postInsert(InsertInput input, List records) throws QException + { + return (apply(input, HookEvent.POST_INSERT, records, null, false)); + } + + /*************************************************************************** + ** Observe the currently readable backend row; unchecked prior values are omitted. + ***************************************************************************/ + @Override + public List postUpdate(UpdateInput input, List records, Optional> prior) throws QException + { + return (apply(input, HookEvent.POST_UPDATE, records, null, false)); + } + + /*************************************************************************** + ** Observe backend deletion; the supplied values are the deleted snapshot. + ***************************************************************************/ + @Override + public List postDelete(DeleteInput input, List records) throws QException + { + try + { + return (apply(input, HookEvent.POST_DELETE, records, DeleteHookSnapshots.take(input), false)); + } + catch(QException e) + { + return (apply(input, HookEvent.POST_DELETE, records, Map.of(), false)); + } + } + + /*************************************************************************** + ** Recheck current permissions even inside a native authorized action scope. + ***************************************************************************/ + private static TablePermissionSubType permission(HookEvent event) + { + return (switch(event) + { + case PRE_INSERT, POST_INSERT, AFTER_INSERT -> TablePermissionSubType.INSERT; + case PRE_UPDATE, POST_UPDATE, AFTER_UPDATE -> TablePermissionSubType.EDIT; + case PRE_DELETE, POST_DELETE, AFTER_DELETE -> TablePermissionSubType.DELETE; + }); + } + + /*************************************************************************** + ** Post failures are visible warnings; they cannot veto a finished backend. + ***************************************************************************/ + private static void report(HookEvent event, QRecord record, String message) + { + if(event.isPostAction()) + { + record.addWarning(new QWarningMessage(message + "; the backend action may already have occurred")); + } + else + { + record.addError(new BadInputStatusMessage(message)); + } + } + + /*************************************************************************** + ** Use core's authorized early phase so its complete validation follows patches. + ***************************************************************************/ + @Override + public PreUpdatePhase whenToRunPreUpdate(UpdateInput input, boolean preview) + { + return (PreUpdatePhase.BEFORE_FIELD_VALIDATIONS); + } + + /*************************************************************************** + ** Hosts retaining the late update phase support veto only. + ***************************************************************************/ + public boolean permitsUpdatePatch(UpdateInput input, boolean preview) + { + return (whenToRunPreUpdate(input, preview) == PreUpdatePhase.BEFORE_FIELD_VALIDATIONS); + } + + /*************************************************************************** + ** Prior/deleted/post snapshots require READ as well as current action access. + ** Standalone earliest insert sees proposed caller data only. + ***************************************************************************/ + protected boolean requiresReadContext(HookEvent event) + { + return (event != HookEvent.PRE_INSERT); + } + + /*************************************************************************** + ** Honor native custom permission checkers, credentials and client channels. + ***************************************************************************/ + private void checkContextPermissions(AbstractTableActionInput input, HookEvent event, QRecord record) throws QException + { + PermissionsHelper.checkTablePermissionThrowing(input, permission(event)); + if(requiresReadContext(event) || (input instanceof InsertInput insert && whenToRunPreInsert(insert, false) != WhenToRun.BEFORE_ALL_VALIDATIONS)) + { + var read = HookRecordAccess.input(input); + read.setPrimaryKey(record.getValue(input.getTable().getPrimaryKeyField())); + PermissionsHelper.checkTablePermissionThrowing(read, TablePermissionSubType.READ); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptHooks.java b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptHooks.java new file mode 100644 index 0000000..b037bd3 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/hooks/ScriptHooks.java @@ -0,0 +1,206 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.hooks; + + +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.InitializableViaCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceWithProperties; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Opt-in metadata composition, called during host instance construction before + ** validation. Host Java customizer runs once first; script sees its output. + ** Existing insert phase is preserved, so late-phase hooks cannot patch values. + *******************************************************************************/ +public final class ScriptHooks +{ + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private ScriptHooks() + { + } + + /*************************************************************************** + ** Idempotently install selected native phases without replacing host logic. + ** Non-Java customizers are refused: this helper never enables legacy script. + ***************************************************************************/ + public static QTableMetaData install(QTableMetaData table, HookEvent... events) + { + Map customizers = table.getCustomizers() == null ? new HashMap<>() : new HashMap<>(table.getCustomizers()); + EnumSet selected = EnumSet.noneOf(HookEvent.class); + selected.addAll(List.of(events)); + if(selected.contains(HookEvent.POST_DELETE)) + { + selected.add(HookEvent.PRE_DELETE); + } + for(HookEvent event : selected) + { + if(event.isAfterCommit()) + { + throw new IllegalArgumentException("Use ScriptAfterHooks.install for durable AFTER phases"); + } + TableCustomizers role = TableCustomizers.valueOf(event.name() + "_RECORD"); + QCodeReference existing = customizers.get(role.getRole()); + if(existing != null && Composed.class.getName().equals(existing.getName())) + { + continue; + } + if(existing != null && (existing.getCodeType() != QCodeType.JAVA || existing.getInlineCode() != null)) + { + throw new IllegalArgumentException("Record script hooks require existing Java customizers"); + } + if(existing != null && ScriptHookCustomizer.class.getName().equals(existing.getName())) + { + existing = null; + } + customizers.put(role.getRole(), existing == null ? new QCodeReference(ScriptHookCustomizer.class) + : new QCodeReferenceWithProperties(Composed.class, Map.of("host", existing))); + } + table.setCustomizers(customizers); + return (table); + } + + /*************************************************************************** + ** One host callback followed by one isolated callback for its assigned role. + ***************************************************************************/ + public static class Composed extends ScriptHookCustomizer implements InitializableViaCodeReference + { + private TableCustomizerInterface host; + + /*********************************************************************** + ** Resolve only host-controlled Java metadata; invalid setup fails closed. + ***********************************************************************/ + @Override + public void initialize(QCodeReference reference) + { + if(!(reference instanceof QCodeReferenceWithProperties properties) + || !(properties.getProperties().get("host") instanceof QCodeReference code) + || code.getCodeType() != QCodeType.JAVA || code.getInlineCode() != null) + { + throw new IllegalArgumentException("Missing Java host customizer"); + } + host = QCodeLoader.getAdHoc(TableCustomizerInterface.class, code); + if(host == null) + { + throw new IllegalArgumentException("Host customizer could not be resolved"); + } + } + + /*********************************************************************** + ** Never silently move an existing host callback to a different phase. + ***********************************************************************/ + @Override + public WhenToRun whenToRunPreInsert(InsertInput input, boolean preview) + { + return (host.whenToRunPreInsert(input, preview)); + } + + /*********************************************************************** + ** Host callbacks can derive values, so even insert context requires READ. + ***********************************************************************/ + @Override + protected boolean requiresReadContext(HookEvent event) + { + return (true); + } + + /*********************************************************************** + ** Preserve the host update phase; a late host supports script veto only. + ***********************************************************************/ + @Override + public PreUpdatePhase whenToRunPreUpdate(UpdateInput input, boolean preview) + { + return (host.whenToRunPreUpdate(input, preview)); + } + + /*********************************************************************** + ** Preserve host order and pass its final records to the isolated hook. + ***********************************************************************/ + @Override + public List preInsert(InsertInput input, List records, boolean preview) throws QException + { + return (super.preInsert(input, host.preInsert(input, records, preview), preview)); + } + + /*********************************************************************** + ** Preserve host order and pass its final records to the isolated hook. + ***********************************************************************/ + @Override + public List preUpdate(UpdateInput input, List records, boolean preview, Optional> prior) throws QException + { + return (super.preUpdate(input, host.preUpdate(input, records, preview, prior), preview, prior)); + } + + /*********************************************************************** + ** Preserve host order and pass its final records to the isolated hook. + ***********************************************************************/ + @Override + public List preDelete(DeleteInput input, List records, boolean preview) throws QException + { + return (super.preDelete(input, host.preDelete(input, records, preview), preview)); + } + + /*********************************************************************** + ** Preserve host order and pass its final records to the isolated hook. + ***********************************************************************/ + @Override + public List postInsert(InsertInput input, List records) throws QException + { + return (super.postInsert(input, host.postInsert(input, records))); + } + + /*********************************************************************** + ** Preserve host order and pass its final records to the isolated hook. + ***********************************************************************/ + @Override + public List postUpdate(UpdateInput input, List records, Optional> prior) throws QException + { + return (super.postUpdate(input, host.postUpdate(input, records, prior), prior)); + } + + /*********************************************************************** + ** Preserve host order and pass its final records to the isolated hook. + ***********************************************************************/ + @Override + public List postDelete(DeleteInput input, List records) throws QException + { + return (super.postDelete(input, host.postDelete(input, records))); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/observability/ScriptTiming.java b/base/src/main/java/com/kingsrook/qbits/scripting/observability/ScriptTiming.java new file mode 100644 index 0000000..3849bbf --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/observability/ScriptTiming.java @@ -0,0 +1,188 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.kingsrook.qbits.scripting.observability; + +import jdk.jfr.Category; +import jdk.jfr.Enabled; +import jdk.jfr.Event; +import jdk.jfr.EventType; +import jdk.jfr.Label; +import jdk.jfr.Name; +import jdk.jfr.StackTrace; + +/******************************************************************************* + ** Internal optional JFR bridge. Only bounded labels enter events; never caller + ** data. Missing JFR or telemetry failures must not change business decisions. + *******************************************************************************/ +public final class ScriptTiming +{ + public static final String EVENT_NAME = "com.kingsrook.qqq.ScriptingPhase"; + private static final boolean AVAILABLE = available(); + + /*************************************************************************** + ** Fixed measurement boundaries, with no identifiers or user-supplied labels. + ***************************************************************************/ + public enum Phase + { + HOOK_CALLBACK, BINDING_LOOKUP, AUTHORIZATION_PUBLICATION, AUTHORIZED_RECORD_READ, + EXECUTION_TOTAL, ARTIFACT_VERIFICATION, WORKER_DISPATCH, BROKER_CALLBACK, AUDIT_PERSISTENCE + } + + /*************************************************************************** + ** An allocation-free shared disabled span or an enabled recording event. + ***************************************************************************/ + public interface Span extends AutoCloseable + { + /*********************************************************************** + ** Mark propagation of a throwable; never retain the throwable itself. + ***********************************************************************/ + void failed(); + + /*********************************************************************** + ** Finish timing without throwing or changing the primary result. + ***********************************************************************/ + @Override + void close(); + } + + /*************************************************************************** + ** Stateless disabled span, shared across threads and calls. + ***************************************************************************/ + private enum Noop implements Span + { + INSTANCE; + + /*********************************************************************** + ** Nothing is recorded while disabled. + ***********************************************************************/ + public void failed() {} + + /*********************************************************************** + ** Nothing is allocated or committed while disabled. + ***********************************************************************/ + public void close() {} + } + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private ScriptTiming() {} + + /*************************************************************************** + ** Probe once, so a custom runtime without jdk.jfr does not allocate repeated + ** linkage failures. Keep JFR types behind the nested recorder boundary. + ***************************************************************************/ + private static boolean available() + { + try + { + return (Recorder.TYPE != null); + } + catch(Throwable ignored) + { + return (false); + } + } + + /*************************************************************************** + ** Disabled calls perform no clock read, per-call allocation, or I/O. + ***************************************************************************/ + public static Span start(Phase phase) + { + if(AVAILABLE) + { + try + { + if(Recorder.TYPE.isEnabled()) + { + PhaseEvent event = new PhaseEvent(); + event.phase = phase.name(); + event.begin(); + return (event); + } + } + catch(Throwable ignored) + { + // Telemetry is never an authority, audit, or business failure. + } + } + return (Noop.INSTANCE); + } + + /*************************************************************************** + ** Only this lazily linked holder references JFR registration machinery. + ***************************************************************************/ + private static final class Recorder + { + private static final EventType TYPE = EventType.getEventType(PhaseEvent.class); + + /*********************************************************************** + ** Utility holder. + ***********************************************************************/ + private Recorder() {} + } + + /*************************************************************************** + ** RETURNED means the boundary returned normally, not business success or + ** committed source mutation. JFR supplies duration/thread metadata only. + ***************************************************************************/ + @Name(EVENT_NAME) + @Label("QQQ scripting phase") + @Category({"QQQ", "Scripting"}) + @Enabled(false) + @StackTrace(false) + public static final class PhaseEvent extends Event implements Span + { + private String phase; + private String outcome = "RETURNED"; + private transient boolean closed; + + /*********************************************************************** + ** Record only a fixed outcome, never exception class or message. + ***********************************************************************/ + public void failed() + { + outcome = "THREW"; + } + + /*********************************************************************** + ** Commit at most once and never mask an application's original failure. + ***********************************************************************/ + public void close() + { + if(!closed) + { + closed = true; + try + { + end(); + commit(); + } + catch(Throwable ignored) + { + // Optional observation must not change required persistence. + } + } + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/Actor.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/Actor.java new file mode 100644 index 0000000..9732a08 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/Actor.java @@ -0,0 +1,124 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.util.Set; +import java.util.function.Predicate; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; + + +/******************************************************************************* + ** Who is performing a lifecycle action, and what they may do. Every authoring + ** operation takes an Actor so identity and permission are explicit arguments, + ** never ambient. A lifecycle action by the system session or by a session with + ** no user identity is refused: drafts, checks, tests, publications and bindings + ** must be attributable to a person. + *******************************************************************************/ +public final class Actor +{ + private final String id; + private final Predicate permissions; + + + + /*************************************************************************** + ** Private; use fromSession or of. + ***************************************************************************/ + private Actor(String id, Predicate permissions) + { + this.id = id; + this.permissions = permissions; + } + + + + /*************************************************************************** + ** The actor behind a QQQ session. System and anonymous sessions are denied. + ***************************************************************************/ + public static Actor fromSession(QSession session) throws ScriptingException + { + if(session == null || session instanceof QSystemUserSession || session.getUser() == null + || session.getUser().getIdReference() == null || session.getUser().getIdReference().isBlank()) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "An identified user is required for this action"); + } + return (new Actor(session.getUser().getIdReference(), session::hasPermission)); + } + + + + /*************************************************************************** + ** An actor with a fixed upper-bound permission set. The host must also + ** establish a matching authenticated QQQ session; require checks both. + ***************************************************************************/ + public static Actor of(String id, Set grantedPermissions) + { + if(id == null || id.isBlank()) + { + throw new IllegalArgumentException("An actor needs an identity"); + } + Set copy = Set.copyOf(grantedPermissions); + return (new Actor(id, copy::contains)); + } + + + + /*************************************************************************** + ** The identity recorded on every row this actor creates. + ***************************************************************************/ + public String id() + { + return (id); + } + + + + /*************************************************************************** + ** Refuse unless this actor holds exactly the named permission. + ***************************************************************************/ + public void require(String permission) throws ScriptingException + { + requireCurrentIdentity(id); + if(!permissions.test(permission) || !QContext.getQSession().hasPermission(permission)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Permission " + permission + " is required"); + } + } + + + + /*************************************************************************** + ** Bind an explicit actor to the current authenticated QQQ session. Host + ** integrations must install that session; an Actor is not a bearer token. + ***************************************************************************/ + public static void requireCurrentIdentity(String expectedId) throws ScriptingException + { + Actor current = fromSession(QContext.getQSession()); + if(!current.id().equals(expectedId)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "The actor does not match the active session"); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/AfterHookService.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/AfterHookService.java new file mode 100644 index 0000000..4a1473d --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/AfterHookService.java @@ -0,0 +1,343 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingPermissions; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.HookRecordAccess; +import com.kingsrook.qbits.scripting.hooks.HookResult; +import com.kingsrook.qbits.scripting.hooks.ScriptAfterHooks; +import com.kingsrook.qbits.scripting.service.ScriptingResults.HookBindingView; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessHelper; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.TransactionalRecordChangeScope; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.jobs.DurableJobs; +import com.kingsrook.qqq.backend.core.jobs.FixedProcessRequest; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.BoundedIntentData; +import org.json.JSONObject; + + +/******************************************************************************* + ** Committed event locators and immutable script pins use the core Jobs owner. + ** No source snapshot, queue, private job registry or transaction is created here. + *******************************************************************************/ +public final class AfterHookService +{ + private static final Set KEYS = Set.of("eventId", "ordinal", "bindingId", "publicationId", "pin", "recordId"); + private final ScriptingService service; + private final ScriptingStore store; + private final ScriptExecutor executor; + private final ScriptingRuntime runtime; + + /*************************************************************************** + ** The existing instance service owns all source and audit collaborators. + ***************************************************************************/ + AfterHookService(ScriptingService service, ScriptingStore store, ScriptExecutor executor, ScriptingRuntime runtime) + { + this.service = service; + this.store = store; + this.executor = executor; + this.runtime = runtime; + } + + /*************************************************************************** + ** Immutable resolved execution identity; the digest is equality, not authority. + ***************************************************************************/ + private record Pin(HookBindingView binding, Integer buildId, String fingerprint) {} + + /*************************************************************************** + ** Required listener invocation alone may enlist on the exact source commit. + ** A fresh invocation ID distinguishes repeated legitimate identical writes. + ***************************************************************************/ + public void enlist(RecordChangeEvent event) throws QException + { + var admitted = TransactionalRecordChangeScope.admittedEvent(event); + if(ScriptExecutor.isExecuting()) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Recursive durable script execution is forbidden"); + } + HookEvent phase = HookEvent.valueOf("AFTER_" + event.getType().name()); + HookBindingView binding = service.findBinding(event.getTableName(), phase).orElse(null); + if(binding == null) + { + return; + } + if(event.getRecords() == null || event.getRecords().size() > service.getMaxRecordsPerHook()) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Durable hook record budget exceeded"); + } + Actor actor = Actor.fromSession(QContext.getQSession()); + var instance = QContext.getQInstance(); + var session = QContext.getQSession(); + Pin pin = resolve(binding, actor); + String invocation = admitted.eventId(); + int ordinal = 0; + for(QRecord record : event.getRecords()) + { + Serializable key = key(record == null ? null : record.getValue(instance.getTable(event.getTableName()).getPrimaryKeyField())); + authorize(binding, key); + if(phase != HookEvent.AFTER_DELETE) + { + HookRecordAccess.read(action(binding.tableName(), phase, event.getTransaction()), record); + } + Map parameters = Map.of("eventId", invocation, "ordinal", ordinal, "bindingId", binding.id(), + "publicationId", binding.publicationId(), "pin", pin.fingerprint(), "recordId", key); + DurableJobs.enlistFixedProcess(event, new FixedProcessRequest(ScriptAfterHooks.PROCESS, invocation + ":" + binding.id() + ":" + ordinal, parameters)); + ordinal++; + if(instance != QContext.getQInstance() || session != QContext.getQSession() || !pin.equals(resolve(binding, actor))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Durable hook authority changed"); + } + } + } + + /*************************************************************************** + ** Deliver only the core-verified persisted intent under its current actor. + ** INSERT/UPDATE read current state; DELETE conveys no deleted record values. + ***************************************************************************/ + public void deliver() throws QException + { + var verified = DurableJobs.requireEnlistedExecution(ScriptAfterHooks.PROCESS, ScriptAfterHooks.REVISION); + Map values = verified.parameters(); + if(!values.keySet().equals(KEYS) || !(values.get("eventId") instanceof String eventId) + || !eventId.equals(verified.eventId()) || !eventId.matches("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") + || !(values.get("pin") instanceof String fingerprint) || !fingerprint.matches("[0-9a-f]{64}")) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Invalid committed hook intent"); + } + Integer bindingId = positiveInteger(values.get("bindingId")); + Integer publicationId = positiveInteger(values.get("publicationId")); + int ordinal = nonnegativeInteger(values.get("ordinal")); + if(ordinal >= service.getMaxRecordsPerHook()) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Committed hook ordinal exceeds its budget"); + } + Serializable key = key(values.get("recordId")); + HookEvent phase = HookEvent.valueOf("AFTER_" + verified.eventType()); + Actor actor = Actor.fromSession(QContext.getQSession()); + Actor.requireCurrentIdentity(verified.actorId()); + HookBindingView binding = service.findBinding(verified.sourceTable(), phase).orElseThrow(() -> new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook binding is unavailable")); + if(!binding.id().equals(bindingId) || !binding.publicationId().equals(publicationId)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook binding has changed"); + } + Pin pin = resolve(binding, actor); + if(!fingerprint.equals(pin.fingerprint())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Committed script pin has changed"); + } + var instance = QContext.getQInstance(); + var session = QContext.getQSession(); + JSONObject payload; + try + { + authorize(binding, key); + payload = new JSONObject().put("id", eventId + ":" + ordinal).put("tableName", binding.tableName()).put("recordId", key) + .put("dataState", phase == HookEvent.AFTER_DELETE ? "DELETED_METADATA" : "CURRENT"); + if(phase != HookEvent.AFTER_DELETE) + { + var table = instance.getTable(binding.tableName()); + QRecord current = HookRecordAccess.read(action(binding.tableName(), phase, null), new QRecord().withValue(table.getPrimaryKeyField(), key)); + payload.put("record", new JSONObject(HookRecordAccess.project(table, current))); + } + } + catch(QException refused) + { + service.recordHookRefusal(binding, actor.id()); + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook context was refused"); + } + var summary = executor.execute(new ScriptExecutor.Request("DURABLE_HOOK", actor.id(), pin.buildId(), binding.id(), publicationId, + phase.name(), payload.toString(), null, binding.grants(), eventId + ":" + ordinal, () -> + { + try + { + if(instance != QContext.getQInstance() || session != QContext.getQSession() || !pin.equals(resolve(binding, actor)) + || !verified.equals(DurableJobs.requireEnlistedExecution(ScriptAfterHooks.PROCESS, ScriptAfterHooks.REVISION))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook authority changed before execution"); + } + } + catch(QException refused) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook authority was refused before execution"); + } + })); + if(instance != QContext.getQInstance() || session != QContext.getQSession() + || !verified.equals(DurableJobs.requireEnlistedExecution(ScriptAfterHooks.PROCESS, ScriptAfterHooks.REVISION)) || !pin.equals(resolve(binding, actor))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook authority changed during delivery"); + } + authorize(binding, key); + if(phase != HookEvent.AFTER_DELETE) + { + HookRecordAccess.read(action(binding.tableName(), phase, null), new QRecord().withValue(instance.getTable(binding.tableName()).getPrimaryKeyField(), key)); + } + if(!verified.equals(DurableJobs.requireEnlistedExecution(ScriptAfterHooks.PROCESS, ScriptAfterHooks.REVISION))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook authority changed after delivery"); + } + if(!summary.succeeded() || summary.hook() == null || summary.hook().action() != HookResult.Action.ALLOW) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Committed hook execution was refused"); + } + } + + /*************************************************************************** + ** Reauthorize project and artifact; later publications never retarget a pin. + ***************************************************************************/ + private Pin resolve(HookBindingView binding, Actor actor) throws QException + { + actor.require(ScriptingPermissions.EXECUTE); + ScriptingHookAuthoring.authorizeBinding(binding.tableName(), binding.event(), Set.of()); + HookBindingView active = service.findBinding(binding.tableName(), binding.event()).orElseThrow(() -> new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook binding is disabled")); + if(!active.equals(binding) || !binding.event().isAfterCommit() || !binding.patchableFields().isEmpty()) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed hook binding has changed"); + } + QRecord publication = store.require(ScriptingTables.PUBLICATION, binding.publicationId(), "Publication"); + service.authoring().requireIsolated(actor, publication.getValueInteger("script_id")); + QRecord build = store.require(ScriptingTables.BUILD, publication.getValueInteger("build_id"), "Build"); + if(!Objects.equals(build.getValueInteger("script_id"), publication.getValueInteger("script_id")) + || !CapabilityDeclaration.allWithin(binding.grants(), service.declaredCapabilities(build))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Committed hook capability pin is invalid"); + } + if(runtime.getCoordinator() == null) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Scripting coordinator is unavailable"); + } + QRecord artifact = executor.loadVerifiedArtifact(runtime.getCoordinator(), publication.getValueInteger("build_id")); + if(!Objects.equals(artifact.getValueInteger("id"), publication.getValueInteger("artifact_id"))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Committed artifact pin is invalid"); + } + String digest = BoundedIntentData.fingerprint(Map.ofEntries(Map.entry("binding", binding.id()), Map.entry("publication", binding.publicationId()), + Map.entry("project", publication.getValueInteger("script_id")), Map.entry("build", publication.getValueInteger("build_id")), + Map.entry("artifact", artifact.getValueInteger("id")), Map.entry("source", artifact.getValueString("source_sha256")), + Map.entry("bundle", artifact.getValueString("bundle_sha256")), Map.entry("grants", CapabilityDeclaration.listToJson(binding.grants())), + Map.entry("table", binding.tableName()), Map.entry("event", binding.event().name()))); + return (new Pin(binding, publication.getValueInteger("build_id"), digest)); + } + + /*************************************************************************** + ** Locators require a visible scalar key; metadata is never a hidden-field path. + ***************************************************************************/ + private static void authorize(HookBindingView binding, Serializable key) throws QException + { + ScriptingHookAuthoring.authorizeBinding(binding.tableName(), binding.event(), Set.of()); + requireLocatorAccess(binding.tableName(), key); + GetInput read = new GetInput(binding.tableName()).withPrimaryKey(key).withInputSource(QInputSource.USER).withShouldGenerateDisplayValues(false).withShouldFetchHeavyFields(false); + read.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + PermissionsHelper.checkTablePermissionThrowing(read, TablePermissionSubType.READ); + } + + /*************************************************************************** + ** Internal locators and dry-run metadata never expose a hidden key field. + ***************************************************************************/ + static void requireLocatorAccess(String tableName, Serializable value) throws ScriptingException + { + key(value); + var table = QContext.getQInstance().getTable(tableName); + var primary = table == null ? null : table.getField(table.getPrimaryKeyField()); + if(primary == null || Boolean.TRUE.equals(primary.getIsHidden()) || primary.getType() == QFieldType.PASSWORD || primary.getType() == QFieldType.BLOB + || !FieldAccessHelper.getTier(table, primary.getName()).canReadValue()) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed event locator is inaccessible"); + } + } + + /*************************************************************************** + ** Native read helper receives the exact source transaction during admission. + ***************************************************************************/ + private static AbstractTableActionInput action(String table, HookEvent event, com.kingsrook.qqq.backend.core.actions.QBackendTransaction transaction) + { + AbstractTableActionInput input = switch(event) + { + case AFTER_INSERT -> new InsertInput(table).withTransaction(transaction); + case AFTER_UPDATE -> new UpdateInput(table).withTransaction(transaction); + case AFTER_DELETE -> new DeleteInput(table).withTransaction(transaction); + default -> throw new IllegalArgumentException("Not a committed event"); + }; + input.setInputSource(QInputSource.USER); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (input); + } + + /*************************************************************************** + ** No arbitrary Java values or excessive locator text enter Jobs parameters. + ***************************************************************************/ + private static Serializable key(Serializable value) throws ScriptingException + { + if(!(value instanceof String || value instanceof Integer || value instanceof Long || value instanceof Boolean) + || value instanceof String text && (text.isBlank() || !StandardCharsets.UTF_8.newEncoder().canEncode(text) || text.getBytes(StandardCharsets.UTF_8).length > 512 || text.codePoints().anyMatch(Character::isISOControl))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Committed event requires a bounded scalar locator"); + } + return (value); + } + + /*************************************************************************** + ** Persistent native identifiers must be exact positive integers. + ***************************************************************************/ + private static Integer positiveInteger(Serializable value) throws ScriptingException + { + int result = nonnegativeInteger(value); + if(result == 0) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Invalid committed identity"); + } + return (result); + } + + /*************************************************************************** + ** Reject truncating numeric coercions in persisted dispatch parameters. + ***************************************************************************/ + private static int nonnegativeInteger(Serializable value) throws ScriptingException + { + if(!(value instanceof Integer || value instanceof Long) || ((Number) value).longValue() < 0 || ((Number) value).longValue() > Integer.MAX_VALUE) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Invalid committed ordinal"); + } + return (((Number) value).intValue()); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/AuthoringPreparation.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/AuthoringPreparation.java new file mode 100644 index 0000000..7288ac5 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/AuthoringPreparation.java @@ -0,0 +1,236 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.util.List; +import java.util.Objects; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingPermissions; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.worker.WarmWorkerPreparation; +import com.kingsrook.qbits.scripting.worker.WorkerProvider; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; + + +/******************************************************************************* + ** Authoring-only cold boundary. No record hook or execution path calls prepare. + ** Revalidation is reusable inside the subsequent owner transaction or immediately + ** before dispatch. Warm copies are disposable capacity, never a new registry. + *******************************************************************************/ +final class AuthoringPreparation +{ + private static final ThreadLocal PREPARING = new ThreadLocal<>(); + + /*************************************************************************** + ** No instances; all state belongs to one explicit authoring operation. + ***************************************************************************/ + private AuthoringPreparation() + { + } + + /*************************************************************************** + ** No-op compatibility for providers without a cache; not a readiness claim. + ** A cache provider is called only outside a transaction and worker callback. + ***************************************************************************/ + static Prepared prepare(ScriptingRuntime runtime, ScriptingStore store, ScriptSourceRegistry registry, + ScriptExecutor executor, Actor actor, String permission, Integer buildId, Integer publicationId, + ScriptExecutor.ExecutionAuthority authorization) throws ScriptingException + { + WorkerProvider worker = runtime.getWorker(); + if(!(worker instanceof WarmWorkerPreparation warmer)) + { + return (new Prepared(authorization, () -> {})); + } + if(ScriptExecutor.isExecuting() || Boolean.TRUE.equals(PREPARING.get()) || QContext.getQBackendTransaction() != null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Cold preparation requires a top-level authoring operation"); + } + QInstance instance = QContext.getQInstance(); + QSession session = QContext.getQSession(); + actor.require(permission); + requireStorage(permission); + authorization.check(); + executor.requireQualifiedWorker(); + Pin pin = load(runtime, store, registry, executor, buildId, publicationId); + ScriptExecutor.ExecutionAuthority check = () -> + { + requireContext(runtime, worker, instance, session); + actor.require(permission); + requireStorage(permission); + authorization.check(); + executor.requireQualifiedWorker(); + if(!pin.equals(load(runtime, store, registry, executor, buildId, publicationId))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "The prepared publication or artifact changed"); + } + requireContext(runtime, worker, instance, session); + actor.require(permission); + requireStorage(permission); + }; + check.check(); + WarmWorkerPreparation.UseLease lease = null; + try + { + boolean changedTransaction; + PREPARING.set(true); + try + { + lease = Objects.requireNonNull(warmer.prepareForUse(pin.artifact())); + } + finally + { + PREPARING.remove(); + changedTransaction = QContext.getQBackendTransaction() != null; + if(changedTransaction) + { + // Restore absent ambient context without committing or closing a + // transaction owned by an external collaborator. + QContext.clearTransaction(); + } + } + if(changedTransaction) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Preparation changed the initiating transaction"); + } + check.check(); + return (new Prepared(check, lease)); + } + catch(ScriptingException | RuntimeException | Error failure) + { + if(lease != null) + { + try + { + lease.close(); + } + catch(RuntimeException | Error closeFailure) + { + if(closeFailure != failure) + { + failure.addSuppressed(closeFailure); + } + } + } + throw failure; + } + } + + /*************************************************************************** + ** A single synchronous authoring scope owns both revalidation and cache use. + ** Try-with-resources closes it on successful dispatch/commit or any failure. + ***************************************************************************/ + record Prepared(ScriptExecutor.ExecutionAuthority authority, WarmWorkerPreparation.UseLease lease) + implements ScriptExecutor.ExecutionAuthority, AutoCloseable + { + /*************************************************************************** + ** Revalidate the same original authority immediately before dispatch/commit. + ***************************************************************************/ + @Override + public void check() throws ScriptingException + { + authority.check(); + } + + /*************************************************************************** + + ** No I/O or guest execution belongs in local cache-use release. + + ***************************************************************************/ + @Override + public void close() + { + lease.close(); + } + } + + /*************************************************************************** + ** Apply the existing owner-storage credential/channel ceiling before cold + ** transfer, using the same table set as the subsequent authoring unit. + ***************************************************************************/ + private static void requireStorage(String permission) throws ScriptingException + { + List tables = switch(permission) + { + case ScriptingPermissions.TEST -> List.of(ScriptingTables.EXECUTION); + case ScriptingPermissions.PUBLISH -> List.of(ScriptingTables.BUILD, ScriptingTables.ARTIFACT, ScriptingTables.PUBLICATION, + ScriptingTables.EXECUTION, "script", "scriptRevision", "scriptRevisionFile"); + case ScriptingPermissions.BIND -> List.of(ScriptingTables.BINDING, ScriptingTables.PUBLICATION, ScriptingTables.BUILD, ScriptingTables.EXECUTION); + default -> throw new ScriptingException(ScriptingException.Kind.DENIED, "Cold preparation is not an authoring action"); + }; + for(String table : tables) + { + ScriptingStorageAccess.require(table, true); + } + } + + /*************************************************************************** + ** Preserve the exact original host context and worker across external calls. + ***************************************************************************/ + private static void requireContext(ScriptingRuntime runtime, WorkerProvider worker, QInstance instance, QSession session) throws ScriptingException + { + if(QContext.getQInstance() != instance || QContext.getQSession() != session || runtime.getWorker() != worker) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Preparation changed the initiating context"); + } + } + + /*************************************************************************** + ** Re-read native visibility and signed immutable associations, never a client + ** artifact descriptor. Publication IDs must bind exactly this build/artifact. + ***************************************************************************/ + private static Pin load(ScriptingRuntime runtime, ScriptingStore store, ScriptSourceRegistry registry, + ScriptExecutor executor, Integer buildId, Integer publicationId) throws ScriptingException + { + QRecord build = store.require(ScriptingTables.BUILD, buildId, "Build"); + registry.requireProjectAccess(build.getValueInteger("script_id")); + if(runtime.getCoordinator() == null) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "No build coordinator is configured"); + } + QRecord artifact = executor.loadVerifiedArtifact(runtime.getCoordinator(), buildId); + if(publicationId != null) + { + QRecord publication = store.require(ScriptingTables.PUBLICATION, publicationId, "Publication"); + if(!Objects.equals(buildId, publication.getValueInteger("build_id")) + || !Objects.equals(build.getValueInteger("script_id"), publication.getValueInteger("script_id")) + || !Objects.equals(artifact.getValueInteger("id"), publication.getValueInteger("artifact_id"))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Publication does not match its immutable artifact"); + } + } + return (new Pin(buildId, build.getValueInteger("script_id"), build.getValueInteger("script_revision_id"), + build.getValueString("source_sha256"), build.getValueString("manifest_json"), build.getValueString("manifest_attestation"), + artifact.getValueInteger("id"), new WarmWorkerPreparation.Artifact(artifact.getValueString("bundle_sha256"), + artifact.getValueString("entry_point"), artifact.getValueString("bundle"), artifact.getValueString("toolchain_id")))); + } + + /*************************************************************************** + ** Immutable code and association identity held only for this authoring call. + ***************************************************************************/ + private record Pin(Integer buildId, Integer projectId, Integer revisionId, String sourceHash, String manifest, + String manifestAttestation, Integer artifactId, WarmWorkerPreparation.Artifact artifact) + { + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/IsolatedSourceGuard.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/IsolatedSourceGuard.java new file mode 100644 index 0000000..8521507 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/IsolatedSourceGuard.java @@ -0,0 +1,193 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.scripts.ScriptExecutionIsolation; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptExecutionMode; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; + + +/******************************************************************************* + ** Isolated source is append-only, including its owning project association. + ** Host customizers run first; this final guard checks their resulting values. + ** Legacy source remains mutable. Orphaned associations always fail closed. + *******************************************************************************/ +public class IsolatedSourceGuard implements TableCustomizerInterface +{ + /*************************************************************************** + ** Only the authorized source lifecycle may append isolated revisions/files. + ***************************************************************************/ + @Override + public List preInsert(InsertInput input, List rows, boolean preview) throws QException + { + for(QRecord row : rows) + { + if(isolated(input.getTableName(), row, input.getTransaction()) && !ScriptExecutionIsolation.isIsolationAuthorityActive()) + { + throw new QPermissionDeniedException("Isolated sources must be saved through the scripting service"); + } + } + return (rows); + } + + + + /*************************************************************************** + ** Neither content edits nor reparenting may erase an isolated association. + ***************************************************************************/ + @Override + public List preUpdate(UpdateInput input, List rows, boolean preview, Optional> prior) throws QException + { + for(QRecord row : rows) + { + QRecord old = stored(input.getTableName(), row.getValueInteger("id"), input.getTransaction()); + QRecord merged = new QRecord(); + old.getValues().forEach(merged::setValue); + row.getValues().forEach(merged::setValue); + if(isolated(input.getTableName(), old, input.getTransaction()) || isolated(input.getTableName(), merged, input.getTransaction())) + { + throw new QPermissionDeniedException("Isolated source revisions and files are immutable"); + } + } + return (rows); + } + + + + /*************************************************************************** + ** An isolated source identity must remain resolvable forever. + ***************************************************************************/ + @Override + public List preDelete(DeleteInput input, List rows, boolean preview) throws QException + { + for(QRecord row : rows) + { + if(isolated(input.getTableName(), stored(input.getTableName(), row.getValueInteger("id"), input.getTransaction()), input.getTransaction())) + { + throw new QPermissionDeniedException("Isolated sources cannot be deleted"); + } + } + return (rows); + } + + + + /*************************************************************************** + ** Generic source reads must pass their current parent project's ACL. A + ** projection lacking its association is deliberately not readable. + ***************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface input, List rows) throws QException + { + if(QContext.getQSession() instanceof QSystemUserSession) + { + return (rows); + } + List visible = new ArrayList<>(); + for(QRecord row : rows) + { + try + { + Integer project = project(input.getTableName(), row, input.getTransaction()); + GetInput parent = new GetInput(); + parent.setTableName("script"); + parent.setPrimaryKey(project); + parent.setTransaction(input.getTransaction()); + parent.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + if(new GetAction().execute(parent).getRecord() != null) + { + visible.add(row); + } + } + catch(QException denied) + { + // Hidden, missing and malformed source associations have no visible row. + } + } + return (visible); + } + + + + /*************************************************************************** + ** Resolve the immutable parent, including an uncommitted revision insert. + ***************************************************************************/ + private static Integer project(String table, QRecord row, QBackendTransaction transaction) throws QException + { + Integer id = "scriptRevision".equals(table) ? row.getValueInteger("scriptId") + : stored("scriptRevision", row.getValueInteger("scriptRevisionId"), transaction).getValueInteger("scriptId"); + if(id == null) + { + throw new QPermissionDeniedException("Source project is not accessible"); + } + return (id); + } + + + + /*************************************************************************** + ** The monotonic core marker is the source of execution mode truth. + ***************************************************************************/ + private static boolean isolated(String table, QRecord row, QBackendTransaction transaction) throws QException + { + return (ScriptExecutionIsolation.getExecutionMode(project(table, row, transaction), transaction) == ScriptExecutionMode.ISOLATED); + } + + + + /*************************************************************************** + ** Privileged association lookup is used only for integrity, never disclosure. + ***************************************************************************/ + private static QRecord stored(String table, Integer id, QBackendTransaction transaction) throws QException + { + if(id == null) + { + throw new QPermissionDeniedException("Source association is missing"); + } + GetInput input = new GetInput(); + input.setTableName(table); + input.setPrimaryKey(id); + input.setTransaction(transaction); + QRecord row = SystemScope.run(() -> new GetAction().execute(input).getRecord()); + if(row == null) + { + throw new QPermissionDeniedException("Source association is missing"); + } + return (row); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptExecutor.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptExecutor.java new file mode 100644 index 0000000..a3a62f9 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptExecutor.java @@ -0,0 +1,440 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.build.ArtifactSubmission; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.capability.CapabilityBroker; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookResult; +import com.kingsrook.qbits.scripting.observability.ScriptTiming; +import com.kingsrook.qbits.scripting.service.ScriptingResults.ExecutionSummary; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecuteRequest; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecuteResponse; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Outcome; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import com.kingsrook.qqq.backend.core.actions.scripts.ScriptExecutionIsolation; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.json.JSONException; +import org.json.JSONObject; + + +/******************************************************************************* + ** Runs one pinned, verified artifact in the external worker and records the + ** attempt. It never evaluates script itself. Order of refusal, each fail-closed: + ** coordinator configured; artifact re-verified against its attestation and hash + ** (catches a tampered stored bundle); worker qualification matches the required + ** evidence id; bounded payload; scoped token; worker call; bounded, typed answer. + ** A run whose history cannot be recorded is discarded, not trusted. + *******************************************************************************/ +class ScriptExecutor +{ + private static final int MAX_STORED_MESSAGE = 500; + + private final ScriptingRuntime runtime; + private final ScriptingStore store; + private final CapabilityBroker broker; + + + + /******************************************************************************* + ** Everything one execution needs; priorJson and bindingId may be null. + *******************************************************************************/ + record Request(String kind, String actorId, Integer buildId, Integer bindingId, Integer publicationId, String event, String proposedJson, String priorJson, List grants, String eventId, ExecutionAuthority authority) + { + /*********************************************************************** + ** Existing synchronous callers do not carry a committed event identity. + ***********************************************************************/ + Request(String kind, String actorId, Integer buildId, Integer bindingId, Integer publicationId, String event, String proposedJson, String priorJson, List grants) + { + this(kind, actorId, buildId, bindingId, publicationId, event, proposedJson, priorJson, grants, null, null); + } + } + + /*************************************************************************** + ** Internal owner gate, never populated from a script or HTTP request. + ***************************************************************************/ + @FunctionalInterface + interface ExecutionAuthority + { + /*********************************************************************** + ** Recheck authority after all preparation callbacks and before execution. + ***********************************************************************/ + void check() throws ScriptingException; + } + + + + /*************************************************************************** + ** Create an executor over the runtime collaborators. + ***************************************************************************/ + ScriptExecutor(ScriptingRuntime runtime, ScriptingStore store, CapabilityBroker broker) + { + this.runtime = runtime; + this.store = store; + this.broker = broker; + } + + + + /*************************************************************************** + ** Cross-instance guard for synchronous nested worker execution on this thread. + ***************************************************************************/ + private static final ThreadLocal EXECUTING = new ThreadLocal<>(); + + /*************************************************************************** + ** A worker-origin callback cannot enqueue another lifecycle execution. + ***************************************************************************/ + static boolean isExecuting() + { + return (Boolean.TRUE.equals(EXECUTING.get())); + } + + /*************************************************************************** + ** Nested execution from worker callbacks is forbidden across service instances. + ***************************************************************************/ + ExecutionSummary execute(Request request) throws ScriptingException + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.EXECUTION_TOTAL)) + { + try + { + if("DURABLE_HOOK".equals(request.kind()) && request.authority() == null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Committed dispatch requires native execution authority"); + } + if(Boolean.TRUE.equals(EXECUTING.get())) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Recursive script execution is forbidden"); + } + EXECUTING.set(true); + try + { + return (executeOnce(request)); + } + finally + { + EXECUTING.remove(); + } + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + } + + /*************************************************************************** + ** Independently audit the attempt and outcome, never the record commit. + ***************************************************************************/ + private ExecutionSummary executeOnce(Request request) throws ScriptingException + { + Instant started = runtime.getClock().instant(); + String status; + HookResult hook = null; + String contextJson = null; + String message = null; + Integer artifactRowId = null; + int wallMillis = 0; + Map row = new HashMap<>(); + row.put("operation_id", request.eventId() == null ? UUID.randomUUID().toString() : request.eventId()); + row.put("kind", request.kind()); + row.put("build_id", request.buildId()); + row.put("binding_id", request.bindingId()); + row.put("publication_id", request.publicationId()); + row.put("actor_id", request.actorId()); + row.put("event", request.event()); + row.put("status", "STARTED"); + row.put("created_at", started); + store.history(row); + + try + { + BuildCoordinator coordinator = runtime.getCoordinator(); + if(coordinator == null) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "No build coordinator is configured"); + } + QRecord artifactRow = loadVerifiedArtifact(coordinator, request.buildId()); + artifactRowId = artifactRow.getValueInteger("id"); + requireQualifiedWorker(); + + ExecuteResponse response = runInWorker(request, artifactRow, artifactRowId); + status = response.outcome().name(); + wallMillis = response.wallMillis(); + if(response.outcome() == Outcome.OK) + { + hook = response.hook(); + if(hook != null && (((request.event().startsWith("POST_") || request.event().startsWith("AFTER_")) && hook.action() != HookResult.Action.ALLOW) + || ("PRE_DELETE".equals(request.event()) && hook.action() == HookResult.Action.PATCH))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "This event cannot apply the returned decision"); + } + contextJson = response.contextJson(); + } + else + { + message = "The isolated worker reported " + response.outcome().name(); + } + } + catch(ScriptingException e) + { + status = e.getKind().name(); + message = "Script execution was refused: " + e.getKind().name(); + } + catch(RuntimeException e) + { + status = ScriptingException.Kind.UNAVAILABLE.name(); + message = "The script execution could not be completed"; + } + + if(wallMillis == 0) + { + wallMillis = (int) Math.min(Integer.MAX_VALUE, Duration.between(started, runtime.getClock().instant()).toMillis()); + } + + row.put("artifact_id", artifactRowId); + row.put("status", status); + row.put("duration_ms", wallMillis); + row.put("message", message); + row.put("result_action", hook == null ? null : hook.action().name()); + row.put("created_at", runtime.getClock().instant()); + Integer executionId = store.history(row); + + return (new ExecutionSummary(executionId, status, hook, message, contextJson)); + } + + + + /*************************************************************************** + ** Re-assemble the stored artifact and verify it exactly as at acceptance. + ***************************************************************************/ + QRecord loadVerifiedArtifact(BuildCoordinator coordinator, Integer buildId) throws ScriptingException + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.ARTIFACT_VERIFICATION)) + { + try + { + QRecord build = store.require(ScriptingTables.BUILD, buildId, "Build"); + if(!"ISOLATED".equals(store.require("script", build.getValueInteger("script_id"), "Script").getValueString(ScriptExecutionIsolation.EXECUTION_MODE_FIELD_NAME))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Execution requires an isolated project"); + } + verifyManifest(coordinator, build); + if(!"SUCCEEDED".equals(build.getValueString("status"))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "The build did not succeed and has no artifact"); + } + List artifacts = store.where(ScriptingTables.ARTIFACT, "build_id", buildId); + if(artifacts.size() != 1) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "The build does not have exactly one artifact"); + } + QRecord artifact = artifacts.get(0); + if(!build.getValueInteger("script_id").equals(artifact.getValueInteger("script_id"))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Artifact project does not match its build"); + } + coordinator.verify(new ArtifactSubmission( + artifact.getValueString("coordinator_id"), + artifact.getValueString("source_sha256"), + artifact.getValueString("toolchain_id"), + artifact.getValueString("entry_point"), + artifact.getValueString("bundle"), + artifact.getValueString("bundle_sha256"), + artifact.getValueString("source_map"), + artifact.getValueString("attestation")), build.getValueString("source_sha256")); + return (artifact); + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + } + + + + /*************************************************************************** + ** Check signed capabilities and their associations before use. + ***************************************************************************/ + static void verifyManifest(BuildCoordinator coordinator, QRecord build) throws ScriptingException + { + String json = build.getValueString("manifest_json"); + coordinator.verifyManifest(json, build.getValueString("manifest_attestation")); + try + { + JSONObject manifest = new JSONObject(json); + if(!Integer.valueOf(manifest.getInt("scriptId")).equals(build.getValueInteger("script_id")) + || !Integer.valueOf(manifest.getInt("scriptRevisionId")).equals(build.getValueInteger("script_revision_id")) + || !manifest.getString("sourceSha256").equals(build.getValueString("source_sha256")) + || !manifest.getString("toolchainId").equals(coordinator.getToolchainId()) || !manifest.getBoolean("strict")) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Build does not match its signed manifest"); + } + } + catch(JSONException e) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Build manifest is unreadable"); + } + } + + + + /*************************************************************************** + ** Refuse unless the worker reports exactly the qualification evidence the host + ** requires. A blank requirement refuses everything. + ***************************************************************************/ + void requireQualifiedWorker() throws ScriptingException + { + String required = runtime.getRequiredQualificationId(); + String reported = runtime.getWorker().qualificationId(); + if(required == null || required.isBlank() || !required.equals(reported)) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "The script worker is not qualified for execution"); + } + } + + + + /*************************************************************************** + ** Open a scoped token, call the worker, close the token, decode the answer. + ***************************************************************************/ + private ExecuteResponse runInWorker(Request request, QRecord artifact, Integer artifactRowId) throws ScriptingException + { + String proposed = WorkerProtocol.checkPayload(request.proposedJson()); + boolean context = "CONTEXT".equals(request.event()); + JSONObject payload = new JSONObject().put(context ? "context" : (request.event().startsWith("AFTER_") ? "event" : "proposed"), new JSONObject(proposed)); + if(request.event().startsWith("AFTER_") && "TEST".equals(request.kind())) + { + payload.getJSONObject("event").put("dryRun", true); + } + if(request.priorJson() != null) + { + payload.put("prior", new JSONObject(WorkerProtocol.checkPayload(request.priorJson()))); + } + + String requestId = "x-" + UUID.randomUUID(); + Actor.requireCurrentIdentity(request.actorId()); + QSession session = QContext.getQSession(); + Thread caller = Thread.currentThread(); + AtomicInteger callsLeft = new AtomicInteger(runtime.getLimits().brokerCalls()); + String token = broker.open(request.grants(), runtime.getWorker().executionTransportBudget()); + try(var origin = ScriptExecutionIsolation.openScriptOriginScope()) + { + String requestJson = WorkerProtocol.encodeExecute(new ExecuteRequest(requestId, "a-" + artifactRowId, artifact.getValueString("bundle_sha256"), + artifact.getValueString("bundle"), artifact.getValueString("entry_point"), request.event(), payload.toString(), token, runtime.getLimits())); + if(request.authority() != null) + { + request.authority().check(); + } + String responseJson = dispatch(requestJson, json -> + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.BROKER_CALLBACK)) + { + try + { + if(Thread.currentThread() != caller || QContext.getQSession() != session) + { + return (WorkerProtocol.encodeBrokerError("DENIED", "Broker callback requires the initiating session and thread")); + } + if(callsLeft.decrementAndGet() < 0) + { + return (WorkerProtocol.encodeBrokerError("LIMIT", "Execution read budget is exhausted")); + } + return (broker.handle(json)); + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + }); + if(responseJson == null || responseJson.getBytes(StandardCharsets.UTF_8).length > runtime.getLimits().outputBytes()) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Execution output exceeded its budget"); + } + ExecuteResponse response = WorkerProtocol.decodeExecuteResponse(responseJson, context); + if(!requestId.equals(response.requestId())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Worker answered a different request"); + } + return (response); + } + finally + { + broker.close(token); + } + } + + + + /*************************************************************************** + ** Measure actual transport dispatch, including synchronous broker callbacks. + ***************************************************************************/ + private String dispatch(String requestJson, com.kingsrook.qbits.scripting.worker.WorkerProvider.BrokerChannel channel) throws ScriptingException + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.WORKER_DISPATCH)) + { + try + { + return (runtime.getWorker().execute(requestJson, channel)); + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + } + + + + /*************************************************************************** + ** Bound and de-control untrusted text before it is stored or shown. + ***************************************************************************/ + static String safe(String text) + { + if(text == null) + { + return (null); + } + String cleaned = text.replaceAll("\\p{Cntrl}", " "); + return (cleaned.length() > MAX_STORED_MESSAGE ? cleaned.substring(0, MAX_STORED_MESSAGE) : cleaned); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptSourceRegistry.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptSourceRegistry.java new file mode 100644 index 0000000..e390ecc --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptSourceRegistry.java @@ -0,0 +1,170 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.scripts.ScriptExecutionIsolation; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scripts.Script; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile; + + +/******************************************************************************* + ** The draft source registry. This is NOT a new registry: it reads and appends to + ** the existing core script, scriptRevision and scriptRevisionFile tables, so the + ** existing script UI keeps working and there is one place sources live. + ** + ** Contract with core: this QBit only ever INSERTS new revisions (a save is a new + ** revision with the next sequenceNo) and never updates or deletes a revision or + ** file. Publication alone advances the core pointer through typed isolation + ** authority in its transaction. Lifecycle guards prevent generic source edits + ** and reparenting; build hashes additionally detect out-of-band corruption. + *******************************************************************************/ +class ScriptSourceRegistry +{ + private final ScriptingStore store; + + + + /******************************************************************************* + ** A revision as loaded for building: its project and its hashed snapshot. + *******************************************************************************/ + record LoadedRevision(Integer scriptId, Integer revisionId, SourceSnapshot snapshot) + { + } + + + + /*************************************************************************** + ** Create a registry over the scripting store. + ***************************************************************************/ + ScriptSourceRegistry(ScriptingStore store) + { + this.store = store; + } + + + + /*************************************************************************** + ** Append a new draft revision of an existing project and return its id. + ***************************************************************************/ + Integer saveDraft(Integer scriptId, String author, String commitMessage, SourceSnapshot snapshot) throws ScriptingException + { + requireProjectAccess(scriptId); + + return (ScriptExecutionIsolation.runAsIsolationAuthority("Append authorized isolated source revision", () -> append(scriptId, author, commitMessage, snapshot))); + } + + + + /*************************************************************************** + ** Insert immutable source rows while the narrow authority is active. + ***************************************************************************/ + private Integer append(Integer scriptId, String author, String commitMessage, SourceSnapshot snapshot) throws ScriptingException + { + int next = 1; + for(QRecord revision : store.where(ScriptRevision.TABLE_NAME, "scriptId", scriptId)) + { + Integer sequence = revision.getValueInteger("sequenceNo"); + if(sequence != null && sequence >= next) + { + next = sequence + 1; + } + } + + Integer revisionId = store.insert(ScriptRevision.TABLE_NAME, Map.of( + "scriptId", scriptId, + "sequenceNo", next, + "author", author, + "commitMessage", commitMessage == null ? "" : commitMessage)); + + for(SourceFile file : snapshot.files()) + { + store.insert(ScriptRevisionFile.TABLE_NAME, Map.of( + "scriptRevisionId", revisionId, + "fileName", file.path(), + "contents", file.content())); + } + return (revisionId); + } + + + + /*************************************************************************** + ** Load a revision and its files as an immutable, hashed snapshot. A file that + ** violates source policy makes the whole revision unbuildable. + ***************************************************************************/ + LoadedRevision load(Integer revisionId) throws ScriptingException + { + QRecord revision = store.require(ScriptRevision.TABLE_NAME, revisionId, "Script revision"); + requireProjectAccess(revision.getValueInteger("scriptId")); + List files = new ArrayList<>(); + try + { + for(QRecord file : store.where(ScriptRevisionFile.TABLE_NAME, "scriptRevisionId", revisionId)) + { + files.add(new SourceFile(file.getValueString("fileName"), file.getValueString("contents"))); + } + } + catch(IllegalArgumentException e) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "The revision contains a file that violates the source policy"); + } + return (new LoadedRevision(revision.getValueInteger("scriptId"), revisionId, SourceSnapshot.of("draft:" + revisionId, files))); + } + + + + /*************************************************************************** + ** Authorize the owning core project under the caller before privileged + ** source storage. Hidden or inaccessible projects have the same response. + ***************************************************************************/ + void requireProjectAccess(Integer scriptId) throws ScriptingException + { + try + { + GetInput input = new GetInput(); + input.setTableName(Script.TABLE_NAME); + input.setPrimaryKey(scriptId); + input.setTransaction(store.currentTransaction()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + if(scriptId == null || new GetAction().execute(input).getRecord() == null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Script project is not accessible"); + } + } + catch(QException e) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Script project is not accessible"); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingAuthoring.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingAuthoring.java new file mode 100644 index 0000000..4c8274f --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingAuthoring.java @@ -0,0 +1,287 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingPermissions; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Diagnostic; +import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessHelper; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.json.JSONArray; +import org.json.JSONObject; + + +/******************************************************************************* + ** Bounded project authoring views. Never exposes worker bundles, signatures, + ** configuration secrets or host records. Source reads require owning-project ACL. + *******************************************************************************/ +public final class ScriptingAuthoring +{ + private final ScriptingRuntime runtime; + private final ScriptingStore store; + private final ScriptSourceRegistry registry; + + /*************************************************************************** + ** Per-service collaborators; this facade has no parallel persistence. + ***************************************************************************/ + ScriptingAuthoring(ScriptingRuntime runtime, ScriptingStore store, ScriptSourceRegistry registry) + { + this.runtime = runtime; + this.store = store; + this.registry = registry; + } + + /*************************************************************************** + ** Source revision summary; source text is retrieved separately. + ***************************************************************************/ + public record Revision(Integer id, Integer sequence, String message, String author) {} + + /*************************************************************************** + ** Lifecycle rows contain only selected public fields, never artifact bytes. + ***************************************************************************/ + public record Status(Integer id, Integer revisionId, Integer artifactId, Integer version, String status, String actor, String createdAt) {} + + /*************************************************************************** + ** Most recent bounded history and current server-authorized controls. + ***************************************************************************/ + public record Project(Integer scriptId, String executionMode, Integer latestRevisionId, List revisions, + List builds, List publications, List actions, boolean workerAvailable, boolean importAvailable) {} + + /*************************************************************************** + ** Immutable source snapshot for one authorized revision. + ***************************************************************************/ + public record Draft(Integer revisionId, String sourceSha256, List files, ImportProvenance provenance) {} + + /*************************************************************************** + ** Repository origin is a retained receipt, separate from editable draft notes. + ***************************************************************************/ + public record ImportProvenance(String repository, String commitSha, String subdirectory, String sourceSha256, String importedBy) {} + + /*************************************************************************** + ** Native project view, newest100 entries per history. This is a bounded + ** recent-history view, not a claim that older history has been deleted. + ***************************************************************************/ + public Project project(Actor actor, Integer projectId) throws ScriptingException + { + actor.require(ScriptingPermissions.VIEW); + registry.requireProjectAccess(projectId); + QRecord project = store.require("script", projectId, "Script"); + List revisions = recent("scriptRevision", "scriptId", projectId); + List actions = new ArrayList<>(); + for(String action : List.of("draft", "check", "test", "publish", "bind", "execute")) + { + try + { + actor.require("scripting." + action); + actions.add(action); + } + catch(ScriptingException denied) + { + // The view reports only controls authorized in the current session. + } + } + String required = runtime.getRequiredQualificationId(); + boolean available = runtime.getCoordinator() != null && required != null && !required.isBlank() + && required.equals(runtime.getWorker().qualificationId()); + return (new Project(projectId, Objects.requireNonNullElse(project.getValueString("executionMode"), "LEGACY"), + revisions.isEmpty() ? null : revisions.getFirst().getValueInteger("id"), + revisions.stream().map(row -> new Revision(row.getValueInteger("id"), row.getValueInteger("sequenceNo"), + row.getValueString("commitMessage"), row.getValueString("author"))).toList(), + recent(ScriptingTables.BUILD, "script_id", projectId).stream().map(row -> status(row, "requested_by")).toList(), + recent(ScriptingTables.PUBLICATION, "script_id", projectId).stream().map(row -> status(row, "published_by")).toList(), + List.copyOf(actions), available, runtime.getImporter() != null)); + } + + /*************************************************************************** + ** A project-scoped source read cannot name another project's revision. + ***************************************************************************/ + public Draft draft(Actor actor, Integer projectId, Integer revisionId) throws ScriptingException + { + actor.require(ScriptingPermissions.VIEW); + registry.requireProjectAccess(projectId); + var loaded = registry.load(revisionId); + if(!projectId.equals(loaded.scriptId())) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Revision is not in this project"); + } + List receipts = recent(ScriptingTables.IMPORT, "script_revision_id", revisionId); + ImportProvenance provenance = null; + if(!receipts.isEmpty()) + { + QRecord receipt = receipts.getFirst(); + if(!projectId.equals(receipt.getValueInteger("script_id")) || !loaded.snapshot().sha256().equals(receipt.getValueString("source_sha256"))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Imported source does not match its retained receipt"); + } + provenance = new ImportProvenance(receipt.getValueString("repository"), receipt.getValueString("commit_sha"), + receipt.getValueString("subdirectory"), receipt.getValueString("source_sha256"), receipt.getValueString("imported_by")); + } + return (new Draft(revisionId, loaded.snapshot().sha256(), loaded.snapshot().files(), provenance)); + } + + /*************************************************************************** + ** Semantic endpoints require the monotonic core marker. This never upgrades + ** a legacy project implicitly; trusted provisioning must establish isolation. + ***************************************************************************/ + public void requireIsolated(Actor actor, Integer projectId) throws ScriptingException + { + Actor.requireCurrentIdentity(actor.id()); + registry.requireProjectAccess(projectId); + if(!QContext.getQInstance().getTable("script").getFields().containsKey("executionMode") + || !"ISOLATED".equals(store.require("script", projectId, "Script").getValueString("executionMode"))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "This action requires an isolated script project"); + } + } + + /*************************************************************************** + ** Check a child identity against the already-authorized project boundary. + ***************************************************************************/ + public void requireBuild(Actor actor, Integer projectId, Integer buildId) throws ScriptingException + { + requireIsolated(actor, projectId); + if(!projectId.equals(store.require(ScriptingTables.BUILD, buildId, "Build").getValueInteger("script_id"))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Build is not in this project"); + } + } + + /*************************************************************************** + ** Recover bounded compiler diagnostics after an editor reload, without + ** exposing the stored executable bundle or its host signature. + ***************************************************************************/ + public ScriptingResults.BuildSummary build(Actor actor, Integer projectId, Integer buildId) throws ScriptingException + { + actor.require(ScriptingPermissions.VIEW); + requireBuild(actor, projectId, buildId); + QRecord build = store.require(ScriptingTables.BUILD, buildId, "Build"); + List diagnostics = new ArrayList<>(); + JSONArray values = new JSONArray(build.getValueString("diagnostics_json")); + for(int i = 0; i < Math.min(100, values.length()); i++) + { + JSONObject value = values.getJSONObject(i); + diagnostics.add(new Diagnostic(value.getString("file"), value.getInt("line"), value.getInt("column"), value.getString("code"), value.getString("message"))); + } + List artifacts = recent(ScriptingTables.ARTIFACT, "build_id", buildId); + QRecord artifact = artifacts.isEmpty() ? new QRecord() : artifacts.getFirst(); + return (new ScriptingResults.BuildSummary(buildId, "SUCCEEDED".equals(build.getValueString("status")), diagnostics, + artifact.getValueInteger("id"), artifact.getValueInteger("version"))); + } + + /*************************************************************************** + ** Generate table/field-specific editor declarations from explicit requested + ** capabilities and current read policy. No table enumeration or credentials. + ** These are editor assistance; the broker remains the authority at runtime. + ***************************************************************************/ + public String typings(Actor actor, Integer projectId, List declarations) throws ScriptingException + { + actor.require(ScriptingPermissions.VIEW); + registry.requireProjectAccess(projectId); + if(declarations == null || declarations.size() > CapabilityDeclaration.MAX_DECLARATIONS) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Too many type declarations"); + } + String events = Arrays.stream(HookEvent.values()).map(event -> "\"" + event.name() + "\"").collect(Collectors.joining(" | ")); + StringBuilder output = new StringBuilder("declare module \"@qqq/sdk\" {\n export type Scalar = string | number | boolean | null;\n export type HookEvent = " + events + ";\n") + .append(" export type CommittedEvent = Readonly<{ id: string; tableName: string; recordId: Exclude; dryRun?: boolean }> & (Readonly<{ dataState: \"CURRENT\"; record: Readonly> }> | Readonly<{ dataState: \"DELETED_METADATA\"; record?: never }>);\n") + .append(" export interface Context { readonly event: HookEvent | \"CONTEXT\"; readonly payload: Readonly> & { readonly event?: CommittedEvent }; }\n export interface Criterion { field: string; operator: string; values: Scalar[]; }\n export interface Qqq {\n"); + for(CapabilityDeclaration declaration : declarations) + { + var table = QContext.getQInstance().getTable(declaration.tableName()); + if(table == null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Requested API types are not accessible"); + } + try + { + QueryInput input = new QueryInput(table.getName()); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + CorePermissionEnforcer.checkTablePermissions(input, TablePermissionSubType.READ); + } + catch(QException denied) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Requested API types are not accessible"); + } + List fields = new ArrayList<>(); + for(String name : declaration.fields().stream().sorted().toList()) + { + var field = table.getFields().get(name); + if(field == null || Boolean.TRUE.equals(field.getIsHidden()) || field.getType() == QFieldType.PASSWORD + || field.getType() == QFieldType.BLOB || !FieldAccessHelper.getTier(table, name).canReadValue()) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Requested API types are not accessible"); + } + fields.add(JSONObject.quote(name)); + } + output.append(" query(table: ").append(JSONObject.quote(table.getName())).append(", fields: Array<") + .append(String.join(" | ", fields)).append(">, criteria?: Criterion[], limit?: number): Promise>>>>;\n"); + } + return (output.append(" }\n}\n").toString()); + } + + /*************************************************************************** + ** Latest id for optimistic save checks inside the existing transaction. + ***************************************************************************/ + Integer latestRevision(Integer projectId) throws ScriptingException + { + var rows = recent("scriptRevision", "scriptId", projectId); + return (rows.isEmpty() ? null : rows.getFirst().getValueInteger("id")); + } + + /*************************************************************************** + ** Bound reads in the backend, before materializing result rows. + ***************************************************************************/ + private List recent(String table, String key, Integer id) throws ScriptingException + { + return (store.query(table, new QQueryFilter().withCriteria(key, QCriteriaOperator.EQUALS, id) + .withOrderBy(new QFilterOrderBy("id", false)).withLimit(100))); + } + + /*************************************************************************** + ** No manifest, attestation, bundle or raw diagnostics in a history row. + ***************************************************************************/ + private static Status status(QRecord row, String actor) + { + return (new Status(row.getValueInteger("id"), row.getValueInteger("script_revision_id"), row.getValueInteger("artifact_id"), + row.getValueInteger("version"), row.getValueString("status"), row.getValueString(actor), row.getValueString("created_at"))); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingHookAuthoring.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingHookAuthoring.java new file mode 100644 index 0000000..f723b44 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingHookAuthoring.java @@ -0,0 +1,453 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.service; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingPermissions; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.HookRecordAccess; +import com.kingsrook.qbits.scripting.hooks.ScriptAfterHooks; +import com.kingsrook.qbits.scripting.hooks.ScriptHookCustomizer; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qbits.scripting.service.ScriptingResults.ExecutionSummary; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessHelper; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractTableActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.QInputSource; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.json.JSONObject; + + +/******************************************************************************* + ** Project-scoped hook administration and selected-record dry runs. Reads use the + ** native hook context boundary; mutations use the existing binding service only. + *******************************************************************************/ +public final class ScriptingHookAuthoring +{ + private final ScriptingService service; + private final ScriptingStore store; + private final ScriptSourceRegistry registry; + + /*************************************************************************** + ** Per-request facade over the same durable service and source registry. + ***************************************************************************/ + ScriptingHookAuthoring(ScriptingService service, ScriptingStore store, ScriptSourceRegistry registry) + { + this.service = service; + this.store = store; + this.registry = registry; + } + + /*************************************************************************** + ** A host-installed phase and currently readable/writable patch fields. + ***************************************************************************/ + public record Phase(HookEvent event, List patchableFields) {} + + /*************************************************************************** + ** Current authorized metadata target, never an arbitrary table selector. + ***************************************************************************/ + public record Target(String tableName, String label, List phases) {} + + /*************************************************************************** + ** Immutable binding configuration with its current enabled flag. + ***************************************************************************/ + public record Binding(Integer id, String name, String tableName, HookEvent event, Integer publicationId, + boolean enabled, List grants, Set patchableFields) {} + + /*************************************************************************** + ** Stable wire grant matches the build/editor READ declaration contract. + ***************************************************************************/ + public record Grant(String operation, String table, Set fields, int maxRows, int maxCalls) {} + + /*************************************************************************** + ** Bounded recent view; older publications remain retained in ordinary history. + ***************************************************************************/ + public record View(List targets, List bindings, boolean historyLimited, Integer nextBindingId) {} + + /*************************************************************************** + ** First page of authorized targets and bindings across all retained publications. + ***************************************************************************/ + public View view(Actor actor, Integer projectId) throws ScriptingException + { + return (view(actor, projectId, null)); + } + + /*************************************************************************** + ** Native project/target filtering with bounded keyset pages; no publication + ** age can make an active binding disappear from ordinary administration. + ***************************************************************************/ + public View view(Actor actor, Integer projectId, Integer beforeBindingId) throws ScriptingException + { + if(beforeBindingId != null && beforeBindingId < 1) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Invalid binding page cursor"); + } + actor.require(ScriptingPermissions.VIEW); + service.authoring().requireIsolated(actor, projectId); + List targets = new ArrayList<>(); + List readableTables = new ArrayList<>(); + for(QTableMetaData table : QContext.getQInstance().getTables().values().stream().sorted(Comparator.comparing(QTableMetaData::getName)).toList()) + { + try + { + readPermission(table.getName(), null); + readableTables.add(table.getName()); + } + catch(QException denied) + { + continue; + } + List phases = new ArrayList<>(); + for(HookEvent event : HookEvent.values()) + { + try + { + ScriptHookCustomizer hook = target(table.getName(), event); + phases.add(new Phase(event, patchFields(table, event, hook))); + } + catch(ScriptingException unavailable) + { + // Only installed and currently authorized phases enter the catalog. + } + } + if(!phases.isEmpty() && targets.size() < 200) + { + targets.add(new Target(table.getName(), table.getLabel(), List.copyOf(phases))); + } + } + List rows = store.bindings(projectId, beforeBindingId, readableTables); + List bindings = new ArrayList<>(); + for(QRecord row : rows.stream().limit(100).toList()) + { + try + { + // Hidden target metadata cannot leak through a project binding view. + readPermission(row.getValueString("table_name"), null); + bindings.add(binding(row)); + } + catch(QException denied) + { + // Preserve native target visibility even for source-project editors. + } + } + return (new View(List.copyOf(targets), List.copyOf(bindings), rows.size() > 100 || targets.size() == 200, rows.size() > 100 ? rows.get(99).getValueInteger("id") : null)); + } + + /*************************************************************************** + ** Bind only a current project publication to an explicitly installed phase. + ***************************************************************************/ + public Integer bind(Actor actor, Integer projectId, BindingSpec spec) throws ScriptingException + { + actor.require(ScriptingPermissions.BIND); + requirePublication(actor, projectId, spec.publicationId()); + ScriptHookCustomizer hook = target(spec.tableName(), spec.event()); + if(!patchFields(QContext.getQInstance().getTable(spec.tableName()), spec.event(), hook).containsAll(spec.patchableFields())) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Patch fields are not available in this phase"); + } + if(spec.name() == null || spec.name().isBlank() || spec.name().length() > 200) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "A bounded binding name is required"); + } + return (service.bind(actor, spec)); + } + + /*************************************************************************** + ** Enable revalidates target metadata; disabling remains possible after a host + ** removes a phase, with current project and target READ authorization. + ***************************************************************************/ + public void setEnabled(Actor actor, Integer projectId, Integer bindingId, boolean enabled) throws ScriptingException + { + actor.require(ScriptingPermissions.BIND); + service.authoring().requireIsolated(actor, projectId); + QRecord row = store.require(ScriptingTables.BINDING, bindingId, "Binding"); + requirePublication(actor, projectId, row.getValueInteger("publication_id")); + Binding binding = binding(row); + if(enabled) + { + ScriptHookCustomizer hook = target(binding.tableName(), binding.event()); + if(!patchFields(QContext.getQInstance().getTable(binding.tableName()), binding.event(), hook).containsAll(binding.patchableFields())) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Binding patch fields are no longer supported"); + } + } + else + { + try + { + readPermission(binding.tableName(), null); + } + catch(QException denied) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Hook target is not accessible"); + } + } + service.setBindingEnabled(actor, bindingId, enabled); + } + + /*************************************************************************** + ** Simulate a built event against an authorized current record. No native write + ** or record callback is invoked and a returned patch is never applied. Ordinary + ** write validations still belong to an eventual real native operation. + ***************************************************************************/ + public ExecutionSummary test(Actor actor, Integer projectId, Integer buildId, String tableName, HookEvent event, + Serializable recordId, Map changes) throws ScriptingException + { + actor.require(ScriptingPermissions.TEST); + service.authoring().requireBuild(actor, projectId, buildId); + ScriptHookCustomizer hook = target(tableName, event); + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(changes == null || changes.size() > 100 || (!event.permitsPatch() && !changes.isEmpty())) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Only insert/update tests accept bounded proposed changes"); + } + WorkerProtocol.checkPayload(new JSONObject(changes).toString()); + List allowed = patchFields(table, event, hook); + if(!allowed.containsAll(changes.keySet())) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Proposed changes contain an inaccessible field"); + } + AbstractTableActionInput input = action(tableName, event); + var session = QContext.getQSession(); + var instance = QContext.getQInstance(); + try + { + QRecord selected = new QRecord().withValue(table.getPrimaryKeyField(), recordId); + QRecord current = HookRecordAccess.read(input, selected); + String prior = HookRecordAccess.project(table, current); + QRecord proposed = new QRecord(current); + for(Map.Entry entry : changes.entrySet()) + { + Object value = entry.getValue(); + if(value != null && !(value instanceof String || value instanceof Boolean || value instanceof Number)) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Proposed changes must contain JSON scalars"); + } + proposed.setValue(entry.getKey(), (Serializable) value); + } + String payload = HookRecordAccess.project(table, proposed); + if(event.isAfterCommit()) + { + AfterHookService.requireLocatorAccess(tableName, recordId); + JSONObject observation = new JSONObject().put("id", "dry-run").put("dryRun", true).put("tableName", tableName) + .put("recordId", recordId).put("dataState", event == HookEvent.AFTER_DELETE ? "DELETED_METADATA" : "CURRENT"); + if(event != HookEvent.AFTER_DELETE) + { + observation.put("record", new JSONObject(payload)); + } + payload = observation.toString(); + } + ExecutionSummary result = service.test(actor, buildId, event, payload, event == HookEvent.PRE_UPDATE ? prior : null); + if(QContext.getQSession() != session || QContext.getQInstance() != instance) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Test context changed"); + } + Actor.requireCurrentIdentity(actor.id()); + target(tableName, event); + HookRecordAccess.read(input, selected); + return (result); + } + catch(ScriptingException e) + { + throw e; + } + catch(QException denied) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Selected hook test record is not readable"); + } + } + + /*************************************************************************** + ** Project association must be checked before exposing a publication or binding. + ***************************************************************************/ + private void requirePublication(Actor actor, Integer projectId, Integer publicationId) throws ScriptingException + { + service.authoring().requireIsolated(actor, projectId); + QRecord publication = store.require(ScriptingTables.PUBLICATION, publicationId, "Publication"); + registry.requireProjectAccess(publication.getValueInteger("script_id")); + if(!projectId.equals(publication.getValueInteger("script_id"))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Publication is not in this project"); + } + } + + /*************************************************************************** + ** Owner mutations call this inside their transaction, including direct Java + ** callers. The HTTP facade is never the sole authorization boundary. + ***************************************************************************/ + static void authorizeBinding(String tableName, HookEvent event, Set fields) throws ScriptingException + { + ScriptHookCustomizer hook = target(tableName, event); + if(!patchFields(QContext.getQInstance().getTable(tableName), event, hook).containsAll(fields)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Patch fields are not available in this phase"); + } + } + + /*************************************************************************** + ** Disabling a removed phase still requires current native table READ authority. + ***************************************************************************/ + static void authorizeDisable(String tableName) throws ScriptingException + { + try + { + readPermission(tableName, null); + } + catch(QException denied) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Hook target is not accessible"); + } + } + + /*************************************************************************** + ** A target is usable only when trusted host metadata installed this phase. + ***************************************************************************/ + private static ScriptHookCustomizer target(String tableName, HookEvent event) throws ScriptingException + { + QTableMetaData table = QContext.getQInstance().getTable(tableName); + if(table == null || tableName.startsWith("scripting_") || Set.of("script", "scriptRevision", "scriptRevisionFile").contains(tableName)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Hook target is not accessible"); + } + ScriptHookCustomizer hook = null; + if(event.isAfterCommit()) + { + try + { + if(!ScriptAfterHooks.installed(table, event)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "This durable phase is not installed by the host"); + } + } + catch(QException unavailable) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "This durable phase is not installed by the host"); + } + } + else + { + var reference = table.getCustomizer(TableCustomizers.valueOf(event.name() + "_RECORD").getRole()).orElse(null); + if(reference == null || reference.getCodeType() != QCodeType.JAVA || reference.getInlineCode() != null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "This hook phase is not installed by the host"); + } + var configured = QCodeLoader.getAdHoc(TableCustomizerInterface.class, reference); + if(!(configured instanceof ScriptHookCustomizer scriptHook)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "This hook phase is not installed by the host"); + } + hook = scriptHook; + } + try + { + readPermission(tableName, null); + PermissionsHelper.checkTablePermissionThrowing(action(tableName, event), switch(event) + { + case PRE_INSERT, POST_INSERT, AFTER_INSERT -> TablePermissionSubType.INSERT; + case PRE_UPDATE, POST_UPDATE, AFTER_UPDATE -> TablePermissionSubType.EDIT; + case PRE_DELETE, POST_DELETE, AFTER_DELETE -> TablePermissionSubType.DELETE; + }); + } + catch(QException denied) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Hook target is not accessible"); + } + return (hook); + } + + /*************************************************************************** + ** Use actual action classes for extensible host permission dispatch. + ***************************************************************************/ + private static AbstractTableActionInput action(String table, HookEvent event) + { + AbstractTableActionInput input = switch(event) + { + case PRE_INSERT, POST_INSERT, AFTER_INSERT -> new InsertInput(table); + case PRE_UPDATE, POST_UPDATE, AFTER_UPDATE -> new UpdateInput(table); + case PRE_DELETE, POST_DELETE, AFTER_DELETE -> new DeleteInput(table); + }; + input.setInputSource(QInputSource.USER); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + return (input); + } + + /*************************************************************************** + ** Native metadata read authorization; selected record reads are separate. + ***************************************************************************/ + private static void readPermission(String table, Serializable id) throws QException + { + PermissionsHelper.checkTablePermissionThrowing(new GetInput(table).withPrimaryKey(id).withInputSource(QInputSource.USER), TablePermissionSubType.READ); + } + + /*************************************************************************** + ** The catalog mirrors native patch policy, including current field tiers. + ***************************************************************************/ + private static List patchFields(QTableMetaData table, HookEvent event, ScriptHookCustomizer hook) + { + boolean permits = event == HookEvent.PRE_INSERT && hook.permitsInsertPatch(new InsertInput(table.getName()), false) + || event == HookEvent.PRE_UPDATE && hook.permitsUpdatePatch(new UpdateInput(table.getName()), false); + if(!permits) + { + return (List.of()); + } + return (table.getFields().values().stream().filter(field -> !field.getName().equals(table.getPrimaryKeyField()) + && field.getIsEditable() && !Boolean.TRUE.equals(field.getIsHidden()) + && Set.of(QFieldType.STRING, QFieldType.TEXT, QFieldType.INTEGER, QFieldType.LONG, QFieldType.DECIMAL, QFieldType.BOOLEAN).contains(field.getType()) + && FieldAccessHelper.getTier(table, field.getName()).canReadValue() && FieldAccessHelper.getTier(table, field.getName()).canWrite() + && (table.getRecordSecurityLocks() == null || table.getRecordSecurityLocks().stream().noneMatch(lock -> field.getName().equals(lock.getFieldName())))) + .map(field -> field.getName()).sorted().toList()); + } + + /*************************************************************************** + ** Decode only service-controlled binding metadata; no executable source bytes. + ***************************************************************************/ + private static Binding binding(QRecord row) throws ScriptingException + { + String fields = row.getValueString("patchable_fields"); + return (new Binding(row.getValueInteger("id"), row.getValueString("name"), row.getValueString("table_name"), HookEvent.valueOf(row.getValueString("event")), + row.getValueInteger("publication_id"), Integer.valueOf(1).equals(row.getValueInteger("enabled")), + CapabilityDeclaration.listFromJson(row.getValueString("grants_json")).stream().map(grant -> new Grant("READ", grant.tableName(), grant.fields(), grant.maxRows(), grant.maxCalls())).toList(), fields == null || fields.isBlank() ? Set.of() : Set.of(fields.split(",")))); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingResults.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingResults.java new file mode 100644 index 0000000..65a52ee --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingResults.java @@ -0,0 +1,135 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.HookResult; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Diagnostic; + + +/******************************************************************************* + ** Small immutable values returned by ScriptingService. They carry ids, statuses + ** and bounded text only; no entity, record, session or exception object. + *******************************************************************************/ +public final class ScriptingResults +{ + /******************************************************************************* + ** Immutable identity captured by a context consumer. Persist publication, + ** artifact and hash together; capabilities are the signed declaration. + *******************************************************************************/ + public record PublishedContext(Integer publicationId, Integer artifactId, String bundleSha256, List capabilities) + { + /*************************************************************************** + ** Freeze the declared capability list. + ***************************************************************************/ + public PublishedContext + { + capabilities = List.copyOf(capabilities); + } + } + + + + /*************************************************************************** + ** Utility holder. + ***************************************************************************/ + private ScriptingResults() + { + } + + + + /******************************************************************************* + ** Outcome of a check: the build row, and the artifact when it succeeded. + *******************************************************************************/ + public record BuildSummary(Integer buildId, boolean success, List diagnostics, Integer artifactId, Integer artifactVersion) + { + /*************************************************************************** + ** Freeze the diagnostics. + ***************************************************************************/ + public BuildSummary + { + diagnostics = List.copyOf(diagnostics); + } + } + + + + /******************************************************************************* + ** Outcome of one execution, test or hook. hook is non-null only when status is + ** OK. Any other status means the run did not produce a trustworthy decision + ** and callers must fail closed. + *******************************************************************************/ + public record ExecutionSummary(Integer executionId, String status, HookResult hook, String message, String contextJson) + { + public static final String OK = "OK"; + + + + /*************************************************************************** + ** True when the run completed with a decision. + ***************************************************************************/ + public boolean succeeded() + { + return (OK.equals(status) && (hook != null || contextJson != null)); + } + } + + + + /******************************************************************************* + ** What a host asks to bind: a published, pinned artifact to a table event, with + ** the capabilities granted (a subset of what its build declared) and the + ** fields a patch may touch. + *******************************************************************************/ + public record BindingSpec(String name, String tableName, HookEvent event, Integer publicationId, List grants, Set patchableFields) + { + /*************************************************************************** + ** Freeze collections. + ***************************************************************************/ + public BindingSpec + { + grants = List.copyOf(grants); + patchableFields = Set.copyOf(patchableFields); + } + } + + + + /******************************************************************************* + ** An enabled binding as the record hook sees it. + *******************************************************************************/ + public record HookBindingView(Integer id, String tableName, HookEvent event, Integer publicationId, List grants, Set patchableFields) + { + /*************************************************************************** + ** Freeze collections. + ***************************************************************************/ + public HookBindingView + { + grants = List.copyOf(grants); + patchableFields = Set.copyOf(patchableFields); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingRuntime.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingRuntime.java new file mode 100644 index 0000000..d1a4f9c --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingRuntime.java @@ -0,0 +1,328 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.io.Serializable; +import java.time.Clock; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.capability.QqqRecordReader; +import com.kingsrook.qbits.scripting.capability.RecordReader; +import com.kingsrook.qbits.scripting.source.GitSourceImporter; +import com.kingsrook.qbits.scripting.worker.UnconfiguredWorkerProvider; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecutionLimits; +import com.kingsrook.qbits.scripting.worker.WorkerProvider; + + +/******************************************************************************* + ** Host-supplied collaborators for one ScriptingService, set before the QBit is + ** produced. The defaults are deliberately inert: no worker (everything fails + ** closed), no coordinator, no required qualification. Serializable only because + ** QBitConfig is; the collaborators are transient and never serialized. + *******************************************************************************/ +public class ScriptingRuntime implements Serializable +{ + private transient ScriptingTransactionPolicy transactionPolicy; + private transient WorkerProvider worker = new UnconfiguredWorkerProvider(); + private transient BuildCoordinator coordinator; + private transient GitSourceImporter importer; + private transient RecordReader reader = new QqqRecordReader(); + private transient Clock clock = Clock.systemUTC(); + private transient ExecutionLimits limits = ExecutionLimits.hookDefaults(); + private transient WarmReconciliationConfig warmReconciliation; + private transient WarmPublicationSource warmPublicationSource; + private transient volatile WarmRuntimeService warmService; + private String requiredQualificationId; + private boolean requireTestBeforePublish = true; + private int maxRecordsPerHook = 100; + + + + /*************************************************************************** + ** Optional host-owned, bounded context publication inventory. Configure before + ** startup; it supplies metadata only and never runs on an execution path. + ***************************************************************************/ + public ScriptingRuntime withWarmPublicationSource(WarmPublicationSource value) + { + warmPublicationSource = value; + return (this); + } + + /*************************************************************************** + ** Null preserves hook-only reconciliation for existing applications. + ***************************************************************************/ + public WarmPublicationSource getWarmPublicationSource() + { + return (warmPublicationSource); + } + + /*************************************************************************** + ** Explicit metadata authority for automatic per-node cold reconciliation. + ***************************************************************************/ + public ScriptingRuntime withWarmReconciliation(WarmReconciliationConfig value) + { + warmReconciliation = value; + return (this); + } + + /*************************************************************************** + ** Absent policy keeps cache-provider readiness down. + ***************************************************************************/ + public WarmReconciliationConfig getWarmReconciliation() + { + return (warmReconciliation); + } + + /*************************************************************************** + ** Instance-owned native lifecycle object created during metadata production. + ***************************************************************************/ + public WarmRuntimeService getWarmService() + { + return (warmService); + } + + /*************************************************************************** + ** Producer installs exactly one service for this runtime. + ***************************************************************************/ + public void setWarmService(WarmRuntimeService value) + { + if(warmService != null) + { + throw new IllegalStateException("Warm runtime already registered"); + } + warmService = value; + } + + /*************************************************************************** + ** Trusted qualification of every actual owned persistence transaction. + ** Absent policy refuses writes, including execution-history admission. + ***************************************************************************/ + public ScriptingRuntime withTransactionPolicy(ScriptingTransactionPolicy value) + { + transactionPolicy = value; + return (this); + } + + + /*************************************************************************** + ** Current policy, or null when persistence has not been qualified. + ***************************************************************************/ + public ScriptingTransactionPolicy getTransactionPolicy() + { + return (transactionPolicy); + } + + + + /*************************************************************************** + ** The isolated worker. Replace the default only with a provider whose + ** deployment has passed external qualification. + ***************************************************************************/ + public ScriptingRuntime withWorker(WorkerProvider value) + { + worker = value; + return (this); + } + + + + /*************************************************************************** + ** The trusted build coordinator (pinned toolchain and signing key). + ***************************************************************************/ + public ScriptingRuntime withCoordinator(BuildCoordinator value) + { + coordinator = value; + return (this); + } + + + + /*************************************************************************** + ** Optional GitHub snapshot importer. + ***************************************************************************/ + public ScriptingRuntime withImporter(GitSourceImporter value) + { + importer = value; + return (this); + } + + + + /*************************************************************************** + ** Replace the host-side record reader (tests only; default is QQQ). + ***************************************************************************/ + public ScriptingRuntime withReader(RecordReader value) + { + reader = value; + return (this); + } + + + + /*************************************************************************** + ** Replace the clock (tests only). + ***************************************************************************/ + public ScriptingRuntime withClock(Clock value) + { + clock = value; + return (this); + } + + + + /*************************************************************************** + ** Execution limits sent to the worker; must be within the absolute maxima. + ***************************************************************************/ + public ScriptingRuntime withLimits(ExecutionLimits value) + { + limits = value; + return (this); + } + + + + /*************************************************************************** + ** The qualification evidence id the worker must report before anything runs. + ** Blank (the default) means execution is refused outright. + ***************************************************************************/ + public ScriptingRuntime withRequiredQualificationId(String value) + { + requiredQualificationId = value; + return (this); + } + + + + /*************************************************************************** + ** Whether publishing needs a passing recorded test of the same build (default + ** true; turning it off is an explicit host decision). + ***************************************************************************/ + public ScriptingRuntime withRequireTestBeforePublish(boolean value) + { + requireTestBeforePublish = value; + return (this); + } + + + + /*************************************************************************** + ** Largest batch a record hook will process; larger batches are rejected. + ***************************************************************************/ + public ScriptingRuntime withMaxRecordsPerHook(int value) + { + if(value < 1 || value > 1000) + { + throw new IllegalArgumentException("Hook batch size must be between 1 and 1000"); + } + maxRecordsPerHook = value; + return (this); + } + + + + /*************************************************************************** + ** The configured worker. + ***************************************************************************/ + public WorkerProvider getWorker() + { + return (worker); + } + + + + /*************************************************************************** + ** The configured coordinator, or null. + ***************************************************************************/ + public BuildCoordinator getCoordinator() + { + return (coordinator); + } + + + + /*************************************************************************** + ** The configured importer, or null. + ***************************************************************************/ + public GitSourceImporter getImporter() + { + return (importer); + } + + + + /*************************************************************************** + ** The host-side reader. + ***************************************************************************/ + public RecordReader getReader() + { + return (reader); + } + + + + /*************************************************************************** + ** The clock. + ***************************************************************************/ + public Clock getClock() + { + return (clock); + } + + + + /*************************************************************************** + ** The execution limits. + ***************************************************************************/ + public ExecutionLimits getLimits() + { + return (limits); + } + + + + /*************************************************************************** + ** The required qualification evidence id, possibly blank. + ***************************************************************************/ + public String getRequiredQualificationId() + { + return (requiredQualificationId); + } + + + + /*************************************************************************** + ** Whether a passing test is required before publish. + ***************************************************************************/ + public boolean getRequireTestBeforePublish() + { + return (requireTestBeforePublish); + } + + + + /*************************************************************************** + ** The hook batch limit. + ***************************************************************************/ + public int getMaxRecordsPerHook() + { + return (maxRecordsPerHook); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingService.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingService.java new file mode 100644 index 0000000..e4135c6 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingService.java @@ -0,0 +1,998 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingPermissions; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.build.ArtifactSubmission; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.build.BuildOutcome; +import com.kingsrook.qbits.scripting.capability.CapabilityBroker; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHookCustomizer; +import com.kingsrook.qbits.scripting.observability.ScriptTiming; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BuildSummary; +import com.kingsrook.qbits.scripting.service.ScriptingResults.ExecutionSummary; +import com.kingsrook.qbits.scripting.service.ScriptingResults.HookBindingView; +import com.kingsrook.qbits.scripting.service.ScriptingResults.PublishedContext; +import com.kingsrook.qbits.scripting.source.GitCommitRef; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Diagnostic; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; +import com.kingsrook.qqq.backend.core.actions.scripts.ScriptExecutionIsolation; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.scripts.Script; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + + +/******************************************************************************* + ** The lifecycle of a script project, as separate, individually authorized steps: + ** + ** saveDraft (scripting.draft) append an immutable draft revision to core tables + ** check (scripting.check) strict compile via the trusted coordinator + ** test (scripting.test) run a built artifact once, recorded + ** publish (scripting.publish) pin a verified, tested build as a numbered version + ** bind (scripting.bind) attach a published version to a record hook + ** + ** Saving never activates anything; only a binding to a published version runs on + ** live writes, and a binding names its publication, so later drafts, builds and + ** publications never change what it executes. Every step takes an explicit Actor. + *******************************************************************************/ +public class ScriptingService +{ + private final ScriptingRuntime runtime; + private final ScriptingStore store; + private final ScriptSourceRegistry registry; + private final ScriptExecutor executor; + + + + /*************************************************************************** + ** Create a service over host-supplied collaborators. State (open capability + ** leases) is per instance; there is no static registry. + ***************************************************************************/ + public ScriptingService(ScriptingRuntime runtime) + { + this.runtime = runtime; + this.store = new ScriptingStore(runtime); + this.registry = new ScriptSourceRegistry(store); + this.executor = new ScriptExecutor(runtime, store, new CapabilityBroker(runtime.getReader(), runtime.getClock())); + } + + + + /*************************************************************************** + ** Optional cold preparation belongs only to explicit authoring operations. + ***************************************************************************/ + private AuthoringPreparation.Prepared prepareAuthoring(Actor actor, String permission, Integer buildId, Integer publicationId, + ScriptExecutor.ExecutionAuthority authorization) throws ScriptingException + { + return (AuthoringPreparation.prepare(runtime, store, registry, executor, actor, permission, buildId, publicationId, authorization)); + } + + + + /*************************************************************************** + ** Create a server-marked isolated project under the current caller's normal + ** core insert permissions. Never migrates an existing legacy project. + ***************************************************************************/ + public Integer createProject(Actor actor, String name) throws ScriptingException + { + actor.require(ScriptingPermissions.DRAFT); + if(name == null || name.isBlank() || name.length() > 200) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Project name must contain between 1 and 200 characters"); + } + return (store.atomic(List.of(Script.TABLE_NAME), () -> ScriptExecutionIsolation.insertIsolatedScript( + new Script().withName(name.strip()), store.currentTransaction()).getValueInteger("id"))); + } + + + + /*************************************************************************** + ** Save validated files as a new immutable draft revision. Does not build. + ***************************************************************************/ + public Integer saveDraft(Actor actor, Integer scriptId, String commitMessage, List files) throws ScriptingException + { + actor.require(ScriptingPermissions.DRAFT); + authoring().requireIsolated(actor, scriptId); + SourceSnapshot snapshot = SourceSnapshot.of("draft", files); + return (store.atomic(List.of("script", "scriptRevision", "scriptRevisionFile"), () -> registry.saveDraft(scriptId, actor.id(), commitMessage, snapshot))); + } + + + + /*************************************************************************** + ** Public bounded authoring views share the same source registry and ACLs. + ***************************************************************************/ + public ScriptingAuthoring authoring() + { + return (new ScriptingAuthoring(runtime, store, registry)); + } + + + + /*************************************************************************** + ** Editor save with an optimistic base revision. The source schema must also + ** enforce unique scriptId/sequenceNo to arbitrate simultaneous saves. + ***************************************************************************/ + public Integer saveDraft(Actor actor, Integer scriptId, Integer expectedRevisionId, String message, List files) throws ScriptingException + { + actor.require(ScriptingPermissions.DRAFT); + authoring().requireIsolated(actor, scriptId); + SourceSnapshot snapshot = SourceSnapshot.of("draft", files); + return (store.atomic(List.of("script", "scriptRevision", "scriptRevisionFile"), () -> + { + if(!Objects.equals(expectedRevisionId, authoring().latestRevision(scriptId))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Source changed; reload the latest revision before saving"); + } + return (registry.saveDraft(scriptId, actor.id(), message, snapshot)); + })); + } + + + + /*************************************************************************** + ** Import an exact commit through the optional importer and save it as an + ** ordinary draft. Never builds or publishes. + ***************************************************************************/ + public Integer importDraft(Actor actor, Integer scriptId, GitCommitRef ref) throws ScriptingException + { + return (importDraft(actor, scriptId, authoring().project(actor, scriptId).latestRevisionId(), ref)); + } + + + /*************************************************************************** + ** Explicit immutable import; network retrieval occurs before the draft + ** transaction. Current identity, project and base are rechecked afterwards. + ** Provenance and draft commit together; import never builds or publishes. + ***************************************************************************/ + public Integer importDraft(Actor actor, Integer scriptId, Integer expectedRevisionId, GitCommitRef ref) throws ScriptingException + { + actor.require(ScriptingPermissions.DRAFT); + authoring().requireIsolated(actor, scriptId); + if(runtime.getImporter() == null) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "No Git importer is configured"); + } + for(String table : List.of("script", "scriptRevision", "scriptRevisionFile", ScriptingTables.IMPORT)) + { + ScriptingStorageAccess.require(table, true); + } + if(!Objects.equals(expectedRevisionId, authoring().latestRevision(scriptId))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Source changed; reload before importing"); + } + SourceSnapshot imported = runtime.getImporter().importSnapshot(actor, scriptId, ref); + if(imported == null || !imported.verify() || !ref.origin().equals(imported.origin())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Imported snapshot is missing or does not match its hash"); + } + SourceSnapshot snapshot = SourceSnapshot.of(ref.origin(), imported.files()); + return (store.atomic(List.of("script", "scriptRevision", "scriptRevisionFile", ScriptingTables.IMPORT), () -> + { + actor.require(ScriptingPermissions.DRAFT); + authoring().requireIsolated(actor, scriptId); + if(!Objects.equals(expectedRevisionId, authoring().latestRevision(scriptId))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Source changed; reload before importing"); + } + Integer revision = registry.saveDraft(scriptId, actor.id(), "Imported commit " + ref.commitSha(), snapshot); + store.insert(ScriptingTables.IMPORT, Map.of("script_id", scriptId, "script_revision_id", revision, + "repository", ref.repository(), "commit_sha", ref.commitSha(), "subdirectory", ref.subdirectory() == null ? "" : ref.subdirectory(), + "source_sha256", snapshot.sha256(), "imported_by", actor.id(), "created_at", runtime.getClock().instant())); + return (revision); + })); + } + + + + /*************************************************************************** + ** Compile a draft revision strictly. A failed build is recorded with bounded + ** diagnostics and produces no artifact; an unavailable worker records nothing + ** and raises UNAVAILABLE. + ***************************************************************************/ + public BuildSummary check(Actor actor, Integer revisionId, String entryPoint, List declared) throws ScriptingException + { + actor.require(ScriptingPermissions.CHECK); + store.atomic(List.of(ScriptingTables.BUILD, ScriptingTables.ARTIFACT), () -> null); + BuildCoordinator coordinator = requireCoordinator(); + executor.requireQualifiedWorker(); + if(declared == null || declared.size() > CapabilityDeclaration.MAX_DECLARATIONS) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Capability declarations are missing or too many"); + } + + ScriptSourceRegistry.LoadedRevision loaded = registry.load(revisionId); + authoring().requireIsolated(actor, loaded.scriptId()); + BuildOutcome outcome = coordinator.build(runtime.getWorker(), loaded.snapshot(), entryPoint); + + JSONObject manifest = new JSONObject() + .put("scriptId", loaded.scriptId()) + .put("scriptRevisionId", loaded.revisionId()) + .put("sourceSha256", loaded.snapshot().sha256()) + .put("entryPoint", entryPoint) + .put("toolchainId", coordinator.getToolchainId()) + .put("strict", true) + .put("capabilities", new JSONArray(CapabilityDeclaration.listToJson(declared))); + JSONArray diagnostics = new JSONArray(); + for(Diagnostic d : outcome.diagnostics()) + { + diagnostics.put(new JSONObject().put("file", d.file()).put("line", d.line()).put("column", d.column()).put("code", d.code()).put("message", d.message())); + } + + Instant now = runtime.getClock().instant(); + Map build = new HashMap<>(); + build.put("script_id", loaded.scriptId()); + build.put("script_revision_id", loaded.revisionId()); + build.put("source_sha256", loaded.snapshot().sha256()); + build.put("manifest_json", manifest.toString()); + build.put("manifest_attestation", coordinator.attestManifest(manifest.toString())); + build.put("toolchain_id", coordinator.getToolchainId()); + build.put("status", outcome.success() ? "SUCCEEDED" : "FAILED"); + build.put("diagnostics_json", diagnostics.toString()); + build.put("requested_by", actor.id()); + build.put("created_at", now); + return (store.atomic(List.of(ScriptingTables.BUILD, ScriptingTables.ARTIFACT), () -> + { + actor.require(ScriptingPermissions.CHECK); + registry.requireProjectAccess(loaded.scriptId()); + Integer buildId = store.insert(ScriptingTables.BUILD, build); + + if(!outcome.success()) + { + return (new BuildSummary(buildId, false, outcome.diagnostics(), null, null)); + } + + ArtifactSubmission submission = outcome.submission(); + coordinator.verify(submission, loaded.snapshot().sha256()); + int version = maxOf(store.where(ScriptingTables.ARTIFACT, "script_id", loaded.scriptId()), "version") + 1; + + Map artifact = new HashMap<>(); + artifact.put("build_id", buildId); + artifact.put("script_id", loaded.scriptId()); + artifact.put("version", version); + artifact.put("source_sha256", submission.sourceSha256()); + artifact.put("toolchain_id", submission.toolchainId()); + artifact.put("entry_point", submission.entryPoint()); + artifact.put("coordinator_id", submission.coordinatorId()); + artifact.put("bundle", submission.bundle()); + artifact.put("bundle_sha256", submission.bundleSha256()); + artifact.put("size_bytes", submission.bundle().getBytes(StandardCharsets.UTF_8).length); + artifact.put("source_map", submission.sourceMap()); + artifact.put("attestation", submission.attestation()); + artifact.put("created_at", now); + Integer artifactId = store.insert(ScriptingTables.ARTIFACT, artifact); + + return (new BuildSummary(buildId, true, outcome.diagnostics(), artifactId, version)); + })); + } + + + + /*************************************************************************** + ** Run a built artifact once with the capabilities its manifest declared (the + ** tester's own QQQ permissions still apply to every read). A passing run is + ** the evidence publish asks for. + ***************************************************************************/ + public ExecutionSummary test(Actor actor, Integer buildId, HookEvent event, String proposedJson, String priorJson) throws ScriptingException + { + actor.require(ScriptingPermissions.TEST); + QRecord build = store.require(ScriptingTables.BUILD, buildId, "Build"); + registry.requireProjectAccess(build.getValueInteger("script_id")); + try(var prepared = prepareAuthoring(actor, ScriptingPermissions.TEST, buildId, null, + () -> actor.require(ScriptingPermissions.TEST))) + { + return (executor.execute(new ScriptExecutor.Request("TEST", actor.id(), buildId, null, null, event.name(), proposedJson, priorJson, declaredCapabilities(build), null, prepared))); + } + } + + + + /*************************************************************************** + ** Test a context build without publishing it. The explicit grants still + ** cannot exceed its signed capability declaration or current caller rights. + ***************************************************************************/ + public ExecutionSummary testContext(Actor actor, Integer buildId, String inputJson, List grants) throws ScriptingException + { + actor.require(ScriptingPermissions.TEST); + QRecord build = store.require(ScriptingTables.BUILD, buildId, "Build"); + registry.requireProjectAccess(build.getValueInteger("script_id")); + if(grants == null || !CapabilityDeclaration.allWithin(grants, declaredCapabilities(build))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Context grants exceed the build declaration"); + } + try(var prepared = prepareAuthoring(actor, ScriptingPermissions.TEST, buildId, null, + () -> actor.require(ScriptingPermissions.TEST))) + { + return (executor.execute(new ScriptExecutor.Request("TEST_CONTEXT", actor.id(), buildId, null, null, "CONTEXT", inputJson, null, List.copyOf(grants), null, prepared))); + } + } + + + + /*************************************************************************** + ** Pin a build as the next numbered version of its project. Re-verifies the + ** artifact, confirms the source revision still hashes to what was built, and + ** (by default) requires a recorded passing test of this very build. + ***************************************************************************/ + public Integer publish(Actor actor, Integer buildId, String note) throws ScriptingException + { + actor.require(ScriptingPermissions.PUBLISH); + try(var prepared = prepareAuthoring(actor, ScriptingPermissions.PUBLISH, buildId, null, + () -> actor.require(ScriptingPermissions.PUBLISH))) + { + return (store.atomic(List.of(ScriptingTables.BUILD, ScriptingTables.ARTIFACT, ScriptingTables.PUBLICATION, ScriptingTables.EXECUTION, "script", "scriptRevision", "scriptRevisionFile"), () -> publishInTransaction(actor, buildId, note, prepared))); + } + } + + + + /*************************************************************************** + ** Validate and publish under the service-owned transaction. + ***************************************************************************/ + private Integer publishInTransaction(Actor actor, Integer buildId, String note, ScriptExecutor.ExecutionAuthority prepared) throws ScriptingException + { + actor.require(ScriptingPermissions.PUBLISH); + BuildCoordinator coordinator = requireCoordinator(); + QRecord build = store.require(ScriptingTables.BUILD, buildId, "Build"); + registry.requireProjectAccess(build.getValueInteger("script_id")); + QRecord artifact = executor.loadVerifiedArtifact(coordinator, buildId); + + ScriptSourceRegistry.LoadedRevision loaded = registry.load(build.getValueInteger("script_revision_id")); + if(!loaded.snapshot().sha256().equals(build.getValueString("source_sha256"))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "The source revision changed after it was built"); + } + + if(runtime.getRequireTestBeforePublish()) + { + boolean tested = store.where(ScriptingTables.EXECUTION, "build_id", buildId).stream() + .anyMatch(row -> Set.of("TEST", "TEST_CONTEXT").contains(row.getValueString("kind")) && ExecutionSummary.OK.equals(row.getValueString("status"))); + if(!tested) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "A passing test of this build is required before publishing"); + } + } + + Integer scriptId = build.getValueInteger("script_id"); + int version = maxOf(store.where(ScriptingTables.PUBLICATION, "script_id", scriptId), "version") + 1; + Map publication = new HashMap<>(); + publication.put("script_id", scriptId); + publication.put("version", version); + publication.put("build_id", buildId); + publication.put("artifact_id", artifact.getValueInteger("id")); + publication.put("published_by", actor.id()); + publication.put("published_at", runtime.getClock().instant()); + publication.put("note", note == null ? "" : ScriptExecutor.safe(note)); + prepared.check(); + Integer publicationId = store.insert(ScriptingTables.PUBLICATION, publication); + try + { + ScriptExecutionIsolation.activateIsolatedRevision(scriptId, loaded.revisionId(), store.currentTransaction()); + } + catch(QException e) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "The isolated project could not be activated"); + } + if(!loaded.revisionId().equals(store.require("script", scriptId, "Script").getValueInteger("currentScriptRevisionId"))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "The isolated project activation did not persist"); + } + return (publicationId); + } + + + + /*************************************************************************** + ** Bind a published version to a table event. Grants must lie within what the + ** build declared; patch targets must be ordinary editable fields that are + ** not the key or a record-security field. + ***************************************************************************/ + public Integer bind(Actor actor, BindingSpec spec) throws ScriptingException + { + actor.require(ScriptingPermissions.BIND); + QRecord publication = store.require(ScriptingTables.PUBLICATION, spec.publicationId(), "Publication"); + try(var prepared = prepareAuthoring(actor, ScriptingPermissions.BIND, publication.getValueInteger("build_id"), spec.publicationId(), + () -> ScriptingHookAuthoring.authorizeBinding(spec.tableName(), spec.event(), spec.patchableFields()))) + { + return (store.atomic(List.of(ScriptingTables.BINDING, ScriptingTables.PUBLICATION, ScriptingTables.BUILD, ScriptingTables.EXECUTION), () -> bindInTransaction(actor, spec, prepared))); + } + } + + + + /*************************************************************************** + ** Persist one binding; the active-slot database unique key wins races. + ***************************************************************************/ + private Integer bindInTransaction(Actor actor, BindingSpec spec, ScriptExecutor.ExecutionAuthority prepared) throws ScriptingException + { + actor.require(ScriptingPermissions.BIND); + QRecord publication = store.require(ScriptingTables.PUBLICATION, spec.publicationId(), "Publication"); + registry.requireProjectAccess(publication.getValueInteger("script_id")); + QRecord build = store.require(ScriptingTables.BUILD, publication.getValueInteger("build_id"), "Build"); + + if(!CapabilityDeclaration.allWithin(spec.grants(), declaredCapabilities(build))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "A grant exceeds what the published build declared"); + } + QTableMetaData table = QContext.getQInstance().getTable(spec.tableName()); + if(table == null || spec.tableName().startsWith("scripting_")) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "The table cannot carry a script hook"); + } + for(CapabilityDeclaration grant : spec.grants()) + { + QTableMetaData granted = QContext.getQInstance().getTable(grant.tableName()); + if(granted == null || !granted.getFields().keySet().containsAll(grant.fields())) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "A grant names an unknown table or field"); + } + } + if(!spec.event().permitsPatch() && !spec.patchableFields().isEmpty()) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Delete and post-action hooks cannot declare patch fields"); + } + if(spec.event() == HookEvent.PRE_INSERT && !spec.patchableFields().isEmpty()) + { + var customizer = QCodeLoader.getTableCustomizer(table, + TableCustomizers.PRE_INSERT_RECORD.getRole()).orElse(null); + if(customizer instanceof ScriptHookCustomizer hook + && !hook.permitsInsertPatch(new InsertInput().withTableName(spec.tableName()), false)) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "The host insert phase supports veto only; patch fields are not supported"); + } + } + if(spec.event() == HookEvent.PRE_UPDATE && !spec.patchableFields().isEmpty()) + { + var customizer = QCodeLoader.getTableCustomizer(table, + TableCustomizers.PRE_UPDATE_RECORD.getRole()).orElse(null); + if(customizer instanceof ScriptHookCustomizer hook + && !hook.permitsUpdatePatch(new UpdateInput().withTableName(spec.tableName()), false)) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "The host update phase supports veto only; patch fields are not supported"); + } + } + requirePatchable(table, spec.patchableFields()); + ScriptingHookAuthoring.authorizeBinding(spec.tableName(), spec.event(), spec.patchableFields()); + + for(QRecord existing : store.where(ScriptingTables.BINDING, "table_name", spec.tableName())) + { + if(spec.event().name().equals(existing.getValueString("event")) && Integer.valueOf(1).equals(existing.getValueInteger("enabled"))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "An enabled binding already exists for this table and event"); + } + } + + Map binding = new HashMap<>(); + binding.put("name", spec.name()); + binding.put("table_name", spec.tableName()); + binding.put("event", spec.event().name()); + binding.put("publication_id", spec.publicationId()); + binding.put("grants_json", CapabilityDeclaration.listToJson(spec.grants())); + binding.put("patchable_fields", String.join(",", new TreeSet<>(spec.patchableFields()))); + binding.put("enabled", 1); + binding.put("active_slot", spec.tableName() + ":" + spec.event().name()); + binding.put("created_by", actor.id()); + binding.put("created_at", runtime.getClock().instant()); + prepared.check(); + Integer id = store.insert(ScriptingTables.BINDING, binding); + bindingChange(actor, id, publication, spec.event(), "CREATED"); + return (id); + } + + + + /*************************************************************************** + ** Enable or disable a binding (the only mutation of an existing row). + ***************************************************************************/ + public void setBindingEnabled(Actor actor, Integer bindingId, boolean enabled) throws ScriptingException + { + actor.require(ScriptingPermissions.BIND); + AuthoringPreparation.Prepared prepared = new AuthoringPreparation.Prepared(() -> actor.require(ScriptingPermissions.BIND), () -> {}); + if(enabled) + { + QRecord binding = store.require(ScriptingTables.BINDING, bindingId, "Binding"); + Integer publicationId = binding.getValueInteger("publication_id"); + QRecord publication = store.require(ScriptingTables.PUBLICATION, publicationId, "Publication"); + String table = binding.getValueString("table_name"); + String event = binding.getValueString("event"); + String fields = binding.getValueString("patchable_fields"); + String grants = binding.getValueString("grants_json"); + prepared = prepareAuthoring(actor, ScriptingPermissions.BIND, publication.getValueInteger("build_id"), publicationId, () -> + { + QRecord current = store.require(ScriptingTables.BINDING, bindingId, "Binding"); + if(!Objects.equals(publicationId, current.getValueInteger("publication_id")) || !Objects.equals(table, current.getValueString("table_name")) + || !Objects.equals(event, current.getValueString("event")) || !Objects.equals(fields, current.getValueString("patchable_fields")) + || !Objects.equals(grants, current.getValueString("grants_json"))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Binding changed during preparation"); + } + ScriptingHookAuthoring.authorizeBinding(table, HookEvent.valueOf(event), fields == null || fields.isBlank() ? Set.of() : Set.of(fields.split(","))); + }); + } + try(var finalPrepared = prepared) + { + store.atomic(List.of(ScriptingTables.BINDING, ScriptingTables.PUBLICATION, ScriptingTables.EXECUTION), () -> + { + setBindingEnabledInTransaction(actor, bindingId, enabled, finalPrepared); + return (null); + }); + } + } + + + + /*************************************************************************** + ** Atomically change enabled state and its unique active slot. + ***************************************************************************/ + private void setBindingEnabledInTransaction(Actor actor, Integer bindingId, boolean enabled, ScriptExecutor.ExecutionAuthority prepared) throws ScriptingException + { + actor.require(ScriptingPermissions.BIND); + QRecord binding = store.require(ScriptingTables.BINDING, bindingId, "Binding"); + QRecord publication = store.require(ScriptingTables.PUBLICATION, binding.getValueInteger("publication_id"), "Publication"); + registry.requireProjectAccess(publication.getValueInteger("script_id")); + if(enabled) + { + String fields = binding.getValueString("patchable_fields"); + ScriptingHookAuthoring.authorizeBinding(binding.getValueString("table_name"), HookEvent.valueOf(binding.getValueString("event")), + fields == null || fields.isBlank() ? Set.of() : Set.of(fields.split(","))); + Optional active = findBinding(binding.getValueString("table_name"), HookEvent.valueOf(binding.getValueString("event"))); + if(active.isPresent() && !active.get().id().equals(bindingId)) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "An enabled binding already exists for this table and event"); + } + } + if(!enabled) + { + ScriptingHookAuthoring.authorizeDisable(binding.getValueString("table_name")); + } + Map change = new HashMap<>(); + change.put("enabled", enabled ? 1 : 0); + change.put("active_slot", enabled ? binding.getValueString("table_name") + ":" + binding.getValueString("event") : null); + prepared.check(); + store.update(ScriptingTables.BINDING, bindingId, change); + bindingChange(actor, bindingId, publication, HookEvent.valueOf(binding.getValueString("event")), enabled ? "ENABLED" : "DISABLED"); + } + + + + /*************************************************************************** + ** The enabled binding for a table event, if any. Used by the record hook, + ** which runs on behalf of whoever is writing, not as an authoring actor. + ***************************************************************************/ + public Optional findBinding(String tableName, HookEvent event) throws ScriptingException + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.BINDING_LOOKUP)) + { + try + { + for(QRecord row : store.where(ScriptingTables.BINDING, "table_name", tableName)) + { + if(event.name().equals(row.getValueString("event")) && Integer.valueOf(1).equals(row.getValueInteger("enabled"))) + { + String csv = row.getValueString("patchable_fields"); + Set fields = csv == null || csv.isBlank() ? Set.of() : Set.of(csv.split(",")); + return (Optional.of(new HookBindingView(row.getValueInteger("id"), tableName, event, row.getValueInteger("publication_id"), + CapabilityDeclaration.listFromJson(row.getValueString("grants_json")), fields))); + } + } + return (Optional.empty()); + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + } + + + + /*************************************************************************** + ** Run the pinned artifact of a binding for one record. The summary is always + ** returned; anything other than succeeded() must be treated as a refusal. + ***************************************************************************/ + public ExecutionSummary runHook(HookBindingView binding, String actorId, String proposedJson, String priorJson) throws ScriptingException + { + QRecord publication; + try(var timing = ScriptTiming.start(ScriptTiming.Phase.AUTHORIZATION_PUBLICATION)) + { + try + { + if(binding.event().isAfterCommit()) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Durable hooks require verified Jobs delivery"); + } + Actor.requireCurrentIdentity(actorId); + HookBindingView active = findBinding(binding.tableName(), binding.event()).orElseThrow(() -> new ScriptingException(ScriptingException.Kind.DENIED, "The hook binding is disabled")); + if(!active.equals(binding)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "The hook binding changed"); + } + publication = store.require(ScriptingTables.PUBLICATION, binding.publicationId(), "Publication"); + QRecord build = store.require(ScriptingTables.BUILD, publication.getValueInteger("build_id"), "Build"); + if(!CapabilityDeclaration.allWithin(binding.grants(), declaredCapabilities(build))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Hook grants exceed the publication"); + } + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + return (executor.execute(new ScriptExecutor.Request("HOOK", actorId, publication.getValueInteger("build_id"), binding.id(), binding.publicationId(), + binding.event().name(), proposedJson, priorJson, binding.grants()))); + } + + + + /*************************************************************************** + ** Durable observations reuse core Jobs and this instance's source services. + ***************************************************************************/ + public AfterHookService afterHooks() + { + return (new AfterHookService(this, store, executor, runtime)); + } + + + /*************************************************************************** + ** Shared semantic hook administration; no second registry or write pipeline. + ***************************************************************************/ + public ScriptingHookAuthoring hooks() + { + return (new ScriptingHookAuthoring(this, store, registry)); + } + + + + /*************************************************************************** + ** Binding state and its activity receipt commit in the same owned transaction. + ***************************************************************************/ + private void bindingChange(Actor actor, Integer id, QRecord publication, HookEvent event, String status) throws ScriptingException + { + Map row = new HashMap<>(); + row.put("operation_id", UUID.randomUUID().toString()); + row.put("kind", "BINDING_CHANGE"); + row.put("build_id", publication.getValueInteger("build_id")); + row.put("binding_id", id); + row.put("publication_id", publication.getValueInteger("id")); + row.put("actor_id", actor.id()); + row.put("event", event.name()); + row.put("status", status); + row.put("message", "BINDING_" + status); + row.put("created_at", runtime.getClock().instant()); + store.insert(ScriptingTables.EXECUTION, row); + } + + + + /*************************************************************************** + ** Record a native hook context refusal without claiming worker execution or + ** record commit. Trusted hook integration only; no source/record values enter + ** this receipt. Normal execution-history project visibility protects reads. + ***************************************************************************/ + public void recordHookRefusal(HookBindingView binding, String actorId) throws ScriptingException + { + Actor.requireCurrentIdentity(actorId); + HookBindingView active = findBinding(binding.tableName(), binding.event()).orElseThrow(() -> new ScriptingException(ScriptingException.Kind.DENIED, "The hook binding is disabled")); + if(!active.equals(binding)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "The hook binding changed"); + } + QRecord publication = store.require(ScriptingTables.PUBLICATION, binding.publicationId(), "Publication"); + Map row = new HashMap<>(); + row.put("operation_id", UUID.randomUUID().toString()); + row.put("kind", "HOOK_REFUSAL"); + row.put("build_id", publication.getValueInteger("build_id")); + row.put("binding_id", binding.id()); + row.put("publication_id", binding.publicationId()); + row.put("actor_id", actorId); + row.put("event", binding.event().name()); + row.put("status", "DENIED"); + row.put("message", "CONTEXT_REFUSED"); + row.put("created_at", runtime.getClock().instant()); + store.history(row); + } + + + + /*************************************************************************** + ** Validate a pinned published script for a context consumer. This does not + ** accept draft, source revision or bare artifact ids. Returns the immutable + ** artifact id after current caller/project authorization and HMAC checks. + ***************************************************************************/ + public PublishedContext validatePublishedContext(Actor actor, Integer publicationId) throws ScriptingException + { + actor.require(ScriptingPermissions.EXECUTE); + QRecord publication = store.require(ScriptingTables.PUBLICATION, publicationId, "Publication"); + registry.requireProjectAccess(publication.getValueInteger("script_id")); + QRecord artifact = executor.loadVerifiedArtifact(requireCoordinator(), publication.getValueInteger("build_id")); + if(!artifact.getValueInteger("id").equals(publication.getValueInteger("artifact_id")) + || !artifact.getValueInteger("script_id").equals(publication.getValueInteger("script_id"))) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Publication does not match its artifact"); + } + QRecord build = store.require(ScriptingTables.BUILD, publication.getValueInteger("build_id"), "Build"); + return (new PublishedContext(publicationId, artifact.getValueInteger("id"), artifact.getValueString("bundle_sha256"), declaredCapabilities(build))); + } + + /*************************************************************************** + ** Authorized selection metadata, not a bearer grant or executable bundle. + ***************************************************************************/ + public record ContextPublication(Integer publicationId, String label, PublishedContext pin) {} + + /*************************************************************************** + ** Select only an isolated publication whose exact build passed CONTEXT testing. + ** Independent of discovery: current permissions and artifact integrity apply. + ***************************************************************************/ + public ContextPublication selectPublishedContext(Actor actor, Integer publicationId) throws ScriptingException + { + actor.require(ScriptingPermissions.VIEW); + actor.require(ScriptingPermissions.EXECUTE); + if(publicationId == null || publicationId < 1) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Select a published context script"); + } + QRecord publication = store.require(ScriptingTables.PUBLICATION, publicationId, "Publication"); + Integer projectId = publication.getValueInteger("script_id"); + authoring().requireIsolated(actor, projectId); + PublishedContext pin = validatePublishedContext(actor, publicationId); + if(store.query(ScriptingTables.EXECUTION, new QQueryFilter() + .withCriteria("build_id", QCriteriaOperator.EQUALS, publication.getValueInteger("build_id")) + .withCriteria("kind", QCriteriaOperator.EQUALS, "TEST_CONTEXT") + .withCriteria("status", QCriteriaOperator.EQUALS, ExecutionSummary.OK).withLimit(1)).isEmpty()) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "This published build has no successful context test"); + } + try + { + GetInput input = new GetInput(); + input.setTableName("script"); + input.setPrimaryKey(projectId); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + input.setShouldGenerateDisplayValues(false); + QRecord project = new GetAction().execute(input).getRecord(); + if(project == null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Context project is not accessible"); + } + String name = Objects.requireNonNullElse(project.getValueString("name"), "Script"); + String label = name.substring(0, Math.min(160, name.length())) + " — version " + publication.getValueInteger("version") + " (" + publicationId + ")"; + actor.require(ScriptingPermissions.EXECUTE); + return (new ContextPublication(publicationId, label, pin)); + } + catch(ScriptingException failure) + { + throw failure; + } + catch(QException failure) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Context project is not accessible"); + } + } + + /*************************************************************************** + ** Native current-user project search, at most20 projects/100 publications. + ** Narrow by project name; this is bounded discovery, not exhaustive history. + ***************************************************************************/ + public List searchPublishedContexts(Actor actor, String term, int limit) throws ScriptingException + { + actor.require(ScriptingPermissions.VIEW); + actor.require(ScriptingPermissions.EXECUTE); + if(term == null || term.length() > 128 || limit < 1 || limit > 50) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Narrow the context script search"); + } + List projects; + try + { + QueryInput input = new QueryInput("script"); + input.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + input.setShouldGenerateDisplayValues(false); + input.setFilter(new QQueryFilter().withCriteria("executionMode", QCriteriaOperator.EQUALS, "ISOLATED") + .withCriteria("name", QCriteriaOperator.CONTAINS, term).withOrderBy(new QFilterOrderBy("id", false)).withLimit(20)); + projects = new QueryAction().execute(input).getRecords(); + } + catch(QException failure) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Context projects are not accessible"); + } + List choices = new ArrayList<>(); + int examined = 0; + for(QRecord project : projects) + { + registry.requireProjectAccess(project.getValueInteger("id")); + for(QRecord publication : store.query(ScriptingTables.PUBLICATION, new QQueryFilter() + .withCriteria("script_id", QCriteriaOperator.EQUALS, project.getValueInteger("id")) + .withOrderBy(new QFilterOrderBy("id", false)).withLimit(100 - examined))) + { + examined++; + try + { + choices.add(selectPublishedContext(actor, publication.getValueInteger("id"))); + } + catch(ScriptingException failure) + { + if(!Set.of(ScriptingException.Kind.INVALID, ScriptingException.Kind.DENIED, ScriptingException.Kind.NOT_FOUND).contains(failure.getKind())) + { + throw failure; + } + } + if(choices.size() == limit || examined == 100) + { + return (List.copyOf(choices)); + } + } + } + return (List.copyOf(choices)); + } + + + + /*************************************************************************** + ** Run an immutable publication as a bounded context transform. Consumers + ** authorize their source record before supplying pure JSON. Grants must be + ** explicit and within the signed declaration; all reads use current caller + ** permissions. A non-success outcome never yields context for rendering. + ***************************************************************************/ + public ExecutionSummary executePublishedContext(Actor actor, PublishedContext pinned, String inputJson, List grants) throws ScriptingException + { + if(pinned == null || !validatePublishedContext(actor, pinned.publicationId()).equals(pinned)) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Published context does not match its pinned identity"); + } + Integer publicationId = pinned.publicationId(); + QRecord publication = store.require(ScriptingTables.PUBLICATION, publicationId, "Publication"); + QRecord build = store.require(ScriptingTables.BUILD, publication.getValueInteger("build_id"), "Build"); + if(grants == null || !CapabilityDeclaration.allWithin(grants, declaredCapabilities(build))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Context grants exceed the published declaration"); + } + return (executor.execute(new ScriptExecutor.Request("CONTEXT", actor.id(), publication.getValueInteger("build_id"), null, publicationId, + "CONTEXT", inputJson, null, List.copyOf(grants)))); + } + + + + /*************************************************************************** + ** The configured batch limit for hooks. + ***************************************************************************/ + public int getMaxRecordsPerHook() + { + return (runtime.getMaxRecordsPerHook()); + } + + + + /*************************************************************************** + ** Builds and publication need a coordinator; without one nothing is trusted. + ***************************************************************************/ + private BuildCoordinator requireCoordinator() throws ScriptingException + { + if(runtime.getCoordinator() == null) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "No build coordinator is configured"); + } + return (runtime.getCoordinator()); + } + + + + /*************************************************************************** + ** Capabilities a build declared, read back from its stored manifest. + ***************************************************************************/ + List declaredCapabilities(QRecord build) throws ScriptingException + { + ScriptExecutor.verifyManifest(requireCoordinator(), build); + try + { + return (CapabilityDeclaration.listFromJson(new JSONObject(build.getValueString("manifest_json")).getJSONArray("capabilities").toString())); + } + catch(JSONException e) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "The build manifest is unreadable"); + } + } + + + + /*************************************************************************** + ** Patch targets must exist, be editable, ordinary scalar fields, and be + ** neither the primary key nor protected by a record security lock. + ***************************************************************************/ + private static void requirePatchable(QTableMetaData table, Set names) throws ScriptingException + { + List locked = new ArrayList<>(); + for(RecordSecurityLock lock : table.getRecordSecurityLocks() == null ? List.of() : table.getRecordSecurityLocks()) + { + locked.add(lock.getFieldName()); + } + Set allowed = Set.of(QFieldType.STRING, QFieldType.TEXT, QFieldType.INTEGER, QFieldType.LONG, QFieldType.DECIMAL, QFieldType.BOOLEAN); + for(String name : names) + { + QFieldMetaData field = table.getFields().get(name); + if(field == null || name.equals(table.getPrimaryKeyField()) || locked.contains(name) || !field.getIsEditable() || Boolean.TRUE.equals(field.getIsHidden()) || !allowed.contains(field.getType())) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "A patchable field is not allowed: " + name); + } + } + } + + + + /*************************************************************************** + ** Largest integer value of a field across rows, or 0. + ***************************************************************************/ + private static int maxOf(List rows, String field) + { + int max = 0; + for(QRecord row : rows) + { + Integer value = row.getValueInteger(field); + if(value != null && value > max) + { + max = value; + } + } + return (max); + } +} + diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingStorageAccess.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingStorageAccess.java new file mode 100644 index 0000000..c76c261 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingStorageAccess.java @@ -0,0 +1,64 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.context.QContext; + + +/******************************************************************************* + ** Preserve caller credential and client-channel ceilings before privileged + ** owner storage. Domain grants and project/record authorization remain separate. + *******************************************************************************/ +final class ScriptingStorageAccess +{ + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private ScriptingStorageAccess() + { + } + + + /*************************************************************************** + ** Check current scope without requiring unrelated generic table edit roles. + ***************************************************************************/ + static void require(String tableName, boolean mutation) throws ScriptingException + { + var instance = QContext.getQInstance(); + var table = instance == null ? null : instance.getTable(tableName); + if(table == null) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Scripting storage is unavailable"); + } + var rules = PermissionsHelper.getEffectivePermissionRules(table, instance); + if(PermissionsHelper.isDeniedForClientChannel(table, rules) + || (mutation && PermissionsHelper.isWriteDeniedForClientChannel(table, rules)) + || PermissionsHelper.isOutsideCredentialScope(table, mutation ? TablePermissionSubType.EDIT : TablePermissionSubType.READ)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "The current credential or client channel cannot access scripting storage"); + } + } +} + diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingStore.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingStore.java new file mode 100644 index 0000000..9564d7a --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingStore.java @@ -0,0 +1,285 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.observability.ScriptTiming; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; + + +/******************************************************************************* + ** The only code that reads and writes the scripting-owned tables, through the + ** ordinary QQQ Insert/Query/Update actions as the system session. It is + ** deliberately dumb: it has no policy. ScriptingService authorizes and decides; + ** rows it inserts are never updated except the one binding flag. + *******************************************************************************/ +class ScriptingStore +{ + private final ThreadLocal transaction = new ThreadLocal<>(); + private final ScriptingRuntime runtime; + + + /*************************************************************************** + ** Retain the explicit trusted persistence policy alongside this instance. + ***************************************************************************/ + ScriptingStore(ScriptingRuntime runtime) + { + this.runtime = runtime; + } + + + + /*************************************************************************** + ** Explicit transaction for typed core isolation authority operations. + ***************************************************************************/ + QBackendTransaction currentTransaction() + { + return (transaction.get()); + } + + + + /*************************************************************************** + ** Commit one service-owned unit on a single backend. An ambient transaction + ** is refused instead of silently returning an uncommitted publication. + ** Core memory remains an explicit volatile fixture backend, not durability. + ***************************************************************************/ + T atomic(List tables, SystemScope.Work work) throws ScriptingException + { + if(QContext.getQBackendTransaction() != null || transaction.get() != null) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Scripting authoring requires its own transaction boundary"); + } + String backend = QContext.getQInstance().getTable(tables.get(0)).getBackendName(); + for(String table : tables) + { + ScriptingStorageAccess.require(table, true); + if(!backend.equals(QContext.getQInstance().getTable(table).getBackendName())) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Atomic scripting tables must share one backend"); + } + } + QueryInput input = new QueryInput().withTableName(tables.get(0)); + try(QBackendTransaction owned = QBackendTransaction.openFor(input)) + { + if(runtime.getTransactionPolicy() == null) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Scripting persistence has not been qualified"); + } + runtime.getTransactionPolicy().validate(owned, List.copyOf(tables)); + transaction.set(owned); + QContext.setTransaction(owned); + try + { + T result = work.run(); + owned.commit(); + return (result); + } + catch(QException | RuntimeException e) + { + try + { + owned.rollback(); + } + catch(QException rollback) + { + e.addSuppressed(rollback); + } + throw e; + } + finally + { + transaction.remove(); + QContext.clearTransaction(); + } + } + catch(ScriptingException e) + { + throw e; + } + catch(QException e) + { + ScriptingException failure = new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Scripting transaction failed"); + failure.addSuppressed(e); + throw failure; + } + } + + + + /*************************************************************************** + ** Commit history on its own connection, restoring the caller's transaction + ** without committing or rolling it back. Failure prevents trusting output. + ***************************************************************************/ + Integer history(Map values) throws ScriptingException + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.AUDIT_PERSISTENCE)) + { + try + { + QBackendTransaction caller = QContext.getQBackendTransaction(); + QContext.clearTransaction(); + try + { + return (atomic(List.of(ScriptingTables.EXECUTION), () -> insert(ScriptingTables.EXECUTION, values))); + } + finally + { + QContext.setTransaction(caller); + } + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + } + + + + /*************************************************************************** + ** Insert one row and return its generated id. + ***************************************************************************/ + Integer insert(String table, Map values) throws ScriptingException + { + ScriptingStorageAccess.require(table, true); + return (SystemScope.run(() -> + { + QRecord record = new QRecord(); + values.forEach(record::setValue); + List stored = new InsertAction().execute(new InsertInput().withTableName(table).withTransaction(transaction.get()).withRecords(List.of(record))).getRecords(); + if(stored.isEmpty() || !stored.get(0).getErrors().isEmpty() || stored.get(0).getValueInteger("id") == null) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Scripting record was not stored"); + } + return (stored.get(0).getValueInteger("id")); + })); + } + + + + /*************************************************************************** + ** Update fields of one row by id. + ***************************************************************************/ + void update(String table, Integer id, Map values) throws ScriptingException + { + ScriptingStorageAccess.require(table, true); + SystemScope.run(() -> + { + QRecord record = new QRecord().withValue("id", id); + values.forEach(record::setValue); + List stored = new UpdateAction().execute(new UpdateInput().withTableName(table).withTransaction(transaction.get()).withRecords(List.of(record))).getRecords(); + if(stored.isEmpty() || !stored.get(0).getErrors().isEmpty()) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Scripting record was not updated"); + } + return (id); + }); + } + + + + /*************************************************************************** + ** Rows where field equals value. + ***************************************************************************/ + List where(String table, String field, Serializable value) throws ScriptingException + { + ScriptingStorageAccess.require(table, false); + return (SystemScope.run(() -> new QueryAction().execute(new QueryInput().withTableName(table).withTransaction(transaction.get()).withFilter(new QQueryFilter(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)))).getRecords())); + } + + + + /*************************************************************************** + ** Bounded internal views after the caller has checked project authorization. + ***************************************************************************/ + List query(String table, QQueryFilter filter) throws ScriptingException + { + ScriptingStorageAccess.require(table, false); + return (SystemScope.run(() -> new QueryAction().execute(new QueryInput().withTableName(table).withTransaction(transaction.get()) + .withFilter(filter).withShouldGenerateDisplayValues(false)).getRecords())); + } + + + + /*************************************************************************** + ** Project-scoped keyset binding page across every retained publication. Native + ** join filtering avoids materializing an unbounded publication-id list. + ***************************************************************************/ + List bindings(Integer projectId, Integer beforeId, List readableTables) throws ScriptingException + { + ScriptingStorageAccess.require(ScriptingTables.BINDING, false); + ScriptingStorageAccess.require(ScriptingTables.PUBLICATION, false); + if(readableTables.isEmpty()) + { + return (List.of()); + } + QQueryFilter filter = new QQueryFilter() + .withCriteria(ScriptingTables.PUBLICATION + ".script_id", QCriteriaOperator.EQUALS, projectId) + .withCriteria(ScriptingTables.BINDING + ".table_name", QCriteriaOperator.IN, readableTables) + .withOrderBy(new com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy(ScriptingTables.BINDING + ".id", false)).withLimit(101); + if(beforeId != null) + { + filter.withCriteria(ScriptingTables.BINDING + ".id", QCriteriaOperator.LESS_THAN, beforeId); + } + QJoinMetaData join = new QJoinMetaData().withName("scripting_binding_publication").withType(JoinType.MANY_TO_ONE) + .withLeftTable(ScriptingTables.BINDING).withRightTable(ScriptingTables.PUBLICATION).withJoinOn(new JoinOn("publication_id", "id")); + return (SystemScope.run(() -> new QueryAction().execute(new QueryInput(ScriptingTables.BINDING).withTransaction(transaction.get()) + .withQueryJoin(new QueryJoin(join)).withFilter(filter).withShouldGenerateDisplayValues(false)).getRecords())); + } + + + + /*************************************************************************** + ** One row by id, or NOT_FOUND. + ***************************************************************************/ + QRecord require(String table, Integer id, String what) throws ScriptingException + { + List rows = where(table, "id", id); + if(rows.isEmpty()) + { + throw new ScriptingException(ScriptingException.Kind.NOT_FOUND, what + " was not found"); + } + return (rows.get(0)); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingTransactionPolicy.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingTransactionPolicy.java new file mode 100644 index 0000000..66fbfef --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/ScriptingTransactionPolicy.java @@ -0,0 +1,42 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.util.List; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; + + +/******************************************************************************* + ** Trusted host qualification of the exact owned transaction before any writes. + ** The optional RDBMS adapter supplies production database checks. A test policy + ** may model volatile storage, but must never be advertised as durable evidence. + *******************************************************************************/ +@FunctionalInterface +public interface ScriptingTransactionPolicy +{ + /*************************************************************************** + ** Refuse unsupported connections, storage engines or deployment settings. + ** Implementations may inspect but must not commit or mutate the transaction. + ***************************************************************************/ + void validate(QBackendTransaction transaction, List tables) throws ScriptingException; +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/SystemScope.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/SystemScope.java new file mode 100644 index 0000000..f36573d --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/SystemScope.java @@ -0,0 +1,87 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; + + +/******************************************************************************* + ** Runs storage work as the system session and restores the caller's session + ** afterwards. The service has already authorized the human actor explicitly + ** (permission and identity); the system session is how the QBit alone writes its + ** own tables, which ScriptingTablePermission refuses to anyone else. + *******************************************************************************/ +final class SystemScope +{ + /******************************************************************************* + ** Storage work that may fail with a QQQ exception. + *******************************************************************************/ + @FunctionalInterface + interface Work + { + /*************************************************************************** + ** Do the work. + ***************************************************************************/ + T run() throws QException; + } + + + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private SystemScope() + { + } + + + + /*************************************************************************** + ** Run as system; storage failures become a generic UNAVAILABLE error that + ** does not leak backend detail. + ***************************************************************************/ + static T run(Work work) throws ScriptingException + { + QSession original = QContext.getQSession(); + try + { + QContext.setQSession(new QSystemUserSession()); + return (work.run()); + } + catch(ScriptingException e) + { + throw e; + } + catch(QException e) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Scripting storage operation failed"); + } + finally + { + QContext.setQSession(original); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmBindings.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmBindings.java new file mode 100644 index 0000000..f467636 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmBindings.java @@ -0,0 +1,296 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TreeMap; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingPermissions; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.service.ScriptingResults.ExecutionSummary; +import com.kingsrook.qbits.scripting.worker.WarmWorkerMaintenance; +import com.kingsrook.qbits.scripting.worker.WarmWorkerPreparation; +import com.kingsrook.qqq.backend.core.context.CapturedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.serviceidentity.ServiceIdentities; +import com.kingsrook.qqq.backend.core.serviceidentity.ServiceIdentitiesMetaData; +import org.json.JSONObject; + + +/******************************************************************************* + ** A bounded current view of active immutable code pins. Never reads source-table + ** records, creates execution capabilities, or impersonates a binding's author. + *******************************************************************************/ +final class WarmBindings +{ + private final QInstance instance; + private final ScriptingRuntime runtime; + private final WarmReconciliationConfig config; + private final WarmWorkerMaintenance worker; + + /*************************************************************************** + ** Bind one native runtime lifecycle to exact host collaborators. + ***************************************************************************/ + WarmBindings(QInstance instance, ScriptingRuntime runtime, WarmReconciliationConfig config, WarmWorkerMaintenance worker) + { + this.instance = instance; + this.runtime = runtime; + this.config = config; + this.worker = worker; + } + + /*************************************************************************** + ** Scoped current identity, minted without any calling user's backend variants. + ** Temporary contexts restore transaction, session and action scope on failure. + ***************************************************************************/ + void authorized(Work work) throws QException + { + QContext.withTemporaryContext(new CapturedContext(instance, null), () -> + { + QSession session = ServiceIdentities.session(instance, config.principal()); + QContext.setQSession(session); + WarmPublicationSource source = runtime.getWarmPublicationSource(); + Map authority = authority(session); + Map declaration = new JSONObject(ServiceIdentitiesMetaData.getPrincipal(instance, config.principal())).toMap(); + Check check = () -> + { + if(QContext.getQInstance() != instance || QContext.getQSession() != session || QContext.getQBackendTransaction() != null + || runtime.getWorker() != worker || runtime.getWarmReconciliation() != config || runtime.getWarmPublicationSource() != source + || !ServiceIdentities.isDeclared(instance, config.principal()) || !ServiceIdentities.isMintedServiceSession(session) + || !config.principal().equals(ServiceIdentities.principalOf(session)) + || !session.hasPermission(ScriptingPermissions.WARM) || !session.hasPermission(ScriptingPermissions.VIEW) + || !authority.equals(authority(session)) + || !declaration.equals(new JSONObject(ServiceIdentitiesMetaData.getPrincipal(instance, config.principal())).toMap())) + { + throw refused(); + } + }; + check.run(); + work.run(check); + check.run(); + }); + } + + /*************************************************************************** + ** Copy only policy fields, not access tokens, session values or credentials. + ***************************************************************************/ + private static Map authority(QSession session) + { + return (new JSONObject().put("permissions", session.getPermissions()).put("keys", session.getSecurityKeyValues()) + .put("variants", session.getBackendVariants()).put("channel", session.getClientChannel()) + .put("actor", session.getUser() == null ? "" : session.getUser().getIdReference()) + .put("scope", session.getCredentialScope() == null ? JSONObject.NULL : new JSONObject(session.getCredentialScope())).toMap()); + } + + /*************************************************************************** + ** Every scoped binding is counted before project visibility is tested. Hidden + ** required code causes refusal, never a deceptively ready partial inventory. + ***************************************************************************/ + Snapshot read(Check check, long stop) throws QException + { + requireTime(stop); + check.run(); + for(String target : config.targetTables()) + { + if(instance.getTable(target) == null) + { + throw refused(); + } + } + ScriptingStore store = new ScriptingStore(runtime); + ScriptExecutor executor = new ScriptExecutor(runtime, store, null); + executor.requireQualifiedWorker(); + if(runtime.getCoordinator() == null) + { + throw refused(); + } + List bindings = store.query(ScriptingTables.BINDING, new QQueryFilter() + .withCriteria(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, 1)) + .withCriteria(new QFilterCriteria("table_name", QCriteriaOperator.IN, new ArrayList<>(config.targetTables()))) + .withOrderBy(new QFilterOrderBy("id")).withLimit(config.maxBindings() + 1)); + check.run(); + if(bindings.size() > config.maxBindings()) + { + throw refused(); + } + List required = new ArrayList<>(); + for(QRecord binding : bindings) + { + required.add(new RequiredPublication(binding, binding.getValueInteger("publication_id"), null)); + } + WarmPublicationSource source = runtime.getWarmPublicationSource(); + if(source != null) + { + int remaining = config.maxBindings() - bindings.size(); + List supplied = source.publications(config.targetTables(), remaining); + check.run(); + requireTime(stop); + if(supplied == null || supplied.size() > remaining) + { + throw refused(); + } + Map pins = new TreeMap<>(); + for(WarmPublicationSource.Pin pin : supplied) + { + if(pin == null || pins.putIfAbsent(pin.publicationId(), pin) != null || pins.size() > remaining) + { + throw refused(); + } + } + check.run(); + for(WarmPublicationSource.Pin pin : pins.values()) + { + required.add(new RequiredPublication(null, pin.publicationId(), pin)); + } + } + List identities = new ArrayList<>(); + Map artifacts = new LinkedHashMap<>(); + long bytes = 0; + for(RequiredPublication pin : required) + { + requireTime(stop); + QRecord publication = store.require(ScriptingTables.PUBLICATION, pin.publicationId(), "Publication"); + check.run(); + Integer buildId = publication.getValueInteger("build_id"); + QRecord build = store.require(ScriptingTables.BUILD, buildId, "Build"); + check.run(); + new ScriptSourceRegistry(store).requireProjectAccess(build.getValueInteger("script_id")); + check.run(); + if(pin.context() != null) + { + if(!"ISOLATED".equals(store.require("script", build.getValueInteger("script_id"), "Script").getValueString("executionMode"))) + { + throw refused(); + } + check.run(); + if(store.query(ScriptingTables.EXECUTION, new QQueryFilter() + .withCriteria("build_id", QCriteriaOperator.EQUALS, buildId) + .withCriteria("kind", QCriteriaOperator.EQUALS, "TEST_CONTEXT") + .withCriteria("status", QCriteriaOperator.EQUALS, ExecutionSummary.OK).withLimit(1)).isEmpty()) + { + throw refused(); + } + check.run(); + } + QRecord artifact = executor.loadVerifiedArtifact(runtime.getCoordinator(), buildId); + check.run(); + if(!Objects.equals(publication.getValueInteger("script_id"), build.getValueInteger("script_id")) + || !Objects.equals(publication.getValueInteger("artifact_id"), artifact.getValueInteger("id"))) + { + throw refused(); + } + Integer artifactId = artifact.getValueInteger("id"); + if(pin.context() != null && (!Objects.equals(pin.context().artifactId(), artifactId) + || !Objects.equals(pin.context().bundleSha256(), artifact.getValueString("bundle_sha256")))) + { + throw refused(); + } + WarmWorkerMaintenance.CodeIdentity codeIdentity = new WarmWorkerMaintenance.CodeIdentity(artifact.getValueString("bundle_sha256"), + artifact.getValueString("entry_point"), artifact.getValueString("toolchain_id")); + if(!artifacts.containsKey(codeIdentity)) + { + WarmWorkerPreparation.Artifact code = new WarmWorkerPreparation.Artifact(artifact.getValueString("bundle_sha256"), + artifact.getValueString("entry_point"), artifact.getValueString("bundle"), artifact.getValueString("toolchain_id")); + bytes += code.bundle().getBytes(StandardCharsets.UTF_8).length; + if(bytes > config.maxCodeBytes()) + { + throw refused(); + } + artifacts.put(codeIdentity, code); + } + identities.add(BuildCoordinator.sha256Hex(new JSONObject().put("binding", pin.binding() == null ? JSONObject.NULL : pin.binding().getValues()) + .put("context", pin.context() != null).put("publication", publication.getValues()) + .put("build", build.getValues()).put("artifactId", artifactId).put("hash", artifact.getValueString("bundle_sha256")) + .put("entry", artifact.getValueString("entry_point")).put("toolchain", artifact.getValueString("toolchain_id")).toString())); + } + requireTime(stop); + check.run(); + return (new Snapshot(List.copyOf(identities), Map.copyOf(artifacts))); + } + + /*************************************************************************** + ** Finite cooperative work bound; native database timeouts remain prerequisite. + ***************************************************************************/ + static void requireTime(long stop) throws ScriptingException + { + if(Thread.currentThread().isInterrupted() || System.nanoTime() >= stop) + { + throw refused(); + } + } + + /*************************************************************************** + ** Public health exposes no source, identifiers, authority or failure detail. + ***************************************************************************/ + private static ScriptingException refused() + { + return (new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Warm scripting metadata is not ready")); + } + + /*************************************************************************** + ** Code lives only for a bounded pass; the runtime stores identity digests only. + ***************************************************************************/ + record Snapshot(List identities, Map artifacts) {} + + /*************************************************************************** + ** One required hook row or externally inventoried context publication. + ***************************************************************************/ + private record RequiredPublication(QRecord binding, Integer publicationId, WarmPublicationSource.Pin context) {} + + /*************************************************************************** + ** Internal context guard after each host callback or metadata operation. + ***************************************************************************/ + @FunctionalInterface + interface Check + { + /*********************************************************************** + ** Check exact current authority. + ***********************************************************************/ + void run() throws ScriptingException; + } + + /*************************************************************************** + ** One freshly authenticated metadata unit. + ***************************************************************************/ + @FunctionalInterface + interface Work + { + /*********************************************************************** + ** Perform code-only work under the supplied current authority guard. + ***********************************************************************/ + void run(Check check) throws QException; + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmPublicationSource.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmPublicationSource.java new file mode 100644 index 0000000..533ae58 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmPublicationSource.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.util.List; +import java.util.Set; +import com.kingsrook.qqq.backend.core.exceptions.QException; + + +/******************************************************************************* + ** Optional metadata-only inventory of required immutable CONTEXT publications. + ** Called under the fresh declared warm service principal, never an invoking user. + ** Implementations must enforce current owner metadata visibility and scope, and + ** return all required positive, distinct publication pins or throw on overflow, + ** inaccessible metadata or uncertainty. Never truncate to make readiness pass. + ** No record values, credentials, executable bytes or execution authority enter + ** this SPI. The scripting owner independently validates every returned pin. + *******************************************************************************/ +@FunctionalInterface +public interface WarmPublicationSource +{ + /*************************************************************************** + ** Return a non-null bounded inventory for these explicit existing source + ** tables. Limit is the remaining combined hook/publication budget, possibly + ** zero. An empty result authoritatively means no additional required pins. + ** Caller copies/sorts the result; callback must not alter QContext/authority. + ***************************************************************************/ + List publications(Set targetTables, int limit) throws QException; + + /*************************************************************************** + ** Exact immutable consumer reference. This is metadata, not an authorization + ** token; the foundation independently checks publication, code and signature. + ***************************************************************************/ + record Pin(Integer publicationId, Integer artifactId, String bundleSha256) + { + /*********************************************************************** + ** Reject coercion, absent identities and noncanonical hashes at intake. + ***********************************************************************/ + public Pin + { + if(publicationId == null || publicationId < 1 || artifactId == null || artifactId < 1 + || bundleSha256 == null || !bundleSha256.matches("[0-9a-f]{64}")) + { + throw new IllegalArgumentException("Invalid warm publication pin"); + } + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmReconciliationConfig.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmReconciliationConfig.java new file mode 100644 index 0000000..302db89 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmReconciliationConfig.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.time.Duration; +import java.util.Set; + + +/******************************************************************************* + ** Explicit host-only metadata authority and finite per-node maintenance limits. + ** JDBC connect/socket/query deadlines must also be configured by the host. + *******************************************************************************/ +public record WarmReconciliationConfig(String principal, Set targetTables, int maxBindings, + long maxCodeBytes, Duration interval, Duration passBudget, Duration maxAge) +{ + /*************************************************************************** + ** No implicit all-table scope or unbounded timer is admitted. + ***************************************************************************/ + public WarmReconciliationConfig + { + if(principal == null || !principal.matches("[A-Za-z0-9_.-]{1,100}") || targetTables == null || targetTables.isEmpty() + || targetTables.size() > 64 || targetTables.stream().anyMatch(name -> name == null || !name.matches("[A-Za-z][A-Za-z0-9_]{0,99}") || name.startsWith("scripting_")) + || maxBindings < 1 || maxBindings > 1000 || maxCodeBytes < 1 || maxCodeBytes > 64L * 1024 * 1024 + || !bounded(interval, 1, 300) || !bounded(passBudget, 1, 120) || !bounded(maxAge, 1, 600) + || maxAge.compareTo(interval.plus(passBudget)) < 0) + { + throw new IllegalArgumentException("Invalid warm reconciliation policy"); + } + targetTables = Set.copyOf(targetTables); + } + + /*************************************************************************** + ** Idle sessions confirmed by a completed exchange inside this window need no + ** new wire heartbeat. It is at most a quarter of readiness age and always + ** leaves one full interval plus pass budget before an acknowledgment can age + ** out, so a skipped slot can still be refreshed before readiness is lost. + ***************************************************************************/ + public Duration heartbeatSkipWindow() + { + Duration headroom = maxAge.minus(interval).minus(passBudget); + Duration quarter = maxAge.dividedBy(4); + Duration window = headroom.compareTo(quarter) < 0 ? headroom : quarter; + return (window.isNegative() ? Duration.ZERO : window); + } + + /*************************************************************************** + ** Avoid overflow and zero-delay scheduling before converting to nanoseconds. + ***************************************************************************/ + private static boolean bounded(Duration value, int minimum, int maximum) + { + return (value != null && value.compareTo(Duration.ofSeconds(minimum)) >= 0 && value.compareTo(Duration.ofSeconds(maximum)) <= 0); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmRuntimeService.java b/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmRuntimeService.java new file mode 100644 index 0000000..bd55e12 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/service/WarmRuntimeService.java @@ -0,0 +1,312 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingQBitConfig; +import com.kingsrook.qbits.scripting.worker.WarmWorkerMaintenance; +import com.kingsrook.qbits.scripting.worker.WarmWorkerPreparation; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCheck; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; +import com.kingsrook.qqq.backend.core.instances.QRuntimeServiceInterface; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; + + +/******************************************************************************* + ** One native per-node cold lifecycle. The health check reads current metadata + ** under a freshly declared service identity; it never prepares or executes code. + ** All transport happens off record-save threads. Host JDBC timeouts are required. + *******************************************************************************/ +public final class WarmRuntimeService implements QRuntimeServiceInterface, HealthCheck +{ + private final AtomicBoolean reconciling = new AtomicBoolean(); + private final AtomicBoolean checking = new AtomicBoolean(); + private volatile boolean stopped = true; + private volatile List pins; + private volatile long confirmedAt; + private QInstance instance; + private ScriptingRuntime runtime; + private WarmReconciliationConfig policy; + private WarmWorkerMaintenance worker; + private WarmBindings bindings; + private ScheduledThreadPoolExecutor scheduler; + private ScheduledFuture future; + + /*************************************************************************** + ** Stable native lifecycle identity; no globally registered service state. + ***************************************************************************/ + @Override + public String getName() + { + return ("scripting-warm-runtime"); + } + + /*************************************************************************** + ** Register before scheduling so readiness is down until a complete cold pass. + ** Missing explicit policy remains inert and down rather than warming all code. + ***************************************************************************/ + @Override + public synchronized void start(QInstance value) throws QException + { + if(runtime != null) + { + throw new QException("Warm scripting lifecycle already started"); + } + instance = value; + ScriptingQBitConfig config = ScriptingQBitConfig.configFor(instance); + if(config == null || !(config.getRuntime().getWorker() instanceof WarmWorkerMaintenance maintenance)) + { + throw new QException("Warm scripting lifecycle has no maintenance provider"); + } + runtime = config.getRuntime(); + worker = maintenance; + policy = runtime.getWarmReconciliation(); + runtime.setWarmService(this); + if(policy == null) + { + return; + } + bindings = new WarmBindings(instance, runtime, policy, worker); + stopped = false; + try + { + scheduler = new ScheduledThreadPoolExecutor(1, runnable -> Thread.ofPlatform().daemon().name("qqq-scripting-warm").unstarted(runnable)); + scheduler.setRemoveOnCancelPolicy(true); + scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + future = scheduler.scheduleWithFixedDelay(this::reconcile, 0, policy.interval().toMillis(), TimeUnit.MILLISECONDS); + } + catch(RuntimeException failure) + { + stop(); + throw new QException("Cannot start warm scripting lifecycle", failure); + } + } + + /*************************************************************************** + ** Coalesced bounded pass. No untrusted code, capabilities, record values or + ** initiating user grants are admitted. Failures invalidate readiness only. + ***************************************************************************/ + void reconcile() + { + if(stopped || !reconciling.compareAndSet(false, true)) + { + return; + } + long stop = System.nanoTime() + policy.passBudget().toNanos(); + try + { + bindings.authorized(check -> + { + WarmBindings.Snapshot before = bindings.read(check, stop); + worker.heartbeat(remaining(stop), policy.heartbeatSkipWindow()); + check.run(); + requireSame(before, bindings.read(check, stop)); + worker.retireUnused(codeIdentities(before), remaining(stop)); + check.run(); + requireSame(before, bindings.read(check, stop)); + for(WarmWorkerPreparation.Artifact artifact : before.artifacts().values()) + { + boolean ready = prepared(artifact); + check.run(); + if(!ready) + { + requireSame(before, bindings.read(check, stop)); + worker.prepareVerified(artifact, remaining(stop)); + check.run(); + requireSame(before, bindings.read(check, stop)); + } + } + WarmBindings.Snapshot after = bindings.read(check, stop); + requireSame(before, after); + if(!allPrepared(after)) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Required code is not warm"); + } + check.run(); + requireSame(after, bindings.read(check, stop)); + check.run(); + WarmBindings.requireTime(stop); + if(!stopped) + { + pins = after.identities(); + confirmedAt = System.nanoTime(); + } + }); + } + catch(QException | RuntimeException failure) + { + pins = null; + confirmedAt = 0; + } + finally + { + reconciling.set(false); + } + } + + /*************************************************************************** + ** Native readiness delegates to the exact instance-owned lifecycle object. + ** Current pins are re-read; a stale cached boolean cannot mask new bindings. + ***************************************************************************/ + @Override + public HealthCheckOutcome run(HealthCheckInput input) + { + if(runtime == null) + { + ScriptingQBitConfig config = ScriptingQBitConfig.configFor(input.getQInstance()); + WarmRuntimeService active = config == null ? null : config.getRuntime().getWarmService(); + return (active == null ? down() : active.run(input)); + } + if(stopped || input.getQInstance() != instance || pins == null || confirmedAt == 0 + || System.nanoTime() - confirmedAt > policy.maxAge().toNanos() || !checking.compareAndSet(false, true)) + { + return (down()); + } + try + { + Duration budget = input.getDeadline() == null ? policy.passBudget() : Duration.between(Instant.now(), input.getDeadline()); + if(budget.isNegative() || budget.isZero()) + { + return (down()); + } + long stop = System.nanoTime() + Math.min(budget.toNanos(), policy.passBudget().toNanos()); + bindings.authorized(check -> + { + WarmBindings.Snapshot current = bindings.read(check, stop); + if(stopped || !current.identities().equals(pins) || System.nanoTime() - confirmedAt > policy.maxAge().toNanos()) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Warm metadata changed"); + } + if(!allPrepared(current)) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Required cache epoch is not ready"); + } + check.run(); + requireSame(current, bindings.read(check, stop)); + check.run(); + WarmBindings.requireTime(stop); + }); + return (stopped ? down() : HealthCheckOutcome.up("Required scripting code is warm")); + } + catch(QException | RuntimeException failure) + { + return (down()); + } + finally + { + checking.set(false); + } + } + + /*************************************************************************** + ** Local-only readiness does not open sockets, prepare, or allocate guests. + ***************************************************************************/ + private boolean prepared(WarmWorkerPreparation.Artifact artifact) + { + return (worker.isPrepared(artifact.sha256(), artifact.entryPoint(), artifact.toolchainId(), policy.maxAge())); + } + + /*************************************************************************** + ** Atomically check all code identities without serializing bundle bytes. + ***************************************************************************/ + private boolean allPrepared(WarmBindings.Snapshot snapshot) + { + return (worker.arePrepared(codeIdentities(snapshot), policy.maxAge())); + } + + /*************************************************************************** + ** Bounded code identities only, shared by cold retirement and local readiness. + ***************************************************************************/ + private static List codeIdentities(WarmBindings.Snapshot snapshot) + { + return (snapshot.artifacts().values().stream() + .map(artifact -> new WarmWorkerMaintenance.CodeIdentity(artifact.sha256(), artifact.entryPoint(), artifact.toolchainId())).toList()); + } + + /*************************************************************************** + ** Reject pin changes after every external callback before accepting readiness. + ***************************************************************************/ + private static void requireSame(WarmBindings.Snapshot left, WarmBindings.Snapshot right) throws ScriptingException + { + if(!left.equals(right)) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Warm binding changed during preparation"); + } + } + + /*************************************************************************** + ** One remaining deadline covers every connection and artifact in this pass. + ***************************************************************************/ + private static Duration remaining(long stop) throws ScriptingException + { + WarmBindings.requireTime(stop); + return (Duration.ofNanos(Math.max(1, stop - System.nanoTime()))); + } + + /*************************************************************************** + ** Generic outcome excludes source IDs, metadata scope and transport details. + ***************************************************************************/ + private static HealthCheckOutcome down() + { + return (HealthCheckOutcome.down("Required scripting code is not ready")); + } + + /*************************************************************************** + ** Cancel only this per-node future; never wait indefinitely for a JDBC driver. + ***************************************************************************/ + @Override + public synchronized void stop() + { + stopped = true; + pins = null; + confirmedAt = 0; + if(future != null) + { + future.cancel(true); + } + if(scheduler != null) + { + scheduler.shutdownNow(); + } + } + + /*************************************************************************** + ** Native launcher calls this after all producer services have stopped. + ***************************************************************************/ + @Override + public void afterApplicationStop() + { + if(worker != null) + { + worker.close(); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/source/GitCommitRef.java b/base/src/main/java/com/kingsrook/qbits/scripting/source/GitCommitRef.java new file mode 100644 index 0000000..db47489 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/source/GitCommitRef.java @@ -0,0 +1,68 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.source; + + +import java.util.regex.Pattern; + + +/******************************************************************************* + ** An exact Git commit to import from. Branch and tag names are deliberately not + ** representable: a moving ref would make a build non-reproducible. Resolving a + ** branch to a commit is the operator's explicit, audited step, outside this type. + *******************************************************************************/ +public record GitCommitRef(String repository, String commitSha, String subdirectory) +{ + private static final Pattern REPOSITORY = Pattern.compile("[A-Za-z0-9_.-]{1,100}/[A-Za-z0-9_.-]{1,100}"); + private static final Pattern SHA = Pattern.compile("[0-9a-f]{40}"); + private static final Pattern SUBDIR = Pattern.compile("([A-Za-z0-9_][A-Za-z0-9_.-]*)(/[A-Za-z0-9_][A-Za-z0-9_.-]*)*"); + + + + /*************************************************************************** + ** Validate shape only; the importer verifies the commit really exists. + ***************************************************************************/ + public GitCommitRef + { + if(repository == null || !REPOSITORY.matcher(repository).matches() || repository.contains("..")) + { + throw new IllegalArgumentException("Repository must be owner/name"); + } + if(commitSha == null || !SHA.matcher(commitSha).matches()) + { + throw new IllegalArgumentException("A full 40-character lower-case commit SHA is required"); + } + if(subdirectory != null && (subdirectory.length() > 200 || !SUBDIR.matcher(subdirectory).matches() || subdirectory.contains(".."))) + { + throw new IllegalArgumentException("Subdirectory must be a plain relative path"); + } + } + + + + /*************************************************************************** + ** Provenance label used as the snapshot origin. + ***************************************************************************/ + public String origin() + { + return ("git:" + repository + "@" + commitSha + (subdirectory == null ? "" : ":" + subdirectory)); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/source/GitSourceImporter.java b/base/src/main/java/com/kingsrook/qbits/scripting/source/GitSourceImporter.java new file mode 100644 index 0000000..9b5491e --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/source/GitSourceImporter.java @@ -0,0 +1,45 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.source; + + +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.service.Actor; + + +/******************************************************************************* + ** Optional Git-backed authoring port. The separate github artifact supplies an + ** adapter; ordinary IDE drafts do not require a network source provider. + ** + ** An implementation must: read the tree of exactly the given commit using + ** credentials held by the coordinator (never the browser, never a worker); + ** return only .ts files through SourceSnapshot.of so path, count and size policy + ** is enforced; reject symlinks, submodules and archives; and never run npm, + ** build scripts, hooks or repository CI. The result is an immutable snapshot + ** that is then saved as an ordinary draft; importing never builds or publishes. + *******************************************************************************/ +public interface GitSourceImporter +{ + /*************************************************************************** + ** Import the exact commit as an immutable snapshot, or fail. + ***************************************************************************/ + SourceSnapshot importSnapshot(Actor actor, Integer scriptId, GitCommitRef ref) throws ScriptingException; +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/source/SourceFile.java b/base/src/main/java/com/kingsrook/qbits/scripting/source/SourceFile.java new file mode 100644 index 0000000..b2466bf --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/source/SourceFile.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.source; + + +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; +import com.kingsrook.qbits.scripting.ScriptingException; + + +/******************************************************************************* + ** One TypeScript source file. Only .ts files are accepted: tsconfig, package + ** manifests, build scripts and plugins are server-controlled policy, never + ** project input. The path is a relative, normalized, forward-slash name that is + ** only ever used as a label for the compiler, never as a filesystem path. + *******************************************************************************/ +public record SourceFile(String path, String content) +{ + public static final int MAX_PATH_LENGTH = 100; // core ScriptRevisionFile.fileName is limited to 100 + public static final int MAX_CONTENT_BYTES = 256 * 1024; + + private static final Pattern SEGMENT = Pattern.compile("[A-Za-z0-9_][A-Za-z0-9_.-]*"); + + + + /*************************************************************************** + ** Validate on construction so an invalid file can never exist. + ***************************************************************************/ + public SourceFile + { + if(path == null || content == null) + { + throw new IllegalArgumentException("Source file path and content are required"); + } + try + { + validatePath(path); + } + catch(ScriptingException e) + { + throw new IllegalArgumentException(e.getMessage(), e); + } + if(content.getBytes(StandardCharsets.UTF_8).length > MAX_CONTENT_BYTES) + { + throw new IllegalArgumentException("Source file exceeds " + MAX_CONTENT_BYTES + " bytes"); + } + if(content.indexOf('\0') >= 0) + { + throw new IllegalArgumentException("Source file contains a NUL character"); + } + } + + + + /*************************************************************************** + ** Reject anything that is not a plain relative .ts path. + ***************************************************************************/ + public static void validatePath(String path) throws ScriptingException + { + if(path.isEmpty() || path.length() > MAX_PATH_LENGTH || !path.endsWith(".ts")) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Source path must be a relative .ts name of at most " + MAX_PATH_LENGTH + " characters"); + } + for(String segment : path.split("/", -1)) + { + if(!SEGMENT.matcher(segment).matches() || segment.equals("..") || segment.contains("..")) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Source path contains an unsupported segment"); + } + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/source/SourceSnapshot.java b/base/src/main/java/com/kingsrook/qbits/scripting/source/SourceSnapshot.java new file mode 100644 index 0000000..577c5fa --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/source/SourceSnapshot.java @@ -0,0 +1,152 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.source; + + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingException; + + +/******************************************************************************* + ** An immutable, hashed set of source files: the unit that is built. The hash + ** covers every path and byte in a canonical order, so any later change to a + ** draft, however small, produces a different snapshot identity. Origin is a + ** provenance label only (for example "draft:42" or "git:owner/repo@sha"). + *******************************************************************************/ +public record SourceSnapshot(String origin, List files, String sha256) +{ + public static final int MAX_FILES = 200; + public static final int MAX_TOTAL_BYTES = 2 * 1024 * 1024; + + + + /*************************************************************************** + ** Defensive copy so the record is deeply immutable. + ***************************************************************************/ + public SourceSnapshot + { + files = List.copyOf(files); + } + + + + /*************************************************************************** + ** Build a snapshot: enforce count/size/duplicate rules, order files by path + ** and compute the content hash. + ***************************************************************************/ + public static SourceSnapshot of(String origin, List input) throws ScriptingException + { + if(input == null || input.isEmpty() || input.size() > MAX_FILES) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "A snapshot needs between 1 and " + MAX_FILES + " files"); + } + + long total = 0; + Set seen = new HashSet<>(); + for(SourceFile file : input) + { + if(!seen.add(file.path().toLowerCase(Locale.ROOT))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Duplicate source path (case-insensitive)"); + } + total += file.content().getBytes(StandardCharsets.UTF_8).length; + } + if(total > MAX_TOTAL_BYTES) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Snapshot exceeds " + MAX_TOTAL_BYTES + " bytes"); + } + + List sorted = new ArrayList<>(input); + sorted.sort(Comparator.comparing(SourceFile::path)); + return (new SourceSnapshot(origin, sorted, hash(sorted))); + } + + + + /*************************************************************************** + ** True when the file list still hashes to the recorded identity. + ***************************************************************************/ + public boolean verify() + { + return (hash(files).equals(sha256)); + } + + + + /*************************************************************************** + ** Whether a file with this path exists. + ***************************************************************************/ + public boolean hasFile(String path) + { + return (files.stream().anyMatch(f -> f.path().equals(path))); + } + + + + /*************************************************************************** + ** Length-prefixed canonical encoding, so path/content boundaries cannot be + ** shifted to collide two different snapshots. + ***************************************************************************/ + private static String hash(List sorted) + { + try + { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + for(SourceFile file : sorted) + { + byte[] path = file.path().getBytes(StandardCharsets.UTF_8); + byte[] content = file.content().getBytes(StandardCharsets.UTF_8); + digest.update(ByteBuffer.allocate(8).putInt(path.length).putInt(content.length).array()); + digest.update(path); + digest.update(content); + } + return (hex(digest.digest())); + } + catch(NoSuchAlgorithmException e) + { + throw new IllegalStateException("SHA-256 is required by the Java platform", e); + } + } + + + + /*************************************************************************** + ** Lower-case hex encoding. + ***************************************************************************/ + public static String hex(byte[] bytes) + { + StringBuilder builder = new StringBuilder(bytes.length * 2); + for(byte b : bytes) + { + builder.append(String.format("%02x", b)); + } + return (builder.toString()); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/CoordinatorWorkerProvider.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/CoordinatorWorkerProvider.java new file mode 100644 index 0000000..4d0684a --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/CoordinatorWorkerProvider.java @@ -0,0 +1,362 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import com.kingsrook.qbits.scripting.ScriptingException; +import org.json.JSONObject; + + +/******************************************************************************* + ** Trusted host transport to the disposable-worker coordinator. It launches a + ** fixed operator-installed executable and CLI with no shell or user arguments. + ** The coordinator independently verifies runsc, image/evidence and lifecycle + ** journaling; this adapter does not certify a deployment or execute user code. + *******************************************************************************/ +public final class CoordinatorWorkerProvider implements WorkerProvider +{ + private static final int MAX_FRAME = 4 * 1024 * 1024; + private static final int MAX_TOTAL_OUTPUT = 8 * 1024 * 1024; + private static final int MAX_STDERR = 64 * 1024; + + private final Configuration config; + + + + /******************************************************************************* + ** Operator-only configuration; never load this from script records or HTTP + ** input. A stable private POSIX state directory and deployed reaper are + ** required by the real coordinator. Deadlines include setup and cleanup. + *******************************************************************************/ + public record Configuration(Path executable, Path coordinatorCli, String image, String qualificationId, + String requiredQualificationId, String ownerId, Path stateDirectory, String executableSearchPath, + Duration deadline, Duration cleanupGrace) + { + /*************************************************************************** + ** Reject absent trust configuration, mutable image tags and unsafe bounds. + ***************************************************************************/ + public Configuration + { + Objects.requireNonNull(executable); + Objects.requireNonNull(coordinatorCli); + Objects.requireNonNull(stateDirectory); + Objects.requireNonNull(deadline); + Objects.requireNonNull(cleanupGrace); + if(!executable.isAbsolute() || !Files.isRegularFile(executable) || !coordinatorCli.isAbsolute() || !Files.isRegularFile(coordinatorCli) + || !stateDirectory.isAbsolute() || !Files.isDirectory(stateDirectory) + || image == null || !image.matches("[a-z0-9./:_-]+@sha256:[a-f0-9]{64}") + || qualificationId == null || !qualificationId.matches("[A-Za-z0-9._:-]{1,128}") || !qualificationId.equals(requiredQualificationId) + || ownerId == null || !ownerId.matches("[A-Za-z0-9._-]{1,64}") || executableSearchPath == null + || deadline.isNegative() || deadline.isZero() || deadline.compareTo(Duration.ofSeconds(120)) > 0 + || cleanupGrace.isNegative() || cleanupGrace.compareTo(Duration.ofSeconds(20)) > 0) + { + throw new IllegalArgumentException("Worker coordinator configuration is incomplete or outside policy"); + } + } + } + + + + /*************************************************************************** + ** Capture immutable trusted configuration. Production deadline should allow + ** at least 90 seconds plus 15 seconds cleanup for Docker/journal operations. + ***************************************************************************/ + public CoordinatorWorkerProvider(Configuration config) + { + this.config = Objects.requireNonNull(config); + } + + + + /*************************************************************************** + ** Report the externally supplied evidence id, not an automatic certification. + ***************************************************************************/ + @Override + public String qualificationId() + { + return (config.qualificationId()); + } + + + + /*************************************************************************** + ** Include container setup in the host token lifetime, but do not grant + ** broker access during cleanup after the transport deadline. + ***************************************************************************/ + @Override + public Duration executionTransportBudget() + { + return (config.deadline()); + } + + + + /*************************************************************************** + ** Compilation has no broker authority. + ***************************************************************************/ + @Override + public String compile(String compileRequestJson) throws ScriptingException + { + return (exchange(compileRequestJson, "compile", "compileResult", null)); + } + + + + /*************************************************************************** + ** Broker callbacks remain on this authenticated calling thread. + ***************************************************************************/ + @Override + public String execute(String executeRequestJson, BrokerChannel broker) throws ScriptingException + { + return (exchange(executeRequestJson, "execute", "executeResult", Objects.requireNonNull(broker))); + } + + + + /*************************************************************************** + ** Bound framing, identity, lifecycle exit and time. A valid result is trusted + ** only after the coordinator exits successfully, which follows its cleanup. + ***************************************************************************/ + private String exchange(String request, String requestType, String resultType, BrokerChannel broker) throws ScriptingException + { + Process process = null; + try + { + JSONObject input = WorkerProtocol.parse(request, MAX_FRAME, requestType); + String requestId = input.getString("requestId"); + String token = input.optString("capabilityToken", ""); + ProcessBuilder builder = new ProcessBuilder(config.executable().toString(), config.coordinatorCli().toString()); + Map environment = builder.environment(); + environment.clear(); + environment.put("PATH", config.executableSearchPath()); + environment.put("LANG", "C.UTF-8"); + environment.put("QQQ_WORKER_IMAGE", config.image()); + environment.put("QQQ_WORKER_QUALIFICATION_ID", config.qualificationId()); + environment.put("QQQ_REQUIRED_QUALIFICATION_ID", config.requiredQualificationId()); + environment.put("QQQ_WORKER_OWNER_ID", config.ownerId()); + environment.put("QQQ_WORKER_STATE_DIRECTORY", config.stateDirectory().toString()); + String systemRoot = System.getenv("SystemRoot"); + if(systemRoot != null) + { + environment.put("SystemRoot", systemRoot); + } + process = builder.start(); + Process child = process; + AtomicBoolean timedOut = new AtomicBoolean(); + AtomicBoolean stderrExceeded = new AtomicBoolean(); + Thread stderr = Thread.ofPlatform().daemon().name("scripting-coordinator-stderr").start(() -> discardStderr(child, stderrExceeded)); + try(var watchdog = Executors.newSingleThreadScheduledExecutor(runnable -> + { + Thread thread = new Thread(runnable, "scripting-coordinator-deadline"); + thread.setDaemon(true); + return (thread); + })) + { + watchdog.schedule(() -> + { + timedOut.set(true); + child.destroy(); + }, config.deadline().toMillis(), TimeUnit.MILLISECONDS); + watchdog.schedule(() -> terminate(child), config.deadline().plus(config.cleanupGrace()).toMillis(), TimeUnit.MILLISECONDS); + try(InputStream stdout = process.getInputStream(); OutputStream stdin = process.getOutputStream()) + { + writeFrame(stdin, request); + String result = null; + int total = 0; + while(true) + { + String frame = readFrame(stdout); + if(frame == null) + { + break; + } + total += frame.getBytes(StandardCharsets.UTF_8).length; + if(total > MAX_TOTAL_OUTPUT || timedOut.get() || stderrExceeded.get()) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Worker coordinator exceeded its transport budget"); + } + JSONObject message = WorkerProtocol.parse(frame, MAX_FRAME, null); + String type = message.getString("type"); + if("broker".equals(type) && result == null && broker != null) + { + if(!token.equals(WorkerProtocol.decodeBrokerRequest(frame).token())) + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Worker broker token does not match this execution"); + } + String answer = broker.handle(frame); + WorkerProtocol.parse(answer, 256 * 1024, "brokerResult"); + writeFrame(stdin, answer); + } + else if(resultType.equals(type) && result == null && requestId.equals(message.optString("requestId"))) + { + result = frame; + } + else + { + throw new ScriptingException(ScriptingException.Kind.INTEGRITY, "Unexpected worker coordinator message"); + } + } + int exit = process.waitFor(); + stderr.join(1000); + if(exit != 0 || result == null || timedOut.get() || stderrExceeded.get()) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Worker coordinator did not complete successfully"); + } + return (result); + } + finally + { + watchdog.shutdownNow(); + } + } + } + catch(ScriptingException e) + { + throw e; + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Worker coordinator was interrupted"); + } + catch(IOException | RuntimeException e) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Worker coordinator is unavailable"); + } + finally + { + if(process != null && process.isAlive()) + { + process.destroy(); + try + { + if(!process.waitFor(config.cleanupGrace().toMillis(), TimeUnit.MILLISECONDS)) + { + terminate(process); + } + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + terminate(process); + } + } + } + } + + + + /*************************************************************************** + ** Read one bounded JSON line without BufferedReader's unbounded allocation. + ***************************************************************************/ + private static String readFrame(InputStream input) throws IOException, ScriptingException + { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + int value; + while((value = input.read()) != -1) + { + if(value == '\n') + { + return (StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(bytes.toByteArray())).toString()); + } + if(bytes.size() >= MAX_FRAME) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Worker coordinator frame is too large"); + } + bytes.write(value); + } + if(bytes.size() != 0) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Worker coordinator frame was incomplete"); + } + return (null); + } + + + + /*************************************************************************** + ** Only bounded JSON messages cross stdin; no commands or credentials. + ***************************************************************************/ + private static void writeFrame(OutputStream output, String frame) throws IOException + { + output.write(frame.getBytes(StandardCharsets.UTF_8)); + output.write('\n'); + output.flush(); + } + + + + /*************************************************************************** + ** Drain diagnostics without retaining or exposing their potentially private + ** contents. Excessive diagnostics terminate the trusted coordinator. + ***************************************************************************/ + private static void discardStderr(Process process, AtomicBoolean exceeded) + { + try(InputStream stderr = process.getErrorStream()) + { + byte[] buffer = new byte[4096]; + int total = 0; + int count; + while((count = stderr.read(buffer)) != -1) + { + total += count; + if(total > MAX_STDERR) + { + exceeded.set(true); + process.destroy(); + return; + } + } + } + catch(IOException e) + { + // Process termination closes its diagnostic stream. + } + } + + + + /*************************************************************************** + ** Last-resort process cleanup. Container ownership leases survive this and + ** require the deployed reaper after hard kill or host failure. + ***************************************************************************/ + private static void terminate(Process process) + { + process.descendants().forEach(child -> child.destroyForcibly()); + process.destroyForcibly(); + } +} + diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/PersistentWorkerProvider.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/PersistentWorkerProvider.java new file mode 100644 index 0000000..3c55303 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/PersistentWorkerProvider.java @@ -0,0 +1,935 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + +import javax.net.ssl.SSLContext; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.worker.SupervisorConnection.Lease; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecutionLimits; +import org.json.JSONObject; + + + +/******************************************************************************* + ** Optional transport to an externally qualified supervisor. No guest code runs + ** here. Warm execute never connects, uploads code, compiles or retries. Only + ** heartbeat handoff may wait within the existing whole-operation deadline. + *******************************************************************************/ +public final class PersistentWorkerProvider implements WorkerProvider, WarmWorkerMaintenance +{ + private final Configuration config; + private final int preparationCopies; + private final List slots = new ArrayList<>(); + private final Set connections = new HashSet<>(); + private final Semaphore cold; + private final Semaphore authoringUses; + private final Map protectedCode = new HashMap<>(); + private final Semaphore executions; + private final ScheduledThreadPoolExecutor deadlines; + private boolean closed; + + /*************************************************************************** + ** Fixed operator endpoint, never supplied by a script or request. + ***************************************************************************/ + public record Endpoint(String host, int port) + { + /*********************************************************************** + ** Permit DNS names and IPv4 literals, not URLs, paths or credentials. + ***********************************************************************/ + public Endpoint + { + if(host == null || !host.matches("[A-Za-z0-9](?:[A-Za-z0-9.-]{0,251}[A-Za-z0-9])?") || port < 1 || port > 65535) + { + throw new IllegalArgumentException("Invalid supervisor endpoint"); + } + } + } + + /*************************************************************************** + ** SSLContext owns operator trust/client identity. Remote service must enforce + ** authenticated namespace access. Matching evidence IDs do not qualify it. + ***************************************************************************/ + public record Configuration(SSLContext sslContext, Endpoint executionEndpoint, Endpoint compileEndpoint, + String namespace, String profile, String toolchain, String qualificationId, String requiredQualificationId, + Duration deadline, int executionConnections, int coldOperations, int entriesPerConnection, long codeBytesPerConnection) + { + /*********************************************************************** + ** Refuse unqualified or unbounded configuration before opening sockets. + ***********************************************************************/ + public Configuration + { + Objects.requireNonNull(sslContext); + Objects.requireNonNull(executionEndpoint); + Objects.requireNonNull(deadline); + if(!identifier(namespace) || !identifier(profile) || !identifier(toolchain) || !identifier(qualificationId) + || !qualificationId.equals(requiredQualificationId) || deadline.toMillis() < 1 || deadline.compareTo(Duration.ofSeconds(120)) > 0 + || executionConnections < 1 || executionConnections > 64 || coldOperations < 1 || coldOperations > 8 + || entriesPerConnection < 1 || entriesPerConnection > 256 || codeBytesPerConnection < 1 || codeBytesPerConnection > 64L * 1024 * 1024) + { + throw new IllegalArgumentException("Supervisor configuration is outside policy"); + } + } + } + + /*************************************************************************** + ** Verified immutable code only. This object never carries invocation data. + ***************************************************************************/ + public record PreparedArtifact(String sha256, String entryPoint, String bundle) + { + /*********************************************************************** + ** Verify bytes before either cache admission or remote transfer. + ***********************************************************************/ + public PreparedArtifact + { + if(sha256 == null || !sha256.matches("[a-f0-9]{64}") || entryPoint == null + || !entryPoint.matches("[A-Za-z0-9_./-]{1,200}") || entryPoint.startsWith("/") || entryPoint.contains("..") + || bundle == null || bundle.isEmpty() || bundle.length() > WorkerProtocol.MAX_BUNDLE_CHARS + || bundle.getBytes(StandardCharsets.UTF_8).length > WorkerProtocol.MAX_EXECUTE_MESSAGE_BYTES + || !sha256.equals(BuildCoordinator.sha256Hex(bundle))) + { + throw new IllegalArgumentException("Invalid prepared artifact"); + } + } + + /*********************************************************************** + ** Never expose source text through ordinary object diagnostics. + ***********************************************************************/ + @Override + public String toString() + { + return ("PreparedArtifact[redacted]"); + } + + /*********************************************************************** + ** Entry point is part of cache identity even for identical bundle bytes. + ***********************************************************************/ + private String key() + { + return (sha256 + ":" + entryPoint); + } + } + + /*************************************************************************** + ** Construction creates no network connection or successful readiness claim. + ***************************************************************************/ + public PersistentWorkerProvider(Configuration config) + { + this(config, 1); + } + + /*************************************************************************** + ** Operator-selected warm copies; existing callers retain one-copy behavior. + ***************************************************************************/ + public PersistentWorkerProvider(Configuration config, int preparationCopies) + { + this.config = Objects.requireNonNull(config); + if(preparationCopies < 1 || preparationCopies > config.executionConnections()) + { + throw new IllegalArgumentException("Preparation copies exceed configured capacity"); + } + this.preparationCopies = preparationCopies; + cold = new Semaphore(config.coldOperations()); + authoringUses = new Semaphore(config.coldOperations()); + executions = new Semaphore(config.executionConnections()); + deadlines = new ScheduledThreadPoolExecutor(1, runnable -> + { + Thread thread = new Thread(runnable, "scripting-supervisor-deadline"); + thread.setDaemon(true); + return (thread); + }); + deadlines.setRemoveOnCancelPolicy(true); + } + + /*************************************************************************** + ** Existing external evidence gate; not automatic runtime certification. + ***************************************************************************/ + @Override + public String qualificationId() + { + return (config.qualificationId()); + } + + /*************************************************************************** + ** Whole exchange allowance, separate from the unchanged guest resource cap. + ***************************************************************************/ + @Override + public Duration executionTransportBudget() + { + return (config.deadline()); + } + + /*************************************************************************** + ** Authoring SPI uses only the operator's profile/toolchain and bounded copies. + ** Warm execution remains prepared-only and never invokes this method. + ***************************************************************************/ + @Override + public void prepareVerified(WarmWorkerPreparation.Artifact artifact) throws ScriptingException + { + Objects.requireNonNull(artifact); + if(!config.toolchain().equals(artifact.toolchainId())) + { + throw unavailable(); + } + prepare(new PreparedArtifact(artifact.sha256(), artifact.entryPoint(), artifact.bundle()), preparationCopies); + } + + /*************************************************************************** + ** Reserve code before cold preparation and retain it through the caller's + ** scoped authoring use. Admission is bounded and immediate; no wait queue, + ** execution data or durable registry is created. Release performs no I/O. + ***************************************************************************/ + @Override + public UseLease prepareForUse(WarmWorkerPreparation.Artifact artifact) throws ScriptingException + { + Objects.requireNonNull(artifact); + if(!config.toolchain().equals(artifact.toolchainId()) || !authoringUses.tryAcquire()) + { + throw unavailable(); + } + String key = artifact.sha256() + ":" + artifact.entryPoint(); + try + { + synchronized(slots) + { + ensureOpen(); + protectedCode.merge(key, 1, Integer::sum); + } + } + catch(ScriptingException | RuntimeException | Error failure) + { + authoringUses.release(); + throw failure; + } + AtomicBoolean released = new AtomicBoolean(); + UseLease lease = () -> + { + if(released.compareAndSet(false, true)) + { + synchronized(slots) + { + int count = protectedCode.getOrDefault(key, 0); + if(count <= 1) + { + protectedCode.remove(key); + } + else + { + protectedCode.put(key, count - 1); + } + } + authoringUses.release(); + } + }; + try + { + prepareVerified(artifact); + return (lease); + } + catch(ScriptingException | RuntimeException | Error failure) + { + lease.close(); + throw failure; + } + } + + /*************************************************************************** + ** Explicit cold preparation fills copies on distinct connections. No active + ** slot is touched or evicted; partial successful warming is safe to retain. + ***************************************************************************/ + public void prepare(PreparedArtifact artifact, int copies) throws ScriptingException + { + if(copies < 1 || copies > config.executionConnections()) + { + throw unavailable(); + } + prepare(artifact, copies, System.nanoTime() + config.deadline().toNanos() * copies); + } + + /*************************************************************************** + ** The native lifecycle supplies a remaining whole-pass cold budget. + ***************************************************************************/ + @Override + public void prepareVerified(WarmWorkerPreparation.Artifact artifact, Duration budget) throws ScriptingException + { + Objects.requireNonNull(artifact); + if(!config.toolchain().equals(artifact.toolchainId())) + { + throw unavailable(); + } + prepare(new PreparedArtifact(artifact.sha256(), artifact.entryPoint(), artifact.bundle()), preparationCopies, stopAt(budget)); + } + + /*************************************************************************** + ** All copies share the explicit bound; successful partial warming is retained. + ***************************************************************************/ + private void prepare(PreparedArtifact artifact, int copies, long stop) throws ScriptingException + { + Objects.requireNonNull(artifact); + if(copies < 1 || copies > config.executionConnections()) + { + throw unavailable(); + } + if(!cold.tryAcquire()) + { + throw unavailable(); + } + try + { + while(true) + { + if(Thread.currentThread().isInterrupted() || System.nanoTime() >= stop) + { + throw unavailable(); + } + SupervisorConnection connection; + boolean fresh = false; + long bytes = artifact.bundle().getBytes(StandardCharsets.UTF_8).length; + synchronized(slots) + { + ensureOpen(); + if(slots.stream().filter(slot -> slot.isOpen() && slot.ready.containsKey(artifact.key())).count() >= copies) + { + return; + } + connection = slots.stream().filter(slot -> slot.lease == Lease.IDLE && slot.executionWaiters == 0 && !slot.ready.containsKey(artifact.key()) + && slot.ready.size() < config.entriesPerConnection() && slot.codeBytes + bytes <= config.codeBytesPerConnection()).findFirst().orElse(null); + if(connection == null) + { + if(slots.size() >= config.executionConnections() || bytes > config.codeBytesPerConnection()) + { + throw unavailable(); + } + connection = new SupervisorConnection(config); + slots.add(connection); + connections.add(connection); + fresh = true; + } + connection.lease = Lease.PREPARATION; + } + boolean success = false; + try + { + connection.begin(deadlines, stop - System.nanoTime()); + if(fresh) + { + connection.connect(config.executionEndpoint()); + } + connection.next(); + String requestId = UUID.randomUUID().toString(); + connection.write(connection.envelope("PREPARE", requestId).put("sha256", artifact.sha256()) + .put("entryPoint", artifact.entryPoint()).put("bundle", artifact.bundle())); + JSONObject ready = connection.read(); + SupervisorProtocol.keys(ready, "protocol", "type", "session", "sequence", "requestId", "sha256", "entryPoint", "readyId"); + connection.identity(ready, requestId); + SupervisorProtocol.equal(ready, "type", "READY"); + SupervisorProtocol.equal(ready, "sha256", artifact.sha256()); + SupervisorProtocol.equal(ready, "entryPoint", artifact.entryPoint()); + String readyId = SupervisorProtocol.identity(ready, "readyId"); + connection.retired(requestId); + synchronized(slots) + { + ensureOpen(); + connection.ready.put(artifact.key(), readyId); + connection.codeBytes += bytes; + } + success = true; + } + finally + { + release(connection, success); + } + } + } + catch(IOException | RuntimeException failure) + { + throw unavailable(); + } + finally + { + cold.release(); + } + } + + /*************************************************************************** + ** Heartbeats run only on idle slots under existing cold admission. A failed + ** session is discarded; busy invocation slots are never stolen or evicted. + ***************************************************************************/ + @Override + public void heartbeat(Duration budget) throws ScriptingException + { + heartbeat(budget, Duration.ZERO); + } + + /*************************************************************************** + ** A slot whose last PING/PREPARE/INVOKE exchange completed with RETIRED inside + ** the window needs no further wire exchange yet. Unconfirmed, stale, closed or + ** busy slots are never skipped as ready; readiness age is unchanged. + ***************************************************************************/ + @Override + public void heartbeat(Duration budget, Duration confirmedWithin) throws ScriptingException + { + if(confirmedWithin == null || confirmedWithin.isNegative() || confirmedWithin.compareTo(Duration.ofMinutes(10)) > 0) + { + throw unavailable(); + } + long skipNanos = confirmedWithin.toNanos(); + long stop = stopAt(budget); + if(!cold.tryAcquire()) + { + throw unavailable(); + } + try + { + List snapshot; + synchronized(slots) + { + ensureOpen(); + snapshot = List.copyOf(slots); + } + for(SupervisorConnection connection : snapshot) + { + if(Thread.currentThread().isInterrupted() || System.nanoTime() >= stop) + { + throw unavailable(); + } + synchronized(slots) + { + ensureOpen(); + if(!slots.contains(connection) || connection.lease != Lease.IDLE || connection.executionWaiters != 0) + { + continue; + } + if(skipNanos > 0 && connection.isOpen() && connection.confirmedAt != 0 && System.nanoTime() - connection.confirmedAt < skipNanos) + { + continue; + } + connection.lease = Lease.HEARTBEAT; + } + boolean success = false; + try + { + connection.begin(deadlines, stop - System.nanoTime()); + connection.next(); + String requestId = UUID.randomUUID().toString(); + connection.write(connection.envelope("PING", requestId)); + JSONObject pong = connection.read(); + SupervisorProtocol.keys(pong, "protocol", "type", "session", "sequence", "requestId"); + connection.identity(pong, requestId); + SupervisorProtocol.equal(pong, "type", "PONG"); + connection.retired(requestId); + success = true; + } + finally + { + release(connection, success); + } + } + } + catch(IOException | RuntimeException failure) + { + throw unavailable(); + } + finally + { + cold.release(); + } + } + + /*************************************************************************** + ** Cold-only reclamation retires idle sessions containing obsolete code, + ** then lets the normal lifecycle rebuild + ** the complete current inventory. Busy slots are never removed. No new wire + ** operation or hot fallback is introduced; closing loses the entire epoch. + ***************************************************************************/ + @Override + public void retireUnused(List identities, Duration budget) throws ScriptingException + { + long stop = stopAt(budget); + if(identities == null || identities.size() > 1000) + { + throw unavailable(); + } + Set required = new HashSet<>(); + for(CodeIdentity identity : List.copyOf(identities)) + { + if(!config.toolchain().equals(identity.toolchainId())) + { + throw unavailable(); + } + required.add(identity.sha256() + ":" + identity.entryPoint()); + } + if(!cold.tryAcquire()) + { + throw unavailable(); + } + List retired = new ArrayList<>(); + try + { + synchronized(slots) + { + ensureOpen(); + if(Thread.currentThread().isInterrupted() || System.nanoTime() >= stop) + { + throw unavailable(); + } + for(SupervisorConnection slot : List.copyOf(slots)) + { + if(slot.lease == Lease.IDLE && slot.executionWaiters == 0 && slot.ready.keySet().stream().noneMatch(protectedCode::containsKey) + && slot.ready.keySet().stream().anyMatch(key -> !required.contains(key))) + { + slot.lease = Lease.RETIRING; + slots.remove(slot); + connections.remove(slot); + retired.add(slot); + slots.notifyAll(); + } + } + } + } + finally + { + try + { + for(SupervisorConnection slot : retired) + { + slot.close(); + } + } + finally + { + cold.release(); + } + } + } + + /*************************************************************************** + ** No network or preparation occurs in this bounded local readiness check. + ***************************************************************************/ + @Override + public boolean isPrepared(String sha256, String entryPoint, String toolchainId, Duration maxAge) + { + try + { + return (arePrepared(List.of(new CodeIdentity(sha256, entryPoint, toolchainId)), maxAge)); + } + catch(IllegalArgumentException failure) + { + return (false); + } + } + + /*************************************************************************** + ** One lock and timestamp attest the entire required set, not different epochs + ** observed independently around concurrent retirement or preparation. + ***************************************************************************/ + @Override + public boolean arePrepared(List identities, Duration maxAge) + { + if(identities == null || identities.size() > 1000 || maxAge == null || maxAge.isNegative() || maxAge.isZero() + || maxAge.compareTo(Duration.ofMinutes(10)) > 0) + { + return (false); + } + List required; + try + { + required = List.copyOf(identities); + } + catch(RuntimeException failure) + { + return (false); + } + synchronized(slots) + { + long now = System.nanoTime(); + return (!closed && required.stream().allMatch(identity -> config.toolchain().equals(identity.toolchainId()) + && slots.stream().filter(slot -> slot.isOpen() && slot.ready.containsKey(identity.sha256() + ":" + identity.entryPoint()) + && slot.confirmedAt != 0 && now - slot.confirmedAt <= maxAge.toNanos()).count() >= preparationCopies)); + } + } + + /*************************************************************************** + ** External maintenance budgets are finite and never derived from author input. + ***************************************************************************/ + private static long stopAt(Duration budget) throws ScriptingException + { + if(budget == null || budget.isNegative() || budget.isZero() || budget.compareTo(Duration.ofSeconds(120)) > 0) + { + throw unavailable(); + } + return (System.nanoTime() + budget.toNanos()); + } + + /*************************************************************************** + ** Hot execution leases existing readiness, with bounded heartbeat handoff + ** only. A cold miss or another executing/preparing guest never queues. + ***************************************************************************/ + @Override + public String execute(String requestJson, BrokerChannel broker) throws ScriptingException + { + Objects.requireNonNull(broker); + long stop = System.nanoTime() + config.deadline().toNanos(); + if(!executions.tryAcquire()) + { + throw unavailable(); + } + SupervisorConnection connection = null; + boolean success = false; + try + { + JSONObject request = WorkerProtocol.parse(requestJson, WorkerProtocol.MAX_EXECUTE_MESSAGE_BYTES, "execute"); + SupervisorProtocol.keys(request, "protocol", "type", "requestId", "artifactId", "artifactSha256", "bundle", "entryPoint", "event", "payload", "capabilityToken", "limits"); + String requestId = SupervisorProtocol.identity(request, "requestId"); + String token = SupervisorProtocol.identity(request, "capabilityToken"); + SupervisorProtocol.identity(request, "artifactId"); + SupervisorProtocol.identity(request, "event"); + PreparedArtifact artifact = new PreparedArtifact(string(request, "artifactSha256"), string(request, "entryPoint"), string(request, "bundle")); + WorkerProtocol.checkPayload(SupervisorProtocol.object(request, "payload").toString()); + ExecutionLimits limits = limits(SupervisorProtocol.object(request, "limits")); + connection = leaseExecution(artifact.key(), stop); + connection.beginUntil(deadlines, stop); + connection.next(); + request.remove("bundle"); + connection.write(connection.envelope("INVOKE", requestId).put("readyId", connection.ready.get(artifact.key())).put("request", request)); + String result = exchange(connection, requestId, token, limits, "CONTEXT".equals(request.getString("event")), broker); + connection.retired(requestId); + success = true; + return (result); + } + catch(IOException | RuntimeException failure) + { + throw unavailable(); + } + finally + { + try + { + if(connection != null) + { + release(connection, success); + } + } + finally + { + executions.release(); + } + } + } + + /*************************************************************************** + ** Reserve only a matching heartbeat epoch. Existing execution admission + ** bounds waiters; cold work cannot reclaim their slot. Failure never follows + ** a removed epoch or waits behind an actual invocation. + ***************************************************************************/ + private SupervisorConnection leaseExecution(String key, long stop) throws ScriptingException + { + synchronized(slots) + { + requireExecutionTime(stop); + SupervisorConnection idle = slots.stream().filter(slot -> slot.isOpen() && slot.lease == Lease.IDLE + && slot.executionWaiters == 0 && slot.ready.containsKey(key)).findFirst().orElse(null); + if(idle != null) + { + idle.lease = Lease.EXECUTION; + return (idle); + } + SupervisorConnection waiting = slots.stream().filter(slot -> slot.isOpen() && slot.lease == Lease.HEARTBEAT + && slot.ready.containsKey(key)).findFirst().orElse(null); + if(waiting == null) + { + throw unavailable(); + } + waiting.executionWaiters++; + try + { + while(true) + { + requireExecutionTime(stop); + if(!slots.contains(waiting) || !waiting.isOpen() || !waiting.ready.containsKey(key)) + { + throw unavailable(); + } + if(waiting.lease == Lease.IDLE) + { + waiting.lease = Lease.EXECUTION; + return (waiting); + } + if(waiting.lease != Lease.HEARTBEAT) + { + throw unavailable(); + } + TimeUnit.NANOSECONDS.timedWait(slots, Math.max(1, stop - System.nanoTime())); + } + } + catch(InterruptedException failure) + { + Thread.currentThread().interrupt(); + throw unavailable(); + } + finally + { + waiting.executionWaiters--; + slots.notifyAll(); + } + } + } + + /*************************************************************************** + ** Read cancellation and the original deadline while holding the pool monitor. + ***************************************************************************/ + private void requireExecutionTime(long stop) throws ScriptingException + { + ensureOpen(); + if(Thread.currentThread().isInterrupted() || System.nanoTime() >= stop) + { + throw unavailable(); + } + } + + /*************************************************************************** + ** Process broker frames on the current calling thread, never a listener pool. + ***************************************************************************/ + private String exchange(SupervisorConnection connection, String requestId, String token, ExecutionLimits limits, + boolean context, BrokerChannel broker) throws IOException, ScriptingException + { + int calls = 0; + while(true) + { + JSONObject frame = connection.read(); + SupervisorProtocol.keys(frame, "protocol", "type", "session", "sequence", "requestId", "message"); + connection.identity(frame, requestId); + JSONObject message = SupervisorProtocol.object(frame, "message"); + String json = message.toString(); + if("BROKER".equals(frame.opt("type"))) + { + if(++calls > limits.brokerCalls() || !token.equals(WorkerProtocol.decodeBrokerRequest(json).token())) + { + throw SupervisorProtocol.refused(); + } + SupervisorProtocol.equal(message, "requestId", requestId); + String response = broker.handle(json); + connection.check(); + JSONObject answer = WorkerProtocol.parse(response, WorkerProtocol.MAX_BROKER_MESSAGE_BYTES, "brokerResult"); + connection.write(connection.envelope("BROKER_REPLY", requestId).put("message", answer)); + } + else if("RESULT".equals(frame.opt("type"))) + { + if(json.getBytes(StandardCharsets.UTF_8).length > limits.outputBytes()) + { + throw SupervisorProtocol.refused(); + } + var decoded = WorkerProtocol.decodeExecuteResponse(json, context); + if(!requestId.equals(decoded.requestId())) + { + throw SupervisorProtocol.refused(); + } + return (json); + } + else + { + throw SupervisorProtocol.refused(); + } + } + } + + /*************************************************************************** + ** Compilation has independent cold admission/endpoint and no broker channel. + ***************************************************************************/ + @Override + public String compile(String requestJson) throws ScriptingException + { + if(config.compileEndpoint() == null || !cold.tryAcquire()) + { + throw unavailable(); + } + SupervisorConnection connection = null; + try + { + JSONObject request = WorkerProtocol.parse(requestJson, WorkerProtocol.MAX_COMPILE_MESSAGE_BYTES, "compile"); + SupervisorProtocol.keys(request, "protocol", "type", "requestId", "toolchainId", "entryPoint", "sourceSha256", "strict", "files"); + String requestId = SupervisorProtocol.identity(request, "requestId"); + SupervisorProtocol.equal(request, "toolchainId", config.toolchain()); + if(!Boolean.TRUE.equals(request.opt("strict"))) + { + throw SupervisorProtocol.refused(); + } + synchronized(slots) + { + ensureOpen(); + connection = new SupervisorConnection(config); + connections.add(connection); + } + connection.begin(deadlines); + connection.connect(config.compileEndpoint()); + connection.next(); + connection.write(connection.envelope("COMPILE", requestId).put("request", request)); + JSONObject frame = connection.read(); + SupervisorProtocol.keys(frame, "protocol", "type", "session", "sequence", "requestId", "message"); + connection.identity(frame, requestId); + SupervisorProtocol.equal(frame, "type", "RESULT"); + String result = SupervisorProtocol.object(frame, "message").toString(); + var decoded = WorkerProtocol.decodeCompileResponse(result); + if(!requestId.equals(decoded.requestId()) || !config.toolchain().equals(decoded.toolchainId()) || !decoded.strict()) + { + throw SupervisorProtocol.refused(); + } + connection.retired(requestId); + return (result); + } + catch(IOException | RuntimeException failure) + { + throw unavailable(); + } + finally + { + if(connection != null) + { + release(connection, false); + } + cold.release(); + } + } + + /*************************************************************************** + ** Exact bounded integer limits; no strings, floating values or coercion. + ***************************************************************************/ + private static ExecutionLimits limits(JSONObject limits) throws ScriptingException + { + SupervisorProtocol.keys(limits, "wallMillis", "cpuMillis", "memoryBytes", "outputBytes", "brokerCalls"); + for(String key : limits.keySet()) + { + Object value = limits.opt(key); + if(!(value instanceof Integer || value instanceof Long) || ((Number) value).longValue() < 0 || ((Number) value).longValue() > Integer.MAX_VALUE) + { + throw SupervisorProtocol.refused(); + } + } + return (new ExecutionLimits(limits.getInt("wallMillis"), limits.getInt("cpuMillis"), limits.getLong("memoryBytes"), limits.getInt("outputBytes"), limits.getInt("brokerCalls"))); + } + + /*************************************************************************** + ** Source strings must remain strings at this public boundary. + ***************************************************************************/ + private static String string(JSONObject object, String key) throws ScriptingException + { + if(!(object.opt(key) instanceof String value)) + { + throw SupervisorProtocol.refused(); + } + return (value); + } + + /*************************************************************************** + ** Configuration and opaque identities share a finite ASCII grammar. + ***************************************************************************/ + private static boolean identifier(String value) + { + return (value != null && value.matches("[A-Za-z0-9._:-]{1,128}")); + } + + /*************************************************************************** + ** Pool state is guarded together, including close racing cold preparation. + ***************************************************************************/ + private void ensureOpen() throws ScriptingException + { + if(closed) + { + throw unavailable(); + } + } + + /*************************************************************************** + ** Any uncertain slot loses all readiness. No failed operation is replayed. + ***************************************************************************/ + private void release(SupervisorConnection connection, boolean success) + { + connection.end(); + boolean discard; + synchronized(slots) + { + discard = !success || closed; + if(discard) + { + slots.remove(connection); + connections.remove(connection); + } + if(success && !closed) + { + connection.confirmedAt = System.nanoTime(); + } + connection.lease = Lease.IDLE; + slots.notifyAll(); + } + if(discard) + { + connection.close(); + } + } + + /*************************************************************************** + ** Value-free refusal, without endpoints, payloads, tokens or TLS diagnostics. + ***************************************************************************/ + private static ScriptingException unavailable() + { + return (new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Prepared supervisor capacity is unavailable")); + } + + /*************************************************************************** + ** Host shutdown closes only this provider's sockets and deadline scheduler. + ***************************************************************************/ + @Override + public void close() + { + List retired; + synchronized(slots) + { + closed = true; + retired = List.copyOf(connections); + connections.clear(); + slots.clear(); + slots.notifyAll(); + } + try + { + retired.forEach(SupervisorConnection::close); + } + finally + { + deadlines.shutdownNow(); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/SupervisorConnection.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/SupervisorConnection.java new file mode 100644 index 0000000..64219bd --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/SupervisorConnection.java @@ -0,0 +1,356 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; +import java.nio.ByteBuffer; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.scripting.ScriptingException; +import org.json.JSONObject; + + + +/******************************************************************************* + ** One exclusively leased TLS session. No QContext or broker runs on I/O threads. + *******************************************************************************/ +final class SupervisorConnection implements AutoCloseable +{ + private final SSLSocket socket; + private final PersistentWorkerProvider.Configuration config; + private InputStream input; + private OutputStream output; + private String session; + private long sequence; + private long deadline; + private int transcript; + private ScheduledFuture expiration; + private long deadlineEpoch; + private volatile boolean closed; + final Map ready = new HashMap<>(); + long codeBytes; + long confirmedAt; + Lease lease = Lease.IDLE; + int executionWaiters; + + /*************************************************************************** + ** Lease state and waiter count are accessed only under the provider monitor. + ***************************************************************************/ + enum Lease + { + IDLE, PREPARATION, HEARTBEAT, EXECUTION, RETIRING + } + + /*************************************************************************** + ** Allocate an unconnected socket; caller installs deadline before connecting. + ***************************************************************************/ + SupervisorConnection(PersistentWorkerProvider.Configuration config) throws IOException + { + this.config = config; + socket = (SSLSocket) config.sslContext().getSocketFactory().createSocket(); + SSLParameters parameters = socket.getSSLParameters(); + parameters.setEndpointIdentificationAlgorithm("HTTPS"); + parameters.setProtocols(new String[] {"TLSv1.3"}); + socket.setSSLParameters(parameters); + socket.setSoTimeout(250); + socket.setTcpNoDelay(true); + } + + /*************************************************************************** + ** A hard close also bounds blocked writes; socket read polling sees interrupts. + ***************************************************************************/ + synchronized void begin(ScheduledExecutorService scheduler) + { + begin(scheduler, config.deadline().toNanos()); + } + + /*************************************************************************** + ** Maintenance may shorten, never lengthen the configured operation budget. + ***************************************************************************/ + synchronized void begin(ScheduledExecutorService scheduler, long remainingNanos) + { + long budget = Math.min(config.deadline().toNanos(), Math.max(1, remainingNanos)); + beginUntil(scheduler, System.nanoTime() + budget); + } + + /*************************************************************************** + ** Handoff and wire exchange share one absolute deadline; waiting cannot + ** extend the operation or turn an expired lease into a new wire budget. + ***************************************************************************/ + synchronized void beginUntil(ScheduledExecutorService scheduler, long stop) + { + long now = System.nanoTime(); + long budget = Math.min(config.deadline().toNanos(), stop - now); + transcript = 0; + deadline = now + budget; + long epoch = ++deadlineEpoch; + expiration = scheduler.schedule(() -> expire(epoch), Math.max(1, budget), TimeUnit.NANOSECONDS); + } + + /*************************************************************************** + ** Only cold preparation/compilation calls connect and authenticate a session. + ***************************************************************************/ + void connect(PersistentWorkerProvider.Endpoint endpoint) throws IOException, ScriptingException + { + socket.connect(new InetSocketAddress(endpoint.host(), endpoint.port()), (int) config.deadline().toMillis()); + socket.setSoTimeout((int) config.deadline().toMillis()); + socket.startHandshake(); + if(socket.getSession().getLocalCertificates() == null || socket.getSession().getLocalCertificates().length == 0) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Supervisor transport requires mutual TLS"); + } + socket.setSoTimeout(250); + input = new BufferedInputStream(socket.getInputStream(), 8192); + output = new BufferedOutputStream(socket.getOutputStream(), 8192); + String nonce = UUID.randomUUID().toString(); + JSONObject hello = new JSONObject().put("protocol", SupervisorProtocol.VERSION).put("type", "HELLO") + .put("namespace", config.namespace()).put("profile", config.profile()).put("toolchain", config.toolchain()) + .put("evidence", config.qualificationId()).put("nonce", nonce); + write(hello); + JSONObject answer = read(); + SupervisorProtocol.keys(answer, "protocol", "type", "namespace", "profile", "toolchain", "evidence", "nonce", "session"); + SupervisorProtocol.equal(answer, "protocol", SupervisorProtocol.VERSION); + SupervisorProtocol.equal(answer, "type", "ACCEPT"); + for(String key : new String[] {"namespace", "profile", "toolchain", "evidence", "nonce"}) + { + SupervisorProtocol.equal(answer, key, hello.getString(key)); + } + session = SupervisorProtocol.identity(answer, "session"); + } + + /*************************************************************************** + ** Sequence is connection-local and never reused, including failed operations. + ***************************************************************************/ + JSONObject envelope(String type, String requestId) throws ScriptingException + { + check(); + return (new JSONObject().put("protocol", SupervisorProtocol.VERSION).put("type", type) + .put("session", session).put("sequence", sequence).put("requestId", requestId)); + } + + /*************************************************************************** + ** Start an operation after handshake; exhaustion refuses instead of wrapping. + ***************************************************************************/ + void next() throws ScriptingException + { + if(sequence == Long.MAX_VALUE) + { + throw SupervisorProtocol.refused(); + } + sequence++; + } + + /*************************************************************************** + ** Validate exact active operation identity on every inbound envelope. + ***************************************************************************/ + void identity(JSONObject frame, String requestId) throws ScriptingException + { + SupervisorProtocol.equal(frame, "protocol", SupervisorProtocol.VERSION); + SupervisorProtocol.equal(frame, "session", session); + SupervisorProtocol.equal(frame, "requestId", requestId); + Object value = frame.opt("sequence"); + if(!(value instanceof Integer || value instanceof Long) || ((Number) value).longValue() != sequence) + { + throw SupervisorProtocol.refused(); + } + } + + /*************************************************************************** + ** Strict signed-length framing allocates only after checking frame bounds. + ***************************************************************************/ + JSONObject read() throws IOException, ScriptingException + { + byte[] header = new byte[4]; + readFully(header); + int length = ByteBuffer.wrap(header).getInt(); + if(length < 2 || length > SupervisorProtocol.MAX_FRAME) + { + throw SupervisorProtocol.refused(); + } + account(length); + byte[] bytes = new byte[length]; + readFully(bytes); + String text = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(bytes)).toString(); + return (SupervisorProtocol.parse(text)); + } + + /*************************************************************************** + ** Preserve partial reads across timeouts, checking cancellation on each poll. + ***************************************************************************/ + private void readFully(byte[] bytes) throws IOException, ScriptingException + { + int offset = 0; + while(offset < bytes.length) + { + check(); + try + { + int count = input.read(bytes, offset, bytes.length - offset); + if(count < 0) + { + throw new EOFException(); + } + offset += count; + } + catch(SocketTimeoutException ignored) + { + // Keep partial bytes and recheck the whole-operation deadline. + } + } + } + + /*************************************************************************** + ** The caller owns bounded immutable messages; no logging retains their data. + ***************************************************************************/ + void write(JSONObject message) throws IOException, ScriptingException + { + check(); + byte[] bytes = message.toString().getBytes(StandardCharsets.UTF_8); + if(bytes.length > SupervisorProtocol.MAX_FRAME) + { + throw SupervisorProtocol.refused(); + } + account(bytes.length); + output.write(ByteBuffer.allocate(4).putInt(bytes.length).array()); + output.write(bytes); + output.flush(); + } + + /*************************************************************************** + ** Count both directions, including broker answers and protocol overhead. + ***************************************************************************/ + private void account(int bytes) throws ScriptingException + { + transcript += bytes + 4; + if(transcript > SupervisorProtocol.MAX_TRANSCRIPT) + { + throw SupervisorProtocol.refused(); + } + } + + /*************************************************************************** + ** A terminal guest result is insufficient; only supervisor retirement ends it. + ***************************************************************************/ + void retired(String requestId) throws IOException, ScriptingException + { + JSONObject frame = read(); + SupervisorProtocol.keys(frame, "protocol", "type", "session", "sequence", "requestId", "retired"); + identity(frame, requestId); + SupervisorProtocol.equal(frame, "type", "RETIRED"); + if(!Boolean.TRUE.equals(frame.opt("retired"))) + { + throw SupervisorProtocol.refused(); + } + check(); + } + + /*************************************************************************** + ** A slot with uncertain transport state can never be returned to the pool. + ***************************************************************************/ + void check() throws ScriptingException + { + if(closed || Thread.currentThread().isInterrupted() || System.nanoTime() >= deadline) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Supervisor connection is unavailable"); + } + } + + /*************************************************************************** + ** Local readiness never treats a watchdog-closed socket as usable. + ***************************************************************************/ + boolean isOpen() + { + return (!closed); + } + + /*************************************************************************** + ** Stop this operation's watchdog before an idle connection becomes reusable. + ***************************************************************************/ + synchronized void end() + { + deadlineEpoch++; + if(expiration != null) + { + expiration.cancel(false); + } + } + + /*************************************************************************** + ** An already-running old watchdog cannot terminate a later leased operation. + ***************************************************************************/ + private synchronized void expire(long epoch) + { + if(epoch == deadlineEpoch) + { + close(); + } + } + + /*************************************************************************** + ** Best-effort local teardown never replaces a primary application failure. + ***************************************************************************/ + @Override + public void close() + { + closed = true; + try + { + socket.setSoLinger(true, 0); + socket.setSoTimeout(250); + } + catch(IOException ignored) + { + /* Already disconnected or closing. */ + } + try + { + socket.shutdownInput(); + } + catch(IOException ignored) + { + /* Missing peer close_notify must not block the deadline executor. */ + } + try + { + socket.close(); + } + catch(IOException ignored) + { + /* Remote cleanup remains supervisor-owned. */ + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/SupervisorProtocol.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/SupervisorProtocol.java new file mode 100644 index 0000000..8d466c0 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/SupervisorProtocol.java @@ -0,0 +1,162 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + +import java.nio.charset.StandardCharsets; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingException; +import org.json.JSONObject; +import org.json.JSONParserConfiguration; +import org.json.JSONTokener; + + + +/******************************************************************************* + ** Closed trusted-supervisor envelope, separate from the guest worker protocol. + *******************************************************************************/ +final class SupervisorProtocol +{ + static final String VERSION = "qbit-scripting-supervisor/1"; + static final int MAX_FRAME = 4 * 1024 * 1024; + static final int MAX_TRANSCRIPT = 8 * 1024 * 1024; + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private SupervisorProtocol() {} + + /*************************************************************************** + ** Reject excessive nesting before JSON allocates its recursive structure. + ***************************************************************************/ + static JSONObject parse(String text) throws ScriptingException + { + if(text.getBytes(StandardCharsets.UTF_8).length > MAX_FRAME) + { + throw refused(); + } + int depth = 0; + boolean string = false; + boolean escape = false; + for(int i = 0; i < text.length(); i++) + { + char value = text.charAt(i); + if(string) + { + if(escape) + { + escape = false; + } + else if(value == '\\') + { + escape = true; + } + else if(value == '"') + { + string = false; + } + } + else if(value == '"') + { + string = true; + } + else if(value == '{' || value == '[') + { + if(++depth > 16) + { + throw refused(); + } + } + else if(value == '}' || value == ']') + { + depth--; + } + } + try + { + var tokenizer = new JSONTokener(text, new JSONParserConfiguration().withStrictMode(true).withOverwriteDuplicateKey(false)); + JSONObject object = new JSONObject(tokenizer); + if(tokenizer.nextClean() != 0) + { + throw refused(); + } + return (object); + } + catch(RuntimeException failure) + { + throw refused(); + } + } + + /*************************************************************************** + ** Every envelope has an exact key set; null and coercion are not accepted. + ***************************************************************************/ + static void keys(JSONObject object, String... keys) throws ScriptingException + { + if(!object.keySet().equals(Set.of(keys))) + { + throw refused(); + } + } + + /*************************************************************************** + ** Read an opaque bounded identity without JSON string coercion. + ***************************************************************************/ + static String identity(JSONObject object, String key) throws ScriptingException + { + Object value = object.opt(key); + if(!(value instanceof String text) || !text.matches("[A-Za-z0-9._:-]{1,128}")) + { + throw refused(); + } + return (text); + } + + /*************************************************************************** + ** Compare exact strings, never coercing scalar JSON values. + ***************************************************************************/ + static void equal(JSONObject object, String key, String expected) throws ScriptingException + { + if(!expected.equals(object.opt(key))) + { + throw refused(); + } + } + + /*************************************************************************** + ** Read a nested object without accepting encoded JSON strings. + ***************************************************************************/ + static JSONObject object(JSONObject parent, String key) throws ScriptingException + { + if(!(parent.opt(key) instanceof JSONObject value)) + { + throw refused(); + } + return (value); + } + + /*************************************************************************** + ** All malformed supervisor messages use a value-free diagnostic. + ***************************************************************************/ + static ScriptingException refused() + { + return (new ScriptingException(ScriptingException.Kind.INTEGRITY, "Invalid supervisor transport message")); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/UnconfiguredWorkerProvider.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/UnconfiguredWorkerProvider.java new file mode 100644 index 0000000..214dd23 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/UnconfiguredWorkerProvider.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + + +import com.kingsrook.qbits.scripting.ScriptingException; + + +/******************************************************************************* + ** The default provider: there is no worker, so every operation fails closed. It + ** is what the QBit uses until a host configures a real, qualified provider. + *******************************************************************************/ +public class UnconfiguredWorkerProvider implements WorkerProvider +{ + private static final String MESSAGE = "No isolated script worker is configured; scripts cannot be built or run"; + + + + /*************************************************************************** + ** Never qualified. + ***************************************************************************/ + @Override + public String qualificationId() + { + return (null); + } + + + + /*************************************************************************** + ** Always unavailable. + ***************************************************************************/ + @Override + public String compile(String compileRequestJson) throws ScriptingException + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, MESSAGE); + } + + + + /*************************************************************************** + ** Always unavailable. + ***************************************************************************/ + @Override + public String execute(String executeRequestJson, BrokerChannel broker) throws ScriptingException + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, MESSAGE); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/WarmWorkerMaintenance.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WarmWorkerMaintenance.java new file mode 100644 index 0000000..7a5b037 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WarmWorkerMaintenance.java @@ -0,0 +1,99 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + + +import java.time.Duration; +import java.util.List; +import com.kingsrook.qbits.scripting.ScriptingException; + + +/******************************************************************************* + ** Instance-owned cold maintenance, never invoked from record execution. A fresh + ** acknowledgment attests the same authenticated cache epoch, not guest authority. + *******************************************************************************/ +public interface WarmWorkerMaintenance extends WarmWorkerPreparation, AutoCloseable +{ + /*************************************************************************** + ** Check idle authenticated sessions without executing code or exposing keys. + ***************************************************************************/ + void heartbeat(Duration budget) throws ScriptingException; + + /*************************************************************************** + ** Heartbeat that may leave an idle session without a wire exchange when a + ** completed, retired exchange already confirmed it within the given window. + ** The window is derived from the native readiness policy; the default + ** preserves existing providers by always performing the full heartbeat. + ***************************************************************************/ + default void heartbeat(Duration budget, Duration confirmedWithin) throws ScriptingException + { + heartbeat(budget); + } + + /*************************************************************************** + ** Prepare within an explicit whole-operation budget, including every copy. + ***************************************************************************/ + void prepareVerified(Artifact artifact, Duration budget) throws ScriptingException; + + /*************************************************************************** + ** Optional cold capacity reclamation for this complete required inventory. + ** Implementations may retire idle cache sessions only; busy invocations must + ** finish normally. The lifecycle revalidates/reprepares every required pin. + ** Default preserves existing providers; aggregate readiness remains mandatory. + ***************************************************************************/ + default void retireUnused(List required, Duration budget) throws ScriptingException {} + + /*************************************************************************** + ** Local-only exact code/copy/profile readiness with bounded acknowledgment age. + ***************************************************************************/ + boolean isPrepared(String sha256, String entryPoint, String toolchainId, Duration maxAge); + + /*************************************************************************** + ** All required identities must be ready in one local cache snapshot. No code, + ** transport, callbacks into QQQ, or guest allocation belongs in this check. + ***************************************************************************/ + boolean arePrepared(List identities, Duration maxAge); + + /*************************************************************************** + ** Immutable code locator only, never a bearer credential or artifact bytes. + ***************************************************************************/ + record CodeIdentity(String sha256, String entryPoint, String toolchainId) + { + /*********************************************************************** + ** Exact bounded digest, portable TypeScript entrypoint and pinned toolchain. + ***********************************************************************/ + public CodeIdentity + { + if(sha256 == null || !sha256.matches("[a-f0-9]{64}") || entryPoint == null + || !entryPoint.matches("[A-Za-z0-9_./-]{1,197}\\.ts") || entryPoint.startsWith("/") || entryPoint.contains("..") + || toolchainId == null || toolchainId.isBlank() || toolchainId.length() > 200) + { + throw new IllegalArgumentException("Invalid warm code identity"); + } + } + } + + /*************************************************************************** + ** Close this instance's transport after application producers stop. + ***************************************************************************/ + @Override + void close(); +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/WarmWorkerPreparation.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WarmWorkerPreparation.java new file mode 100644 index 0000000..e2b20f4 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WarmWorkerPreparation.java @@ -0,0 +1,97 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + + +import java.nio.charset.StandardCharsets; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; + + +/******************************************************************************* + ** Optional cold-lane preparation of coordinator-verified immutable code. This is + ** not execution or qualification. Implementations enforce their operator-owned + ** profile, copy count, admission and deadline, and never receive invocation + ** context, credentials or capability tokens. A failure must be reported; no hot + ** execution fallback or implicit preparation is permitted. + *******************************************************************************/ +public interface WarmWorkerPreparation +{ + /*************************************************************************** + ** Populate bounded warm capacity for the exact immutable artifact. + ***************************************************************************/ + void prepareVerified(Artifact artifact) throws ScriptingException; + + /*************************************************************************** + ** Authoring holds this bounded local use across its next dispatch/commit. + ** Existing non-reclaiming providers keep their compatible preparation path. + ** This is never called by hot record execution or render/context invocation. + ***************************************************************************/ + default UseLease prepareForUse(Artifact artifact) throws ScriptingException + { + prepareVerified(artifact); + return (() -> {}); + } + + /*************************************************************************** + ** Nonthrowing, idempotent local release; no transport or authority is carried. + ***************************************************************************/ + @FunctionalInterface + interface UseLease extends AutoCloseable + { + /*************************************************************************** + ** Release only this authoring reservation. + ***************************************************************************/ + @Override + void close(); + } + + /*************************************************************************** + ** Code identity only; construction verifies bytes, not coordinator authority. + ** Services must verify the signed artifact before invoking this SPI. + ***************************************************************************/ + record Artifact(String sha256, String entryPoint, String bundle, String toolchainId) + { + /*********************************************************************** + ** Preserve the worker's TypeScript entrypoint and bounded exact bytes. + ***********************************************************************/ + public Artifact + { + if(sha256 == null || !sha256.matches("[a-f0-9]{64}") || entryPoint == null + || !entryPoint.matches("[A-Za-z0-9_./-]{1,197}\\.ts") || entryPoint.startsWith("/") || entryPoint.contains("..") + || bundle == null || bundle.isEmpty() || bundle.length() > WorkerProtocol.MAX_BUNDLE_CHARS + || bundle.getBytes(StandardCharsets.UTF_8).length > WorkerProtocol.MAX_EXECUTE_MESSAGE_BYTES + || !sha256.equals(BuildCoordinator.sha256Hex(bundle)) || toolchainId == null || toolchainId.isBlank() || toolchainId.length() > 200) + { + throw new IllegalArgumentException("Invalid immutable preparation artifact"); + } + } + + /*********************************************************************** + ** Source bytes are never ordinary diagnostics. + ***********************************************************************/ + @Override + public String toString() + { + return ("Artifact[redacted]"); + } + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerMessages.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerMessages.java new file mode 100644 index 0000000..baccea5 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerMessages.java @@ -0,0 +1,162 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + + +import java.util.List; +import com.kingsrook.qbits.scripting.hooks.HookResult; +import com.kingsrook.qbits.scripting.source.SourceFile; + + +/******************************************************************************* + ** The typed messages exchanged with an external isolated worker. Nothing here + ** can carry a credential, a session, a database handle or a Java object: only + ** strings, numbers and bounded JSON text. Encoding and bounds live in + ** WorkerProtocol. + *******************************************************************************/ +public final class WorkerMessages +{ + /*************************************************************************** + ** Utility holder. + ***************************************************************************/ + private WorkerMessages() + { + } + + + + /******************************************************************************* + ** Build-side request: compile these files, strictly, with this toolchain. + *******************************************************************************/ + public record CompileRequest(String requestId, String toolchainId, String entryPoint, String sourceSha256, List files) + { + /*************************************************************************** + ** Freeze the list. + ***************************************************************************/ + public CompileRequest + { + files = List.copyOf(files); + } + } + + + + /******************************************************************************* + ** One compiler diagnostic, bounded by the decoder. + *******************************************************************************/ + public record Diagnostic(String file, int line, int column, String code, String message) + { + } + + + + /******************************************************************************* + ** Build-side answer. The worker asserts strict mode and its toolchain; the + ** coordinator does not believe either without comparing to its own policy. + *******************************************************************************/ + public record CompileResponse(String requestId, boolean success, boolean strict, String toolchainId, List diagnostics, String bundle, String bundleSha256, String sourceMap) + { + /*************************************************************************** + ** Freeze the list. + ***************************************************************************/ + public CompileResponse + { + diagnostics = List.copyOf(diagnostics); + } + } + + + + /******************************************************************************* + ** Hard execution limits the worker must enforce. The host clamps what it + ** sends to ABSOLUTE maxima, so configuration cannot ask for unbounded work. + *******************************************************************************/ + public record ExecutionLimits(int wallMillis, int cpuMillis, long memoryBytes, int outputBytes, int brokerCalls) + { + public static final int MAX_WALL_MILLIS = 10_000; + public static final int MAX_CPU_MILLIS = 5_000; + public static final long MAX_MEMORY_BYTES = 256L * 1024 * 1024; + public static final int MAX_OUTPUT_BYTES = 128 * 1024; + public static final int MAX_BROKER_CALLS = 50; + + + + /*************************************************************************** + ** Reject non-positive or over-maximum limits. + ***************************************************************************/ + public ExecutionLimits + { + if(wallMillis < 1 || wallMillis > MAX_WALL_MILLIS + || cpuMillis < 1 || cpuMillis > MAX_CPU_MILLIS + || memoryBytes < 1 || memoryBytes > MAX_MEMORY_BYTES + || outputBytes < 1 || outputBytes > MAX_OUTPUT_BYTES + || brokerCalls < 0 || brokerCalls > MAX_BROKER_CALLS) + { + throw new IllegalArgumentException("Execution limits are outside the supported range"); + } + } + + + + /*************************************************************************** + ** Conservative defaults for record hooks. + ***************************************************************************/ + public static ExecutionLimits hookDefaults() + { + return (new ExecutionLimits(2_000, 1_000, 64L * 1024 * 1024, 64 * 1024, 10)); + } + } + + + + /******************************************************************************* + ** Run-side request. The worker gets the bundle plus its expected id and hash and + ** must verify the hash before running anything. The capability token is opaque + ** and is the ONLY authority the worker holds; there is no credential field. + ** payloadJson is a JSON object holding proposed and prior record values. + *******************************************************************************/ + public record ExecuteRequest(String requestId, String artifactId, String artifactSha256, String bundle, String entryPoint, String event, String payloadJson, String capabilityToken, ExecutionLimits limits) + { + } + + + + /******************************************************************************* + ** How a run ended, as reported by the worker. + *******************************************************************************/ + public enum Outcome + { + OK, + SCRIPT_ERROR, + LIMIT_EXCEEDED, + CRASHED + } + + + + /******************************************************************************* + ** Run-side answer. errorCode/errorMessage are bounded but untrusted and must + ** not enter project history or execution diagnostics. Hook exists only for OK. + *******************************************************************************/ + public record ExecuteResponse(String requestId, Outcome outcome, HookResult hook, String errorCode, String errorMessage, int cpuMillis, int wallMillis, String contextJson) + { + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerProtocol.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerProtocol.java new file mode 100644 index 0000000..c797c75 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerProtocol.java @@ -0,0 +1,589 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.capability.BrokerRequest; +import com.kingsrook.qbits.scripting.hooks.HookResult; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.CompileRequest; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.CompileResponse; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Diagnostic; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecuteRequest; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecuteResponse; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Outcome; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + + +/******************************************************************************* + ** Bounded JSON codec for the worker boundary. Every inbound message is size + ** checked BEFORE parsing, depth checked, version checked, unknown-key checked and + ** field-bounded; malformed or oversized input fails closed with a generic error + ** that never echoes the payload. Every outbound message is size checked too. + ** + ** This is a protocol contract, not a transport: how bytes reach a worker is the + ** provider's business (see WorkerProvider). + *******************************************************************************/ +public final class WorkerProtocol +{ + public static final String VERSION = "qbit-scripting-worker/1"; + + public static final int MAX_COMPILE_MESSAGE_BYTES = 4 * 1024 * 1024; + public static final int MAX_EXECUTE_MESSAGE_BYTES = 2 * 1024 * 1024; + public static final int MAX_RESULT_MESSAGE_BYTES = 128 * 1024; + public static final int MAX_BROKER_MESSAGE_BYTES = 64 * 1024; + public static final int MAX_BUNDLE_CHARS = 1024 * 1024; + public static final int MAX_PAYLOAD_BYTES = 256 * 1024; + public static final int MAX_JSON_DEPTH = 12; + public static final int MAX_DIAGNOSTICS = 100; + public static final int MAX_TEXT_CHARS = 500; + public static final int MAX_PATCH_STRING_CHARS = 2000; + + private static final Pattern IDENTIFIER = Pattern.compile("[A-Za-z0-9._:-]{1,128}"); + private static final Pattern FIELD_NAME = Pattern.compile("[A-Za-z][A-Za-z0-9_]{0,99}"); + private static final Pattern HEX64 = Pattern.compile("[0-9a-f]{64}"); + + + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private WorkerProtocol() + { + } + + + + /*************************************************************************** + ** Encode a compile request. + ***************************************************************************/ + public static String encodeCompile(CompileRequest request) throws ScriptingException + { + JSONArray files = new JSONArray(); + for(SourceFile file : request.files()) + { + files.put(new JSONObject().put("path", file.path()).put("content", file.content())); + } + JSONObject object = envelope("compile", request.requestId()) + .put("toolchainId", request.toolchainId()) + .put("entryPoint", request.entryPoint()) + .put("sourceSha256", request.sourceSha256()) + .put("strict", true) + .put("files", files); + return (bounded(object.toString(), MAX_COMPILE_MESSAGE_BYTES)); + } + + + + /*************************************************************************** + ** Decode and validate a compile answer. + ***************************************************************************/ + public static CompileResponse decodeCompileResponse(String json) throws ScriptingException + { + try + { + JSONObject o = parse(json, MAX_COMPILE_MESSAGE_BYTES, "compileResult"); + onlyKeys(o, Set.of("protocol", "type", "requestId", "success", "strict", "toolchainId", "diagnostics", "bundle", "bundleSha256", "sourceMap")); + + List diagnostics = new ArrayList<>(); + JSONArray array = o.optJSONArray("diagnostics"); + if(array != null) + { + if(array.length() > MAX_DIAGNOSTICS) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Too many diagnostics"); + } + for(int i = 0; i < array.length(); i++) + { + JSONObject d = array.getJSONObject(i); + onlyKeys(d, Set.of("file", "line", "column", "code", "message")); + diagnostics.add(new Diagnostic(text(d, "file", 200), Math.max(0, d.optInt("line", 0)), Math.max(0, d.optInt("column", 0)), text(d, "code", 40), text(d, "message", MAX_TEXT_CHARS))); + } + } + + boolean success = o.getBoolean("success"); + String bundle = o.has("bundle") ? o.getString("bundle") : null; + String bundleSha256 = o.has("bundleSha256") ? o.getString("bundleSha256") : null; + String sourceMap = o.has("sourceMap") ? o.getString("sourceMap") : null; + if(success) + { + if(bundle == null || bundle.isEmpty() || bundle.length() > MAX_BUNDLE_CHARS || bundleSha256 == null || !HEX64.matcher(bundleSha256).matches()) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "A successful build needs a bounded bundle and its SHA-256"); + } + } + else if(bundle != null) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "A failed build must not return a bundle"); + } + if(sourceMap != null && sourceMap.length() > MAX_BUNDLE_CHARS) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Source map is too large"); + } + return (new CompileResponse(identifier(o, "requestId"), success, o.getBoolean("strict"), identifier(o, "toolchainId"), diagnostics, bundle, bundleSha256, sourceMap)); + } + catch(JSONException e) + { + throw malformed(); + } + } + + + + /*************************************************************************** + ** Encode an execute request. The payload must already be a bounded JSON object. + ***************************************************************************/ + public static String encodeExecute(ExecuteRequest request) throws ScriptingException + { + if(request.bundle().length() > MAX_BUNDLE_CHARS || request.payloadJson().getBytes(StandardCharsets.UTF_8).length > MAX_PAYLOAD_BYTES) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Execution input is too large"); + } + JSONObject payload; + try + { + checkDepth(request.payloadJson()); + payload = new JSONObject(request.payloadJson()); + } + catch(JSONException e) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Execution payload is not a JSON object"); + } + JSONObject limits = new JSONObject() + .put("wallMillis", request.limits().wallMillis()) + .put("cpuMillis", request.limits().cpuMillis()) + .put("memoryBytes", request.limits().memoryBytes()) + .put("outputBytes", request.limits().outputBytes()) + .put("brokerCalls", request.limits().brokerCalls()); + JSONObject object = envelope("execute", request.requestId()) + .put("artifactId", request.artifactId()) + .put("artifactSha256", request.artifactSha256()) + .put("bundle", request.bundle()) + .put("entryPoint", request.entryPoint()) + .put("event", request.event()) + .put("payload", payload) + .put("capabilityToken", request.capabilityToken()) + .put("limits", limits); + return (bounded(object.toString(), MAX_EXECUTE_MESSAGE_BYTES)); + } + + + + /*************************************************************************** + ** Decode and validate a run answer into a typed outcome and hook result. + ***************************************************************************/ + public static ExecuteResponse decodeExecuteResponse(String json) throws ScriptingException + { + return (decodeExecuteResponse(json, false)); + } + + + + /*************************************************************************** + ** Context executions return a bounded JSON object; hook executions retain + ** their strict ALLOW/PATCH/REJECT contract. + ***************************************************************************/ + public static ExecuteResponse decodeExecuteResponse(String json, boolean context) throws ScriptingException + { + try + { + JSONObject o = parse(json, MAX_RESULT_MESSAGE_BYTES, "executeResult"); + onlyKeys(o, Set.of("protocol", "type", "requestId", "outcome", "result", "error", "usage")); + Outcome outcome = Outcome.valueOf(o.getString("outcome")); + + HookResult hook = null; + String contextJson = null; + if(outcome == Outcome.OK) + { + if(context) + { + contextJson = checkPayload(o.getJSONObject("result").toString()); + } + else + { + hook = decodeHookResult(o.getJSONObject("result")); + } + } + + String errorCode = null; + String errorMessage = null; + JSONObject error = o.optJSONObject("error"); + if(error != null) + { + onlyKeys(error, Set.of("code", "message")); + errorCode = text(error, "code", 40); + errorMessage = text(error, "message", MAX_TEXT_CHARS); + } + + int cpu = 0; + int wall = 0; + JSONObject usage = o.optJSONObject("usage"); + if(usage != null) + { + onlyKeys(usage, Set.of("cpuMillis", "wallMillis")); + cpu = Math.max(0, usage.optInt("cpuMillis", 0)); + wall = Math.max(0, usage.optInt("wallMillis", 0)); + } + return (new ExecuteResponse(identifier(o, "requestId"), outcome, hook, errorCode, errorMessage, cpu, wall, contextJson)); + } + catch(JSONException | IllegalArgumentException e) + { + throw malformed(); + } + } + + + + /*************************************************************************** + ** Decode and validate a worker's broker read request. + ***************************************************************************/ + public static BrokerRequest decodeBrokerRequest(String json) throws ScriptingException + { + try + { + JSONObject o = parse(json, MAX_BROKER_MESSAGE_BYTES, "broker"); + onlyKeys(o, Set.of("protocol", "type", "requestId", "operation", "token", "table", "fields", "criteria", "limit")); + if(!"query".equals(o.getString("operation"))) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Only query is supported"); + } + + List fields = new ArrayList<>(); + JSONArray fieldArray = o.getJSONArray("fields"); + for(int i = 0; i < fieldArray.length(); i++) + { + fields.add(fieldName(fieldArray.getString(i))); + } + if(fields.isEmpty()) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "At least one field is required"); + } + + List criteria = new ArrayList<>(); + JSONArray criteriaArray = o.optJSONArray("criteria"); + if(criteriaArray != null) + { + if(criteriaArray.length() > BrokerRequest.MAX_CRITERIA) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Too many criteria"); + } + for(int i = 0; i < criteriaArray.length(); i++) + { + criteria.add(decodeCriterion(criteriaArray.getJSONObject(i))); + } + } + + return (new BrokerRequest(text(o, "token", 200), fieldName(o.getString("table")), fields, criteria, o.getInt("limit"))); + } + catch(JSONException e) + { + throw malformed(); + } + } + + + + /*************************************************************************** + ** Encode a successful broker answer. + ***************************************************************************/ + public static String encodeBrokerResponse(List> rows, boolean truncated) + { + JSONArray array = new JSONArray(); + for(Map row : rows) + { + JSONObject object = new JSONObject(); + for(Map.Entry entry : row.entrySet()) + { + object.put(entry.getKey(), entry.getValue() == null ? JSONObject.NULL : entry.getValue()); + } + array.put(object); + } + return (new JSONObject().put("protocol", VERSION).put("type", "brokerResult").put("ok", true).put("rows", array).put("truncated", truncated).toString()); + } + + + + /*************************************************************************** + ** Encode a refused broker answer with a generic, bounded message. + ***************************************************************************/ + public static String encodeBrokerError(String code, String message) + { + String safe = message == null ? "" : message.substring(0, Math.min(message.length(), MAX_TEXT_CHARS)); + return (new JSONObject().put("protocol", VERSION).put("type", "brokerResult").put("ok", false).put("code", code).put("message", safe).toString()); + } + + + + /*************************************************************************** + ** Validate a standalone payload object (proposed/prior record JSON) and return + ** it in canonical form. Used by the host before it ever reaches a worker. + ***************************************************************************/ + public static String checkPayload(String json) throws ScriptingException + { + if(json == null || json.getBytes(StandardCharsets.UTF_8).length > MAX_PAYLOAD_BYTES) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Payload is too large"); + } + try + { + checkDepth(json); + return (new JSONObject(json).toString()); + } + catch(JSONException e) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Payload is not a JSON object"); + } + } + + + + /*************************************************************************** + ** Decode one broker filter condition. + ***************************************************************************/ + private static BrokerRequest.Criterion decodeCriterion(JSONObject c) throws ScriptingException + { + onlyKeys(c, Set.of("field", "operator", "values")); + String operator = c.getString("operator"); + if(!BrokerRequest.OPERATORS.contains(operator)) + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Operator is not supported"); + } + JSONArray valueArray = c.getJSONArray("values"); + if(valueArray.length() < 1 || valueArray.length() > BrokerRequest.MAX_VALUES) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Criterion needs between 1 and " + BrokerRequest.MAX_VALUES + " values"); + } + List values = new ArrayList<>(); + for(int i = 0; i < valueArray.length(); i++) + { + Object value = valueArray.get(i); + if(!(value instanceof String || value instanceof Number || value instanceof Boolean) || (value instanceof String s && s.length() > MAX_PATCH_STRING_CHARS)) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Criterion values must be short strings, numbers or booleans"); + } + values.add(value); + } + return (new BrokerRequest.Criterion(fieldName(c.getString("field")), operator, values)); + } + + + + /*************************************************************************** + ** Decode and bound a hook decision. + ***************************************************************************/ + private static HookResult decodeHookResult(JSONObject r) throws ScriptingException + { + onlyKeys(r, Set.of("action", "patch", "message")); + HookResult.Action action = HookResult.Action.valueOf(r.getString("action")); + String message = r.has("message") ? text(r, "message", HookResult.MAX_MESSAGE_CHARS) : null; + Map patch = new LinkedHashMap<>(); + JSONObject patchObject = r.optJSONObject("patch"); + if(patchObject != null) + { + for(String key : patchObject.keySet()) + { + Object value = patchObject.get(key); + if(value == JSONObject.NULL) + { + value = null; + } + boolean allowed = value == null || value instanceof Number || value instanceof Boolean || (value instanceof String s && s.length() <= MAX_PATCH_STRING_CHARS); + if(!allowed) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Patch values must be short strings, numbers, booleans or null"); + } + patch.put(fieldName(key), value); + } + } + return (new HookResult(action, patch, message)); + } + + + + /*************************************************************************** + ** Common envelope fields. + ***************************************************************************/ + private static JSONObject envelope(String type, String requestId) + { + return (new JSONObject().put("protocol", VERSION).put("type", type).put("requestId", requestId)); + } + + + + /*************************************************************************** + ** Size check, then depth check, then parse, then version/type check. + ***************************************************************************/ + static JSONObject parse(String json, int maxBytes, String expectedType) throws ScriptingException + { + if(json == null || json.getBytes(StandardCharsets.UTF_8).length > maxBytes) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Worker message is missing or too large"); + } + checkDepth(json); + JSONObject o = new JSONObject(json); + if(!VERSION.equals(o.optString("protocol")) || (expectedType != null && !expectedType.equals(o.optString("type")))) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Unexpected worker protocol version or message type"); + } + return (o); + } + + + + /*************************************************************************** + ** Reject deeply nested input without recursing: count brackets outside strings. + ***************************************************************************/ + private static void checkDepth(String json) throws ScriptingException + { + int depth = 0; + boolean inString = false; + boolean escaped = false; + for(int i = 0; i < json.length(); i++) + { + char c = json.charAt(i); + if(inString) + { + if(escaped) + { + escaped = false; + } + else if(c == '\\') + { + escaped = true; + } + else if(c == '"') + { + inString = false; + } + } + else if(c == '"') + { + inString = true; + } + else if(c == '{' || c == '[') + { + depth++; + if(depth > MAX_JSON_DEPTH) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Worker message is nested too deeply"); + } + } + else if(c == '}' || c == ']') + { + depth--; + } + } + } + + + + /*************************************************************************** + ** Reject any key the contract does not define. + ***************************************************************************/ + private static void onlyKeys(JSONObject o, Set allowed) throws ScriptingException + { + for(String key : o.keySet()) + { + if(!allowed.contains(key)) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Unexpected field in worker message"); + } + } + } + + + + /*************************************************************************** + ** A bounded string value. + ***************************************************************************/ + private static String text(JSONObject o, String key, int max) throws ScriptingException + { + String value = o.getString(key); + if(value.length() > max) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "A worker message field is too long"); + } + return (value); + } + + + + /*************************************************************************** + ** A short safe identifier value. + ***************************************************************************/ + private static String identifier(JSONObject o, String key) throws ScriptingException + { + String value = o.getString(key); + if(!IDENTIFIER.matcher(value).matches()) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "A worker message identifier is invalid"); + } + return (value); + } + + + + /*************************************************************************** + ** A table or field name. + ***************************************************************************/ + private static String fieldName(String value) throws ScriptingException + { + if(!FIELD_NAME.matcher(value).matches()) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "A name in a worker message is invalid"); + } + return (value); + } + + + + /*************************************************************************** + ** Outbound size guard. + ***************************************************************************/ + private static String bounded(String json, int max) throws ScriptingException + { + if(json.getBytes(StandardCharsets.UTF_8).length > max) + { + throw new ScriptingException(ScriptingException.Kind.LIMIT, "Message to worker exceeds " + max + " bytes"); + } + return (json); + } + + + + /*************************************************************************** + ** Generic parse failure that never echoes the payload. + ***************************************************************************/ + private static ScriptingException malformed() + { + return (new ScriptingException(ScriptingException.Kind.INVALID, "Malformed worker message")); + } +} diff --git a/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerProvider.java b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerProvider.java new file mode 100644 index 0000000..1aaacb4 --- /dev/null +++ b/base/src/main/java/com/kingsrook/qbits/scripting/worker/WorkerProvider.java @@ -0,0 +1,92 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + + +import java.time.Duration; +import com.kingsrook.qbits.scripting.ScriptingException; + + +/******************************************************************************* + ** The seam to an EXTERNAL, isolated compile/execute worker (separate process or + ** machine, unprivileged identity, no host mounts, no secrets, enforced CPU, + ** memory, output and wall-clock limits, destroyed after each run). This QBit + ** ships no implementation that runs code: the JVM never evaluates a script, so + ** there is no Nashorn, no embedded JavaScript engine, no shell and no + ** regex-based "type check" here or behind this interface. + ** + ** The boundary is JSON text conforming to WorkerProtocol. Implementations must + ** fail with ScriptingException (UNAVAILABLE) when the worker cannot be reached + ** or does not answer in time; the service never falls back to anything else. + *******************************************************************************/ +public interface WorkerProvider +{ + /******************************************************************************* + ** Relay for a worker's capability reads: takes a broker request JSON, returns a + ** broker response JSON. Provided by the host per run. Callbacks must run + ** synchronously on the initiating QQQ thread and session; asynchronous + ** providers must marshal requests back to that caller. + *******************************************************************************/ + interface BrokerChannel + { + /*************************************************************************** + ** Answer one worker read request; never throws. + ***************************************************************************/ + String handle(String brokerRequestJson); + } + + + + /*************************************************************************** + ** Identifier of the completed external qualification of this worker + ** deployment (hardening review, escape/exhaustion probes). Null or blank means + ** unqualified. The host compares it to the evidence id it was configured to + ** require and refuses to execute on any mismatch. + ***************************************************************************/ + String qualificationId(); + + + + /*************************************************************************** + ** Trusted transport budget including isolated-worker setup. This does not + ** change the script's wall/CPU limits. The host bounds it at two minutes; + ** its per-execution token is always closed immediately after execution. + ***************************************************************************/ + default Duration executionTransportBudget() + { + return (Duration.ofSeconds(15)); + } + + + + /*************************************************************************** + ** Send a compile request JSON; return the compile result JSON. + ***************************************************************************/ + String compile(String compileRequestJson) throws ScriptingException; + + + + /*************************************************************************** + ** Send an execute request JSON; return the execute result JSON. Any worker + ** capability read goes through the supplied channel and nowhere else. + ***************************************************************************/ + String execute(String executeRequestJson, BrokerChannel broker) throws ScriptingException; +} diff --git a/base/src/main/resources/db/mysql/001_scripting.sql b/base/src/main/resources/db/mysql/001_scripting.sql new file mode 100644 index 0000000..f585d1f --- /dev/null +++ b/base/src/main/resources/db/mysql/001_scripting.sql @@ -0,0 +1,142 @@ +-- Fresh-install MySQL 8.4 schema, applied by the host's migration identity. +-- Existing core script/scriptRevision/scriptRevisionFile tables stay host-owned. +-- Configure those tables on the same InnoDB backend and retain their unique keys. +-- Application identities need DML only, never ALTER/TRIGGER/DROP privileges. +-- Delimiters also preserve compound trigger statements for mysql CLI migrations. +DELIMITER $$ + +CREATE TABLE scripting_build ( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + script_id INTEGER NOT NULL, + script_revision_id INTEGER NOT NULL, + source_sha256 VARCHAR(64) NOT NULL, + manifest_json LONGTEXT NOT NULL, + manifest_attestation VARCHAR(64) NOT NULL, + toolchain_id VARCHAR(255) NOT NULL, + status VARCHAR(32) NOT NULL CHECK (status IN ('SUCCEEDED', 'FAILED')), + diagnostics_json LONGTEXT NOT NULL, + requested_by VARCHAR(2000) NOT NULL, + created_at TIMESTAMP(6) NOT NULL, + INDEX scripting_build_project (script_id, created_at) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin$$ + +CREATE TABLE scripting_artifact ( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + build_id INTEGER NOT NULL UNIQUE, + script_id INTEGER NOT NULL, + version INTEGER NOT NULL CHECK (version > 0), + source_sha256 VARCHAR(64) NOT NULL, + toolchain_id VARCHAR(255) NOT NULL, + entry_point VARCHAR(100) NOT NULL, + coordinator_id VARCHAR(255) NOT NULL, + bundle LONGTEXT NOT NULL, + bundle_sha256 VARCHAR(64) NOT NULL, + size_bytes INTEGER NOT NULL CHECK (size_bytes > 0), + source_map LONGTEXT, + attestation VARCHAR(64) NOT NULL, + created_at TIMESTAMP(6) NOT NULL, + UNIQUE (script_id, version), + FOREIGN KEY (build_id) REFERENCES scripting_build(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin$$ + +CREATE TABLE scripting_publication ( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + script_id INTEGER NOT NULL, + version INTEGER NOT NULL CHECK (version > 0), + build_id INTEGER NOT NULL, + artifact_id INTEGER NOT NULL, + published_by VARCHAR(2000) NOT NULL, + published_at TIMESTAMP(6) NOT NULL, + note VARCHAR(500), + UNIQUE (script_id, version), + FOREIGN KEY (build_id) REFERENCES scripting_build(id), + FOREIGN KEY (artifact_id) REFERENCES scripting_artifact(id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin$$ + +CREATE TABLE scripting_binding ( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(2000) NOT NULL, + table_name VARCHAR(255) NOT NULL, + event VARCHAR(32) NOT NULL CONSTRAINT scripting_binding_event_check CHECK (event IN ('PRE_INSERT', 'PRE_UPDATE', 'PRE_DELETE', 'POST_INSERT', 'POST_UPDATE', 'POST_DELETE', 'AFTER_INSERT', 'AFTER_UPDATE', 'AFTER_DELETE')), + publication_id INTEGER NOT NULL, + grants_json LONGTEXT NOT NULL, + patchable_fields LONGTEXT NOT NULL, + enabled INTEGER NOT NULL CHECK (enabled IN (0, 1)), + active_slot VARCHAR(300) UNIQUE, + created_by VARCHAR(2000) NOT NULL, + created_at TIMESTAMP(6) NOT NULL, + CHECK ((enabled = 1 AND active_slot IS NOT NULL AND active_slot = CONCAT(table_name, ':', event)) + OR (enabled = 0 AND active_slot IS NULL)), + FOREIGN KEY (publication_id) REFERENCES scripting_publication(id), + INDEX scripting_binding_event (table_name, event, enabled) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin$$ + +CREATE TABLE scripting_execution ( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + operation_id VARCHAR(64) NOT NULL, + kind VARCHAR(32) NOT NULL, + build_id INTEGER NOT NULL, + binding_id INTEGER, + publication_id INTEGER, + artifact_id INTEGER, + actor_id VARCHAR(2000) NOT NULL, + event VARCHAR(32) NOT NULL, + status VARCHAR(32) NOT NULL, + duration_ms INTEGER, + message VARCHAR(500), + result_action VARCHAR(16), + created_at TIMESTAMP(6) NOT NULL, + FOREIGN KEY (build_id) REFERENCES scripting_build(id), + FOREIGN KEY (binding_id) REFERENCES scripting_binding(id), + FOREIGN KEY (publication_id) REFERENCES scripting_publication(id), + FOREIGN KEY (artifact_id) REFERENCES scripting_artifact(id), + INDEX scripting_execution_build (build_id, created_at), + INDEX scripting_execution_operation (operation_id, id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin$$ + +CREATE TABLE scripting_import ( + id INTEGER AUTO_INCREMENT PRIMARY KEY, + script_id INTEGER NOT NULL, + script_revision_id INTEGER NOT NULL UNIQUE, + repository VARCHAR(201) NOT NULL, + commit_sha VARCHAR(40) NOT NULL CHECK (REGEXP_LIKE(commit_sha, '^[0-9a-f]{40}$', 'c')), + subdirectory VARCHAR(200) NOT NULL, + source_sha256 VARCHAR(64) NOT NULL CHECK (REGEXP_LIKE(source_sha256, '^[0-9a-f]{64}$', 'c')), + imported_by VARCHAR(2000) NOT NULL, + created_at TIMESTAMP(6) NOT NULL, + INDEX scripting_import_project (script_id, id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin$$ + +CREATE TRIGGER scripting_build_no_update BEFORE UPDATE ON scripting_build FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting provenance is append only'$$ +CREATE TRIGGER scripting_build_no_delete BEFORE DELETE ON scripting_build FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting provenance is append only'$$ +CREATE TRIGGER scripting_artifact_no_update BEFORE UPDATE ON scripting_artifact FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting provenance is append only'$$ +CREATE TRIGGER scripting_artifact_no_delete BEFORE DELETE ON scripting_artifact FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting provenance is append only'$$ +CREATE TRIGGER scripting_publication_no_update BEFORE UPDATE ON scripting_publication FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting provenance is append only'$$ +CREATE TRIGGER scripting_publication_no_delete BEFORE DELETE ON scripting_publication FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting provenance is append only'$$ +CREATE TRIGGER scripting_execution_no_update BEFORE UPDATE ON scripting_execution FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting history is append only'$$ +CREATE TRIGGER scripting_execution_no_delete BEFORE DELETE ON scripting_execution FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting history is append only'$$ +CREATE TRIGGER scripting_import_no_update BEFORE UPDATE ON scripting_import FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting provenance is append only'$$ +CREATE TRIGGER scripting_import_no_delete BEFORE DELETE ON scripting_import FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting provenance is append only'$$ +CREATE TRIGGER scripting_binding_no_delete BEFORE DELETE ON scripting_binding FOR EACH ROW + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Scripting bindings are retained'$$ +CREATE TRIGGER scripting_binding_identity BEFORE UPDATE ON scripting_binding FOR EACH ROW +BEGIN + IF NOT (NEW.id <=> OLD.id AND NEW.name <=> OLD.name AND NEW.table_name <=> OLD.table_name + AND NEW.event <=> OLD.event AND NEW.publication_id <=> OLD.publication_id + AND NEW.grants_json <=> OLD.grants_json AND NEW.patchable_fields <=> OLD.patchable_fields + AND NEW.created_by <=> OLD.created_by AND NEW.created_at <=> OLD.created_at) THEN + SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Only binding activation may change'; + END IF; +END$$ + +DELIMITER ; diff --git a/base/src/main/resources/db/mysql/003_durable_after_hooks.sql b/base/src/main/resources/db/mysql/003_durable_after_hooks.sql new file mode 100644 index 0000000..3eda70b --- /dev/null +++ b/base/src/main/resources/db/mysql/003_durable_after_hooks.sql @@ -0,0 +1,10 @@ +-- Upgrade only the original 001 schema, whose first generated CHECK is the event list. +-- Stop if the existing schema differs; inspect SHOW CREATE TABLE before applying. +-- Fresh installations use the named constraint in the current 001 script instead. +-- Native event UUID plus record ordinal must fit without changing existing IDs. +ALTER TABLE scripting_execution MODIFY operation_id VARCHAR(64) NOT NULL; + +ALTER TABLE scripting_binding DROP CHECK scripting_binding_chk_1, + ADD CONSTRAINT scripting_binding_event_check CHECK (event IN + ('PRE_INSERT', 'PRE_UPDATE', 'PRE_DELETE', 'POST_INSERT', 'POST_UPDATE', 'POST_DELETE', + 'AFTER_INSERT', 'AFTER_UPDATE', 'AFTER_DELETE')); diff --git a/base/src/main/resources/db/postgresql/001_scripting.sql b/base/src/main/resources/db/postgresql/001_scripting.sql new file mode 100644 index 0000000..d10936f --- /dev/null +++ b/base/src/main/resources/db/postgresql/001_scripting.sql @@ -0,0 +1,119 @@ +-- QBit Scripting fresh-install schema. Apply through the host's migration system. +-- Core script/scriptRevision/scriptRevisionFile tables must already exist on +-- the same authoring backend. Their physical names belong to the host mapping. +-- This file does not migrate or activate legacy scripts. + +CREATE TABLE scripting_build ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + script_id INTEGER NOT NULL, + script_revision_id INTEGER NOT NULL, + source_sha256 VARCHAR(64) NOT NULL, + manifest_json TEXT NOT NULL, + manifest_attestation VARCHAR(64) NOT NULL, + toolchain_id VARCHAR(255) NOT NULL, + status VARCHAR(32) NOT NULL CHECK (status IN ('SUCCEEDED', 'FAILED')), + diagnostics_json TEXT NOT NULL, + requested_by VARCHAR(2000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE scripting_artifact ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + build_id INTEGER NOT NULL UNIQUE REFERENCES scripting_build(id), + script_id INTEGER NOT NULL, + version INTEGER NOT NULL CHECK (version > 0), + source_sha256 VARCHAR(64) NOT NULL, + toolchain_id VARCHAR(255) NOT NULL, + entry_point VARCHAR(100) NOT NULL, + coordinator_id VARCHAR(255) NOT NULL, + bundle TEXT NOT NULL, + bundle_sha256 VARCHAR(64) NOT NULL, + size_bytes INTEGER NOT NULL CHECK (size_bytes > 0), + source_map TEXT, + attestation VARCHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + UNIQUE (script_id, version) +); + +CREATE TABLE scripting_publication ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + script_id INTEGER NOT NULL, + version INTEGER NOT NULL CHECK (version > 0), + build_id INTEGER NOT NULL REFERENCES scripting_build(id), + artifact_id INTEGER NOT NULL REFERENCES scripting_artifact(id), + published_by VARCHAR(2000) NOT NULL, + published_at TIMESTAMPTZ NOT NULL, + note VARCHAR(500), + UNIQUE (script_id, version) +); + +CREATE TABLE scripting_binding ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + name VARCHAR(2000) NOT NULL, + table_name VARCHAR(255) NOT NULL, + event VARCHAR(32) NOT NULL CONSTRAINT scripting_binding_event_check CHECK (event IN ('PRE_INSERT', 'PRE_UPDATE', 'PRE_DELETE', 'POST_INSERT', 'POST_UPDATE', 'POST_DELETE', 'AFTER_INSERT', 'AFTER_UPDATE', 'AFTER_DELETE')), + publication_id INTEGER NOT NULL REFERENCES scripting_publication(id), + grants_json TEXT NOT NULL, + patchable_fields TEXT NOT NULL, + enabled INTEGER NOT NULL CHECK (enabled IN (0, 1)), + active_slot VARCHAR(300) UNIQUE, + created_by VARCHAR(2000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CHECK ((enabled = 1 AND active_slot IS NOT NULL AND active_slot = table_name || ':' || event) + OR (enabled = 0 AND active_slot IS NULL)) +); + +CREATE TABLE scripting_execution ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + operation_id VARCHAR(64) NOT NULL, + kind VARCHAR(32) NOT NULL, + build_id INTEGER NOT NULL REFERENCES scripting_build(id), + binding_id INTEGER REFERENCES scripting_binding(id), + publication_id INTEGER REFERENCES scripting_publication(id), + artifact_id INTEGER REFERENCES scripting_artifact(id), + actor_id VARCHAR(2000) NOT NULL, + event VARCHAR(32) NOT NULL, + status VARCHAR(32) NOT NULL, + duration_ms INTEGER, + message VARCHAR(500), + result_action VARCHAR(16), + created_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX scripting_build_project ON scripting_build(script_id, created_at); +CREATE INDEX scripting_binding_event ON scripting_binding(table_name, event, enabled); +CREATE INDEX scripting_execution_build ON scripting_execution(build_id, created_at); +CREATE INDEX scripting_execution_operation ON scripting_execution(operation_id, id); + +-- A separate migration identity owns this function. Application users receive +-- no permission to disable triggers or alter these tables. +CREATE FUNCTION scripting_refuse_immutable_change() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'Scripting provenance and history are append only'; +END; +$$; + +CREATE TRIGGER scripting_build_immutable BEFORE UPDATE OR DELETE ON scripting_build + FOR EACH ROW EXECUTE FUNCTION scripting_refuse_immutable_change(); +CREATE TRIGGER scripting_artifact_immutable BEFORE UPDATE OR DELETE ON scripting_artifact + FOR EACH ROW EXECUTE FUNCTION scripting_refuse_immutable_change(); +CREATE TRIGGER scripting_publication_immutable BEFORE UPDATE OR DELETE ON scripting_publication + FOR EACH ROW EXECUTE FUNCTION scripting_refuse_immutable_change(); +CREATE TRIGGER scripting_execution_immutable BEFORE UPDATE OR DELETE ON scripting_execution + FOR EACH ROW EXECUTE FUNCTION scripting_refuse_immutable_change(); + +CREATE FUNCTION scripting_protect_binding_identity() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'Scripting bindings are retained'; + END IF; + IF (to_jsonb(NEW) - ARRAY['enabled', 'active_slot']) IS DISTINCT FROM + (to_jsonb(OLD) - ARRAY['enabled', 'active_slot']) THEN + RAISE EXCEPTION 'Only scripting binding activation may change'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER scripting_binding_identity BEFORE UPDATE OR DELETE ON scripting_binding + FOR EACH ROW EXECUTE FUNCTION scripting_protect_binding_identity(); diff --git a/base/src/main/resources/db/postgresql/002_native_hook_events.sql b/base/src/main/resources/db/postgresql/002_native_hook_events.sql new file mode 100644 index 0000000..d09f940 --- /dev/null +++ b/base/src/main/resources/db/postgresql/002_native_hook_events.sql @@ -0,0 +1,7 @@ +-- Upgrade installations created from the original 001_scripting.sql. +-- PostgreSQL must be qualified by the deploying host before applying migrations. +BEGIN; +ALTER TABLE scripting_binding DROP CONSTRAINT scripting_binding_event_check; +ALTER TABLE scripting_binding ADD CONSTRAINT scripting_binding_event_check + CHECK (event IN ('PRE_INSERT', 'PRE_UPDATE', 'PRE_DELETE', 'POST_INSERT', 'POST_UPDATE', 'POST_DELETE')); +COMMIT; diff --git a/base/src/main/resources/db/postgresql/002_source_import.sql b/base/src/main/resources/db/postgresql/002_source_import.sql new file mode 100644 index 0000000..b86f65c --- /dev/null +++ b/base/src/main/resources/db/postgresql/002_source_import.sql @@ -0,0 +1,15 @@ +-- Apply after 001_scripting.sql. Source bytes remain in existing core tables. +CREATE TABLE scripting_import ( + id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + script_id INTEGER NOT NULL, + script_revision_id INTEGER NOT NULL UNIQUE, + repository VARCHAR(201) NOT NULL, + commit_sha VARCHAR(40) NOT NULL CHECK (commit_sha ~ '^[0-9a-f]{40}$'), + subdirectory VARCHAR(200) NOT NULL, + source_sha256 VARCHAR(64) NOT NULL CHECK (source_sha256 ~ '^[0-9a-f]{64}$'), + imported_by VARCHAR(2000) NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); +CREATE INDEX scripting_import_project ON scripting_import(script_id, id); +CREATE TRIGGER scripting_import_immutable BEFORE UPDATE OR DELETE ON scripting_import + FOR EACH ROW EXECUTE FUNCTION scripting_refuse_immutable_change(); diff --git a/base/src/main/resources/db/postgresql/003_durable_after_hooks.sql b/base/src/main/resources/db/postgresql/003_durable_after_hooks.sql new file mode 100644 index 0000000..b115968 --- /dev/null +++ b/base/src/main/resources/db/postgresql/003_durable_after_hooks.sql @@ -0,0 +1,11 @@ +-- Upgrade installations created from the original PostgreSQL 001_scripting.sql. +-- Fresh installations already contain both changes in the current 001 script. +-- Native event UUID plus record ordinal must fit without changing existing IDs. Widening +-- a VARCHAR needs no table rewrite, so the append-only row trigger is not involved. +BEGIN; +ALTER TABLE scripting_execution ALTER COLUMN operation_id TYPE VARCHAR(64); +ALTER TABLE scripting_binding DROP CONSTRAINT IF EXISTS scripting_binding_event_check; +ALTER TABLE scripting_binding ADD CONSTRAINT scripting_binding_event_check + CHECK (event IN ('PRE_INSERT', 'PRE_UPDATE', 'PRE_DELETE', 'POST_INSERT', 'POST_UPDATE', 'POST_DELETE', + 'AFTER_INSERT', 'AFTER_UPDATE', 'AFTER_DELETE')); +COMMIT; diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/AfterHookAuthoringTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/AfterHookAuthoringTest.java new file mode 100644 index 0000000..6c5a6d5 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/AfterHookAuthoringTest.java @@ -0,0 +1,136 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptAfterHooks; +import com.kingsrook.qbits.scripting.hooks.ScriptHooks; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeEvent; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeType; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.TransactionalRecordChangeMetaData; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Metadata and dry runs do not impersonate committed native delivery. + *******************************************************************************/ +class AfterHookAuthoringTest extends ScriptingTestBase +{ + /*************************************************************************** + ** Explicit installation preserves host callbacks and is idempotent by phase. + ***************************************************************************/ + @Test + void metadataAndBindingDiscoveryPreserveExistingPhases() throws Exception + { + var table = instance.getTable(HOOK_TABLE); + var before = table.getCustomizer(TableCustomizers.PRE_INSERT_RECORD.getRole()); + ScriptAfterHooks.install(instance, table, HookEvent.AFTER_INSERT); + ScriptAfterHooks.install(instance, table, HookEvent.AFTER_UPDATE, HookEvent.AFTER_DELETE); + assertEquals(before, table.getCustomizer(TableCustomizers.PRE_INSERT_RECORD.getRole())); + Integer project = newProject(); + Integer publication = publishNew(project, "after"); + Integer binding = service.bind(everything(), new BindingSpec("after", HOOK_TABLE, HookEvent.AFTER_INSERT, publication, declared(), Set.of())); + assertEquals(binding, service.hooks().view(everything(), project).bindings().getFirst().id()); + var phases = service.hooks().view(everything(), project).targets().stream().filter(target -> target.tableName().equals(HOOK_TABLE)).findFirst().orElseThrow().phases(); + assertTrue(phases.stream().filter(phase -> phase.event().isAfterCommit()).allMatch(phase -> phase.patchableFields().isEmpty())); + assertEquals(3, phases.stream().filter(phase -> phase.event().isAfterCommit()).count()); + assertThrows(ScriptingException.class, () -> service.runHook(service.findBinding(HOOK_TABLE, HookEvent.AFTER_INSERT).orElseThrow(), everything().id(), "{}", null)); + } + + /*************************************************************************** + ** No direct process, fabricated event or synchronous hook claims commit. + ***************************************************************************/ + @Test + void directAndFabricatedInvocationAreRefused() throws Exception + { + ScriptAfterHooks.install(instance, instance.getTable(HOOK_TABLE), HookEvent.AFTER_INSERT); + int before = worker.executeCalls; + assertThrows(QException.class, () -> new ScriptAfterHooks.Dispatch().run(new RunBackendStepInput(), new RunBackendStepOutput())); + assertThrows(QException.class, () -> service.afterHooks().enlist(new RecordChangeEvent().withTableName(HOOK_TABLE).withType(RecordChangeType.INSERT) + .withRecords(List.of(new QRecord().withValue("id", 1))))); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** Required-listener conflicts and accidental synchronous registration fail. + ***************************************************************************/ + @Test + void incompatibleHostMetadataIsNotOverwritten() throws Exception + { + var table = instance.getTable(HOOK_TABLE); + assertThrows(IllegalArgumentException.class, () -> ScriptHooks.install(table, HookEvent.AFTER_INSERT)); + table.withSupplementalMetaData(new TransactionalRecordChangeMetaData(new QCodeReference(String.class), Set.of(RecordChangeType.INSERT))); + assertThrows(QException.class, () -> ScriptAfterHooks.install(instance, table, HookEvent.AFTER_INSERT)); + assertEquals(String.class.getName(), TransactionalRecordChangeMetaData.of(table).getListener().getName()); + } + + /*************************************************************************** + ** Dry-run delete conveys only metadata, while no job or mutation is created. + ***************************************************************************/ + @Test + void selectedDeleteDryRunContainsNoStoredValues() throws Exception + { + Integer id = new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecord(new QRecord().withValue("status", "private-delete-value"))).getRecords().getFirst().getValueInteger("id"); + ScriptAfterHooks.install(instance, instance.getTable(HOOK_TABLE), HookEvent.AFTER_DELETE); + Integer project = newProject(); + var build = build(draft(project, "after-delete-test")); + assertTrue(service.hooks().test(everything(), project, build.buildId(), HOOK_TABLE, HookEvent.AFTER_DELETE, id, java.util.Map.of()).succeeded()); + JSONObject event = new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("event"); + assertTrue(event.getBoolean("dryRun")); + assertEquals("DELETED_METADATA", event.getString("dataState")); + assertFalse(event.has("record")); + assertFalse(worker.lastExecuteRequest.contains("private-delete-value")); + } + + /*************************************************************************** + ** Observational results never apply a patch or reject an already committed row. + ***************************************************************************/ + @Test + void afterDryRunsOnlyAcceptAllow() throws Exception + { + var build = build(draft(newProject(), "after-results")); + for(String result : List.of("{\"action\":\"PATCH\",\"patch\":{\"status\":\"bad\"}}", "{\"action\":\"REJECT\",\"message\":\"no\"}")) + { + worker.hookJson = result; + assertFalse(service.test(everything(), build.buildId(), HookEvent.AFTER_INSERT, "{}", null).succeeded()); + } + worker.hookJson = "{\"action\":\"ALLOW\"}"; + assertTrue(service.test(everything(), build.buildId(), HookEvent.AFTER_INSERT, "{\"dryRun\":false}", null).succeeded()); + assertTrue(new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("event").getBoolean("dryRun")); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/AfterHookJdbcTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/AfterHookJdbcTest.java new file mode 100644 index 0000000..918f0a1 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/AfterHookJdbcTest.java @@ -0,0 +1,698 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptAfterHooks; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.actions.tables.listeners.RecordChangeListenerHelper.RequiredListenerException; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.jobs.DurableJobs; +import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData; +import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider; +import com.kingsrook.qqq.backend.core.jobs.DurableJobsTables; +import com.kingsrook.qqq.backend.core.jobs.model.JobAction; +import com.kingsrook.qqq.backend.core.jobs.model.LaunchRequest; +import com.kingsrook.qqq.backend.core.jobs.runtime.DurableJobsRuntime; +import com.kingsrook.qqq.backend.core.jobs.store.DurableJobStore; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.authentication.implementations.MockAuthenticationModule; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Real file-H2 source/Jobs transactions and native runtime delivery with a trusted + ** protocol worker. These fixtures do not attest a production worker or identity. + *******************************************************************************/ +class AfterHookJdbcTest extends ScriptingTestBase +{ + @TempDir + Path directory; + private String url; + private DurableJobStore jobs; + private static volatile boolean revokeExecution; + private static volatile boolean changeScopeAfterWorker; + + /*************************************************************************** + ** Use synchronous owned file storage and the real core Jobs metadata/schema. + ***************************************************************************/ + @BeforeEach + void useFileJdbc() throws Exception + { + revokeExecution = false; + changeScopeAfterWorker = false; + RDBMSBackendMetaData backend = createBackend(); + String backendName = backend.getName(); + instance.addBackend(backend); + instance.getTable(HOOK_TABLE).withField(new QFieldMetaData("secret", QFieldType.STRING).withIsHidden(true)); + new DurableJobsMetaDataProvider().defineAll(instance, backendName, DurableJobsMetaDataProvider.snakeCaseColumnNames()); + var settings = new DurableJobsMetaData().withTransactionalAdmissionEnabled(true).withTransactionalNamespace("scripting-after-fixture") + .withJobGrantKey(Base64.getEncoder().encodeToString(KEY)).withMaxQueuedJobsPerUser(100).withWorkerThreads(1).withPollIntervalMillis(25); + instance.withSupplementalMetaData(settings); + instance.getAuthentication().withModuleCodeReference(new QCodeReference(Authentication.class)); + QContext.setQSession(restoredSession()); + try(Connection connection = connection(); Statement statement = connection.createStatement()) + { + for(QTableMetaData table : instance.getTables().values()) + { + table.setBackendName(backendName); + table.setBackendDetails(new RDBMSTableBackendDetails().withTableName(table.getName())); + List columns = new ArrayList<>(); + for(QFieldMetaData field : table.getFields().values()) + { + if(field.getBackendName() == null) + { + field.setBackendName(field.getName()); + } + String type = sqlType(field); + columns.add("`" + field.getBackendName() + "` " + (field.getName().equals(table.getPrimaryKeyField()) ? identityType() + : type + (Boolean.TRUE.equals(field.getIsRequired()) ? " NOT NULL" : ""))); + } + if(table.getUniqueKeys() != null) + { + table.getUniqueKeys().forEach(key -> columns.add("UNIQUE (" + String.join(",", key.getFieldNames().stream() + .map(name -> "`" + table.getField(name).getBackendName() + "`").toList()) + ")")); + } + if(!migratedTable(table)) + { + statement.execute("CREATE TABLE " + table.getName() + " (" + String.join(",", columns) + ")" + tableSuffix()); + } + } + statement.execute("CREATE TABLE jobs_admission(id INTEGER PRIMARY KEY)" + tableSuffix()); + statement.execute("INSERT INTO jobs_admission(id) VALUES(1)"); + finishSchema(statement); + } + jobs = new DurableJobStore(settings); + } + + /*************************************************************************** + ** The base fixture uses a synchronous file-backed H2 provider. + ***************************************************************************/ + protected RDBMSBackendMetaData createBackend() throws Exception + { + url = "jdbc:h2:file:" + directory.resolve("after").toString().replace('\\', '/') + ";DATABASE_TO_UPPER=FALSE;WRITE_DELAY=0;LOCK_TIMEOUT=5000"; + return (new RDBMSBackendMetaData().withName("after_jdbc").withVendor("h2").withJdbcUrl(url).withUsername("sa")); + } + + /*************************************************************************** + ** Primitive physical types follow the exact native metadata contract. + ***************************************************************************/ + protected String sqlType(QFieldMetaData field) + { + return (switch(field.getType()) + { + case INTEGER -> "INTEGER"; + case LONG -> "BIGINT"; + case BOOLEAN -> "BOOLEAN"; + case DECIMAL -> "DECIMAL(30,10)"; + case DATE_TIME -> "TIMESTAMP"; + case DATE -> "DATE"; + case TIME -> "TIME"; + case TEXT, HTML -> "CLOB"; + default -> "VARCHAR(" + (field.getMaxLength() == null ? 255 : field.getMaxLength()) + ")"; + }); + } + + /*************************************************************************** + ** Provider subclasses preserve an actual native identity column. + ***************************************************************************/ + protected String identityType() + { + return ("INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY"); + } + + /*************************************************************************** + ** H2 needs no table-level vendor suffix. + ***************************************************************************/ + protected String tableSuffix() + { + return (""); + } + + /*************************************************************************** + ** Provider migrations may own their scripting table DDL instead of metadata. + ***************************************************************************/ + protected boolean migratedTable(QTableMetaData table) + { + return (false); + } + + /*************************************************************************** + ** Optional provider applies shipped migrations and qualified source indexes. + ***************************************************************************/ + protected void finishSchema(Statement statement) throws Exception {} + + /*************************************************************************** + ** Fault injection uses the provider's actual CHECK removal syntax. + ***************************************************************************/ + protected void dropCheck(Statement statement, String name) throws Exception + { + statement.execute("ALTER TABLE scripting_execution DROP CONSTRAINT " + name); + } + + /*************************************************************************** + ** Discard all application pools; optional providers also restart their server. + ***************************************************************************/ + protected void restartPersistence() throws Exception + { + ConnectionManager.closeConnectionProviders(); + } + + /*************************************************************************** + ** Stop the actual runtime and close owned database pools before temp cleanup. + ***************************************************************************/ + @AfterEach + void stopAndClose() + { + DurableJobsRuntime.stop(instance, Duration.ofSeconds(5)); + ConnectionManager.closeConnectionProviders(); + } + + /*************************************************************************** + ** Source rows and enlisted intents are invisible before commit and roll back. + ***************************************************************************/ + @Test + void sourceAndIntentCommitTogetherAndRollbackTogether() throws Exception + { + bind(HookEvent.AFTER_INSERT); + int before = worker.executeCalls; + try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput().withTableName(HOOK_TABLE))) + { + new InsertAction().execute(new InsertInput(HOOK_TABLE).withTransaction(transaction).withRecord(new QRecord().withValue("status", "pending"))); + assertEquals(0, count(HOOK_TABLE)); + assertEquals(0, count(DurableJobsTables.TABLE_JOB)); + transaction.rollback(); + } + assertEquals(0, count(HOOK_TABLE)); + assertEquals(0, count(DurableJobsTables.TABLE_JOB)); + insert("committed"); + assertEquals(1, count(HOOK_TABLE)); + assertEquals(1, count(DurableJobsTables.TABLE_JOB)); + assertEquals(before, worker.executeCalls); + String parameters = allJobs().getFirst().getValueString("paramsJson"); + assertFalse(parameters.contains("committed")); + assertFalse(parameters.contains("source-secret")); + } + + /*************************************************************************** + ** Delivery uses the original artifact and newly authorized current row state. + ***************************************************************************/ + @Test + void restartUsesPinnedScriptAndCurrentVisibleRecord() throws Exception + { + Integer project = newProject(); + Integer publication = publishNew(project, "pinned-source"); + ScriptAfterHooks.install(instance, instance.getTable(HOOK_TABLE), HookEvent.AFTER_INSERT); + service.bind(everything(), new BindingSpec("after", HOOK_TABLE, HookEvent.AFTER_INSERT, publication, declared(), Set.of())); + QRecord row = insert("before"); + String oldPin = allJobs().getFirst().getValueString("paramsJson"); + new UpdateAction().execute(new UpdateInput(HOOK_TABLE).withRecord(new QRecord().withValue("id", row.getValue("id")).withValue("status", "current"))); + publishNew(project, "newer-source"); + assertEquals(oldPin, allJobs().getFirst().getValueString("paramsJson")); + int before = worker.executeCalls; + restartPersistence(); + runJobs(); + assertEquals(before + 1, worker.executeCalls); + assertEquals("COMPLETED", allJobs().getFirst().getValueString("status")); + JSONObject event = new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("event"); + assertEquals("CURRENT", event.getString("dataState")); + assertEquals("current", event.getJSONObject("record").getString("status")); + assertFalse(event.getJSONObject("record").has("secret")); + int after = worker.executeCalls; + runJobs(); + assertEquals(after, worker.executeCalls, "Completed persisted jobs do not run again after restart"); + } + + /*************************************************************************** + ** A deleted row never becomes an implied authorized historical snapshot. + ***************************************************************************/ + @Test + void deleteDeliveryIsMetadataOnly() throws Exception + { + QRecord row = insert("deleted-private-value"); + bind(HookEvent.AFTER_DELETE); + new DeleteAction().execute(new DeleteInput(HOOK_TABLE).withPrimaryKeys(List.of(row.getValue("id")))); + assertEquals(0, count(HOOK_TABLE)); + assertFalse(allJobs().getFirst().getValueString("paramsJson").contains("deleted-private-value")); + runJobs(); + assertEquals("COMPLETED", allJobs().getFirst().getValueString("status")); + JSONObject event = new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("event"); + assertEquals("DELETED_METADATA", event.getString("dataState")); + assertFalse(event.has("record")); + assertFalse(worker.lastExecuteRequest.contains("deleted-private-value")); + } + + /*************************************************************************** + ** Binding revocation and ordinary launch cannot enter the isolated worker. + ***************************************************************************/ + @Test + void disabledBindingAndOrdinaryJobAreRefused() throws Exception + { + Integer binding = bind(HookEvent.AFTER_INSERT); + insert("queued"); + service.setBindingEnabled(everything(), binding, false); + int before = worker.executeCalls; + runJobs(); + assertEquals("FAILED", allJobs().getFirst().getValueString("status")); + assertEquals(before, worker.executeCalls); + Integer ordinary = DurableJobs.launch(new LaunchRequest(ScriptAfterHooks.PROCESS)); + runJobs(); + assertEquals("FAILED", jobs.getJob(ordinary).getValueString("status")); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** Native preview performs no committed mutation or AFTER dispatch enlistment. + ***************************************************************************/ + @Test + void previewDoesNotEnlistOrExecute() throws Exception + { + bind(HookEvent.AFTER_INSERT); + int before = worker.executeCalls; + new InsertAction().execute(new InsertInput(HOOK_TABLE).withDryRun(true).withRecord(new QRecord().withValue("status", "preview"))); + assertEquals(0, count(HOOK_TABLE)); + assertEquals(0, allJobs().size()); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** Restored current permissions must still authorize the initiating actor. + ***************************************************************************/ + @Test + void revokedExecutionGrantPreventsWorkerCall() throws Exception + { + bind(HookEvent.AFTER_INSERT); + insert("queued"); + int before = worker.executeCalls; + revokeExecution = true; + runJobs(); + assertEquals("FAILED", allJobs().getFirst().getValueString("status")); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** Current project and source read policies are checked before worker delivery. + ***************************************************************************/ + @Test + void revokedSourceAndProjectReadsPreventDelivery() throws Exception + { + bind(HookEvent.AFTER_INSERT); + var rules = new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED).withCustomPermissionChecker(new QCodeReference(DenyGet.class)); + for(String table : List.of(HOOK_TABLE, "script")) + { + insert("queued"); + int before = worker.executeCalls; + var prior = instance.getTable(table).getPermissionRules(); + instance.getTable(table).setPermissionRules(rules); + try + { + runJobs(); + assertEquals(before, worker.executeCalls); + assertTrue(allJobs().stream().allMatch(job -> "FAILED".equals(job.getValueString("status")))); + } + finally + { + instance.getTable(table).setPermissionRules(prior); + } + } + } + + /*************************************************************************** + ** Mandatory execution audit failure prevents starting the external worker. + ***************************************************************************/ + @Test + void auditFailurePreventsExecutionAndLeavesCommittedSource() throws Exception + { + bind(HookEvent.AFTER_INSERT); + insert("committed"); + int before = worker.executeCalls; + try(Connection connection = connection(); Statement statement = connection.createStatement()) + { + statement.execute("ALTER TABLE scripting_execution ADD CONSTRAINT reject_after_execution CHECK (kind <> 'DURABLE_HOOK')"); + } + runJobs(); + assertEquals(1, count(HOOK_TABLE)); + assertEquals("FAILED", allJobs().getFirst().getValueString("status")); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** A failed outcome audit leaves an explicit failed attempt, not success. + ** Operator retry may execute again and retains the native event correlation. + ***************************************************************************/ + @Test + void outcomeAuditFailureAndExplicitRetryKeepEventIdentity() throws Exception + { + bind(HookEvent.AFTER_INSERT); + insert("committed"); + Integer jobId = allJobs().getFirst().getValueInteger("id"); + String params = allJobs().getFirst().getValueString("paramsJson"); + int before = worker.executeCalls; + try(Connection connection = connection(); Statement statement = connection.createStatement()) + { + statement.execute("ALTER TABLE scripting_execution ADD CONSTRAINT reject_after_outcome CHECK (kind <> 'DURABLE_HOOK' OR status = 'STARTED')"); + } + runJobs(); + assertEquals(before + 1, worker.executeCalls); + assertEquals("FAILED", jobs.getJob(jobId).getValueString("status")); + assertEquals(1, count(HOOK_TABLE)); + String eventId = new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("event").getString("id"); + assertTrue(eventId.length() >= 38 && eventId.length() <= 64); + try(Connection connection = connection(); Statement statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT operation_id FROM scripting_execution WHERE 1=0")) + { + assertEquals(64, rows.getMetaData().getPrecision(1)); + } + try(Connection connection = connection(); Statement statement = connection.createStatement()) + { + dropCheck(statement, "reject_after_outcome"); + } + DurableJobs.control(jobId, JobAction.RETRY_FAILED); + runJobs(); + assertEquals(before + 2, worker.executeCalls); + assertEquals("COMPLETED", jobs.getJob(jobId).getValueString("status")); + assertEquals(params, jobs.getJob(jobId).getValueString("paramsJson")); + assertEquals(eventId, new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("event").getString("id")); + try(Connection connection = connection(); Statement statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT operation_id,status FROM scripting_execution WHERE kind='DURABLE_HOOK' ORDER BY id")) + { + for(String expected : List.of("STARTED", "STARTED", "OK")) + { + assertTrue(rows.next()); + assertEquals(eventId, rows.getString(1)); + assertEquals(expected, rows.getString(2)); + } + assertFalse(rows.next()); + } + } + + /*************************************************************************** + ** A host read callback cannot broaden the restored actor before dispatch. + ***************************************************************************/ + @Test + void scopeMutationDuringContextReadNeverReachesWorker() throws Exception + { + bind(HookEvent.AFTER_INSERT); + insert("queued"); + int before = worker.executeCalls; + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED) + .withCustomPermissionChecker(new QCodeReference(ChangeReadScope.class))); + runJobs(); + assertEquals("FAILED", allJobs().getFirst().getValueString("status")); + assertEquals(before, worker.executeCalls); + try(Connection connection = connection(); Statement statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT status FROM scripting_execution WHERE kind='DURABLE_HOOK' ORDER BY id")) + { + assertTrue(rows.next()); + assertEquals("STARTED", rows.getString(1)); + assertTrue(rows.next()); + assertEquals("DENIED", rows.getString(1)); + assertFalse(rows.next()); + } + } + + /*************************************************************************** + ** Same-session mutation is deliberate: identity equality alone is inadequate. + ***************************************************************************/ + public static class ChangeReadScope implements CustomPermissionChecker + { + /*********************************************************************** + ** Simulate a host callback changing the restored security scope. + ***********************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) + { + if(input instanceof GetInput get && get.getPrimaryKey() != null) + { + QContext.getQSession().withSecurityKeyValue("changed", "broader"); + } + } + } + + /*************************************************************************** + ** Post-worker callbacks must not broaden authority and still complete a job. + ***************************************************************************/ + @Test + void scopeMutationDuringFinalReadRefusesCompletion() throws Exception + { + bind(HookEvent.AFTER_INSERT); + insert("queued"); + int before = worker.executeCalls; + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED) + .withCustomPermissionChecker(new QCodeReference(ChangeFinalReadScope.class))); + worker.onExecute = channel -> changeScopeAfterWorker = true; + runJobs(); + assertEquals(before + 1, worker.executeCalls); + assertEquals("FAILED", allJobs().getFirst().getValueString("status")); + assertEquals(1, count(HOOK_TABLE)); + } + + /*************************************************************************** + ** This callback changes only a read after the successful worker response. + ***************************************************************************/ + public static class ChangeFinalReadScope implements CustomPermissionChecker + { + /*********************************************************************** + ** Preserve every existing grant and add a detectable broadening key. + ***********************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) + { + if(changeScopeAfterWorker && input instanceof GetInput get && get.getPrimaryKey() != null) + { + QContext.getQSession().withSecurityKeyValue("changed", "broader"); + } + } + } + + /*************************************************************************** + ** Host read policy is deliberately input-class-sensitive, like real endpoints. + ***************************************************************************/ + public static class DenyGet implements CustomPermissionChecker + { + /*********************************************************************** + ** Writes may continue while actual record reads are explicitly denied. + ***********************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException + { + if(input instanceof GetInput) + { + throw new QPermissionDeniedException("Fixture read denied"); + } + } + } + + /*************************************************************************** + ** Identical later writes are distinct mutations rather than content dedupes. + ***************************************************************************/ + @Test + void identicalUpdatesHaveDistinctEventsAndBoundedAdmission() throws Exception + { + QRecord row = insert("same"); + bind(HookEvent.AFTER_UPDATE); + for(int i = 0; i < 2; i++) + { + new UpdateAction().execute(new UpdateInput(HOOK_TABLE).withRecord(new QRecord().withValue("id", row.getValue("id")).withValue("status", "same"))); + } + List saved = allJobs(); + assertEquals(2, saved.size()); + assertNotEquals(new JSONObject(saved.get(0).getValueString("paramsJson")).getString("eventId"), new JSONObject(saved.get(1).getValueString("paramsJson")).getString("eventId")); + runtime.withMaxRecordsPerHook(1); + assertThrows(RequiredListenerException.class, () -> new UpdateAction().execute(new UpdateInput(HOOK_TABLE).withRecords(List.of( + new QRecord().withValue("id", row.getValue("id")).withValue("status", "too-many"), + new QRecord().withValue("id", row.getValue("id")).withValue("status", "too-many"))))); + assertEquals(2, allJobs().size()); + } + + /*************************************************************************** + ** Enlistment failure poisons a caller transaction even if it catches the error. + ***************************************************************************/ + @Test + void missingAdmissionRollsBackAndCannotBeCommitted() throws Exception + { + bind(HookEvent.AFTER_INSERT); + try(Connection connection = connection(); Statement statement = connection.createStatement()) + { + statement.execute("DELETE FROM jobs_admission"); + } + try(QBackendTransaction transaction = QBackendTransaction.openFor(new QueryInput().withTableName(HOOK_TABLE))) + { + assertThrows(RequiredListenerException.class, () -> new InsertAction().execute(new InsertInput(HOOK_TABLE).withTransaction(transaction) + .withRecord(new QRecord().withValue("status", "must-rollback")))); + assertTrue(transaction.isRollbackOnly()); + assertThrows(QException.class, transaction::commit); + transaction.rollback(); + } + assertEquals(0, count(HOOK_TABLE)); + assertEquals(0, allJobs().size()); + } + + /*************************************************************************** + ** Bind a tested immutable publication to one native AFTER phase. + ***************************************************************************/ + protected Integer bind(HookEvent phase) throws Exception + { + ScriptAfterHooks.install(instance, instance.getTable(HOOK_TABLE), phase); + Integer publication = publishNew(newProject(), "after-source"); + return (service.bind(everything(), new BindingSpec("after", HOOK_TABLE, phase, publication, declared(), Set.of()))); + } + + /*************************************************************************** + ** Seed an ordinary native source mutation with an inaccessible extra field. + ***************************************************************************/ + protected QRecord insert(String status) throws QException + { + return (new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecord(new QRecord().withValue("status", status) + .withValue("secret", "source-secret"))).getRecords().getFirst()); + } + + /*************************************************************************** + ** The core store owns persisted job rows; tests do not substitute a queue. + ***************************************************************************/ + protected List allJobs() throws QException + { + return (jobs.query(DurableJobsTables.TABLE_JOB, new QQueryFilter().withLimit(100))); + } + + /*************************************************************************** + ** Bound the native worker observation and always stop its real threads. + ***************************************************************************/ + protected void runJobs() throws Exception + { + DurableJobsRuntime.start(instance); + try + { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(20); + while(System.nanoTime() < deadline) + { + if(allJobs().stream().allMatch(job -> Set.of("COMPLETED", "FAILED", "NEEDS_REVIEW", "CANCELLED").contains(job.getValueString("status")))) + { + return; + } + Thread.sleep(25); + } + throw new AssertionError("Native Jobs delivery exceeded fixture deadline"); + } + finally + { + DurableJobsRuntime.stop(instance, Duration.ofSeconds(5)); + } + } + + /*************************************************************************** + ** Separate connections prove committed visibility, not fixture memory state. + ***************************************************************************/ + protected Connection connection() throws Exception + { + return (DriverManager.getConnection(url, "sa", "")); + } + + /*************************************************************************** + ** Inspect only a test-owned constant table name on an independent connection. + ***************************************************************************/ + private int count(String table) throws Exception + { + try(Connection connection = connection(); Statement statement = connection.createStatement(); var rows = statement.executeQuery("SELECT COUNT(*) FROM " + table)) + { + rows.next(); + return (rows.getInt(1)); + } + } + + /*************************************************************************** + ** Reconstructed synthetic authority can be revoked before native delivery. + ***************************************************************************/ + private static QSession restoredSession() + { + QSession session = sessionWith(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK, + ScriptingPermissions.TEST, ScriptingPermissions.PUBLISH, ScriptingPermissions.BIND, revokeExecution ? "revoked" : ScriptingPermissions.EXECUTE); + session.setUuid("after-fixture-session"); + return (session); + } + + /*************************************************************************** + ** Trusted fixture restoration is not a production identity implementation. + ***************************************************************************/ + public static class Authentication extends MockAuthenticationModule + { + /*********************************************************************** + ** Re-establish only the exact initiating synthetic interactive sign-in. + ***********************************************************************/ + @Override + public QSession createSession(QInstance current, Map context) throws QAuthenticationException + { + if(!"after-fixture-session".equals(context.get("sessionUUID"))) + { + throw new QAuthenticationException("Fixture sign-in unavailable"); + } + return (restoredSession()); + } + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/AuthoringPreparationTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/AuthoringPreparationTest.java new file mode 100644 index 0000000..e0e70d4 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/AuthoringPreparationTest.java @@ -0,0 +1,363 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.worker.WarmWorkerPreparation; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.scripts.Script; +import com.kingsrook.qqq.backend.core.model.session.QCredentialScope; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Real authoring services with a protocol-only cold provider. No script engine, + ** network, or volatile backend durability claim is involved. + *******************************************************************************/ +class AuthoringPreparationTest extends ScriptingTestBase +{ + private ColdWorker cold; + + /*************************************************************************** + ** Replace only the test transport; retain the real registry and coordinator. + ***************************************************************************/ + @BeforeEach + void installColdWorker() + { + cold = new ColdWorker(); + runtime.withWorker(cold); + worker = cold; + } + + /*************************************************************************** + ** Every user authoring boundary warms exact bytes; hooks and saves never do. + ***************************************************************************/ + @Test + void authoringWarmsExactCodeButHooksStayHot() throws Exception + { + Integer project = newProject(); + Integer build = build(draft(project, "one")).buildId(); + assertEquals(0, cold.preparations); + assertTrue(service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null).succeeded()); + assertEquals(1, cold.preparations); + assertEquals(ENTRY, cold.last.entryPoint()); + assertEquals(TOOLCHAIN, cold.last.toolchainId()); + assertEquals(BuildCoordinator.sha256Hex(cold.last.bundle()), cold.last.sha256()); + assertFalse(cold.last.toString().contains(cold.last.bundle())); + Integer publication = service.publish(everything(), build, "one"); + Integer binding = bindHook(publication); + assertEquals(3, cold.preparations); + service.setBindingEnabled(everything(), binding, false); + assertEquals(3, cold.preparations); + service.setBindingEnabled(everything(), binding, true); + assertEquals(4, cold.preparations); + assertTrue(service.runHook(service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).orElseThrow(), everything().id(), "{}", null).succeeded()); + draft(project, "new unactivated source"); + assertEquals(4, cold.preparations); + cold.hookJson = "{\"additional\":true}"; + assertTrue(service.testContext(everything(), build, "{}", List.of()).succeeded()); + assertEquals(5, cold.preparations); + } + + /*************************************************************************** + ** Preparation failure cannot activate a revision, create or enable a binding. + ***************************************************************************/ + @Test + void failedPreparationLeavesPublicationAndBindingIntact() throws Exception + { + Integer project = newProject(); + Integer publication = publishNew(project, "first"); + Integer binding = bindHook(publication); + Integer activeRevision = rows(Script.TABLE_NAME).getFirst().getValueInteger("currentScriptRevisionId"); + Integer next = build(draft(project, "second")).buildId(); + service.test(everything(), next, HookEvent.PRE_INSERT, "{}", null); + service.setBindingEnabled(everything(), binding, false); + cold.failure = true; + assertEquals(ScriptingException.Kind.UNAVAILABLE, assertThrows(ScriptingException.class, () -> service.publish(everything(), next, "two")).getKind()); + assertThrows(ScriptingException.class, () -> service.setBindingEnabled(everything(), binding, true)); + assertThrows(ScriptingException.class, () -> bindHook(publication)); + assertEquals(1, rows(ScriptingTables.PUBLICATION).size()); + assertEquals(1, rows(ScriptingTables.BINDING).size()); + assertEquals(0, rows(ScriptingTables.BINDING).getFirst().getValueInteger("enabled")); + assertEquals(activeRevision, rows(Script.TABLE_NAME).getFirst().getValueInteger("currentScriptRevisionId")); + } + + /*************************************************************************** + ** Denied authority never warms; revoked authority after warming never runs. + ***************************************************************************/ + @Test + void testPermissionRevocationPreventsDispatch() throws Exception + { + Integer build = build(draft(newProject(), "test")).buildId(); + assertThrows(ScriptingException.class, () -> service.test(only(ScriptingPermissions.VIEW), build, HookEvent.PRE_INSERT, "{}", null)); + assertEquals(0, cold.preparations); + cold.callback = () -> QContext.getQSession().setPermissions(Set.of(ScriptingPermissions.VIEW)); + assertEquals(ScriptingException.Kind.DENIED, assertThrows(ScriptingException.class, + () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null)).getKind()); + assertEquals(1, cold.preparations); + assertEquals(0, cold.executeCalls); + assertTrue(rows(ScriptingTables.EXECUTION).isEmpty()); + } + + /*************************************************************************** + ** A same-user replacement session is still a different initiating context. + ***************************************************************************/ + @Test + void replacementSessionIsRejected() throws Exception + { + Integer build = build(draft(newProject(), "test")).buildId(); + cold.callback = () -> QContext.setQSession(sessionWith(ScriptingPermissions.VIEW, ScriptingPermissions.TEST)); + assertEquals(ScriptingException.Kind.DENIED, assertThrows(ScriptingException.class, + () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null)).getKind()); + assertEquals(0, cold.executeCalls); + } + + /*************************************************************************** + ** Current project and hook-target access are re-evaluated after preparation. + ***************************************************************************/ + @Test + void revokedProjectOrTargetPermissionPreventsMutation() throws Exception + { + Integer project = newProject(); + Integer publication = publishNew(project, "first"); + cold.callback = () -> instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + assertThrows(ScriptingException.class, () -> bindHook(publication)); + assertTrue(rows(ScriptingTables.BINDING).isEmpty()); + cold.callback = () -> instance.getTable(Script.TABLE_NAME).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + Integer build = rows(ScriptingTables.BUILD).getFirst().getValueInteger("id"); + assertThrows(ScriptingException.class, () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null)); + assertEquals(1, cold.executeCalls, "Only the earlier publication test executed"); + } + + /*************************************************************************** + ** Changed signed bytes or publication association cannot become warm authority. + ***************************************************************************/ + @Test + void changedArtifactAndPublicationPinsAreRefused() throws Exception + { + Integer project = newProject(); + Integer publication = publishNew(project, "first"); + Integer other = build(draft(project, "second")).buildId(); + cold.callback = () -> MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.PUBLICATION) + .withRecords(List.of(new QRecord().withValue("id", publication).withValue("build_id", other))), false); + assertEquals(ScriptingException.Kind.INTEGRITY, assertThrows(ScriptingException.class, () -> bindHook(publication)).getKind()); + assertTrue(rows(ScriptingTables.BINDING).isEmpty()); + Integer artifact = rows(ScriptingTables.ARTIFACT).stream().filter(row -> other.equals(row.getValueInteger("build_id"))).findFirst().orElseThrow().getValueInteger("id"); + cold.callback = () -> MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.ARTIFACT) + .withRecords(List.of(new QRecord().withValue("id", artifact).withValue("bundle", "changed"))), false); + assertEquals(ScriptingException.Kind.INTEGRITY, assertThrows(ScriptingException.class, + () -> service.test(everything(), other, HookEvent.PRE_INSERT, "{}", null)).getKind()); + assertEquals(1, cold.executeCalls); + } + + /*************************************************************************** + ** Cold callbacks cannot recursively prepare another service authoring action. + ***************************************************************************/ + @Test + void recursivePreparationIsRefusedAndGuardIsCleared() throws Exception + { + Integer build = build(draft(newProject(), "test")).buildId(); + cold.callback = () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null); + assertEquals(ScriptingException.Kind.DENIED, assertThrows(ScriptingException.class, + () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null)).getKind()); + cold.callback = null; + assertTrue(service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null).succeeded()); + assertEquals(2, cold.preparations); + } + + /*************************************************************************** + ** Restore absent ambient state without adopting an external transaction or + ** replacing its primary preparation exception with a cleanup refusal. + ***************************************************************************/ + @Test + void callbackTransactionMutationIsRefusedAndOriginalFailurePreserved() throws Exception + { + Integer build = build(draft(newProject(), "test")).buildId(); + QBackendTransaction unknown = new QBackendTransaction(); + cold.callback = () -> QContext.setTransaction(unknown); + assertEquals(ScriptingException.Kind.DENIED, assertThrows(ScriptingException.class, + () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null)).getKind()); + assertNull(QContext.getQBackendTransaction()); + ScriptingException original = new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Cold lane failed"); + cold.callback = () -> + { + QContext.setTransaction(unknown); + throw original; + }; + assertSame(original, assertThrows(ScriptingException.class, + () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null))); + assertNull(QContext.getQBackendTransaction()); + assertEquals(0, cold.executeCalls); + } + + /*************************************************************************** + ** Existing native write ceilings apply before any cold resource work. + ***************************************************************************/ + @Test + void readOnlyCredentialCannotWarmCode() throws Exception + { + Integer build = build(draft(newProject(), "test")).buildId(); + QContext.getQSession().withCredentialScope(new QCredentialScope().withReadOnly(true)); + assertEquals(ScriptingException.Kind.DENIED, assertThrows(ScriptingException.class, + () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null)).getKind()); + assertEquals(0, cold.preparations); + assertEquals(0, cold.executeCalls); + } + + /*************************************************************************** + ** Re-enabling cannot silently adopt capabilities replaced during preparation. + ***************************************************************************/ + @Test + void changedBindingGrantsCannotBeEnabled() throws Exception + { + Integer binding = bindHook(publishNew(newProject(), "first")); + service.setBindingEnabled(everything(), binding, false); + cold.callback = () -> MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.BINDING) + .withRecords(List.of(new QRecord().withValue("id", binding).withValue("grants_json", "[]"))), false); + assertEquals(ScriptingException.Kind.INTEGRITY, assertThrows(ScriptingException.class, + () -> service.setBindingEnabled(everything(), binding, true)).getKind()); + assertEquals(0, rows(ScriptingTables.BINDING).getFirst().getValueInteger("enabled")); + } + + /*************************************************************************** + ** Legacy non-cache providers continue through the established lifecycle. + ***************************************************************************/ + @Test + void ordinaryProviderNeedsNoPreparationInterface() throws Exception + { + FakeWorker ordinary = new FakeWorker(); + runtime.withWorker(ordinary); + Integer publication = publishNew(newProject(), "ordinary"); + bindHook(publication); + assertEquals(1, ordinary.executeCalls); + assertEquals(0, cold.preparations); + } + + /*************************************************************************** + ** The authoring scope stays live at actual worker dispatch and releases + ** after success, post-prepare denial and publication validation failure. + ***************************************************************************/ + @Test + void authoringUseLeaseClosesOnDispatchAndValidationFailure() throws Exception + { + Integer build = build(draft(newProject(), "lease")).buildId(); + cold.onExecute = broker -> assertEquals(1, cold.activeUses); + assertTrue(service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null).succeeded()); + assertEquals(0, cold.activeUses); + assertEquals(1, cold.releases); + Integer untested = build(draft(newProject(), "untested")).buildId(); + assertThrows(ScriptingException.class, () -> service.publish(everything(), untested, "no passing test")); + assertEquals(0, cold.activeUses); + assertEquals(2, cold.releases); + cold.callback = () -> QContext.getQSession().setPermissions(Set.of(ScriptingPermissions.VIEW)); + assertThrows(ScriptingException.class, () -> service.test(everything(), build, HookEvent.PRE_INSERT, "{}", null)); + assertEquals(0, cold.activeUses); + assertEquals(3, cold.releases); + assertEquals(1, cold.executeCalls); + } + + /*************************************************************************** + ** Test-only callback can model native policy/data changes without scripts. + ***************************************************************************/ + @FunctionalInterface + private interface Callback + { + /*********************************************************************** + ** Execute the trusted test callback. + ***********************************************************************/ + void run() throws Exception; + } + + /*************************************************************************** + ** Protocol-only provider observes the exact cold input and owned transaction. + ***************************************************************************/ + private static final class ColdWorker extends FakeWorker implements WarmWorkerPreparation + { + private int preparations; + private int activeUses; + private int releases; + private Artifact last; + private Callback callback; + private boolean failure; + + /*********************************************************************** + ** Local use is reserved after successful preparation, before revalidation. + ***********************************************************************/ + @Override + public UseLease prepareForUse(Artifact artifact) throws ScriptingException + { + prepareVerified(artifact); + activeUses++; + return (() -> + { + activeUses--; + releases++; + }); + } + + /*********************************************************************** + ** No engine is invoked; an unavailable cache fails explicitly. + ***********************************************************************/ + @Override + public void prepareVerified(Artifact artifact) throws ScriptingException + { + assertNull(QContext.getQBackendTransaction()); + preparations++; + last = artifact; + if(failure) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Preparation unavailable"); + } + if(callback != null) + { + try + { + callback.run(); + } + catch(ScriptingException e) + { + throw e; + } + catch(Exception e) + { + throw new AssertionError(e); + } + } + } + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/CapabilityBrokerTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/CapabilityBrokerTest.java new file mode 100644 index 0000000..867cb78 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/CapabilityBrokerTest.java @@ -0,0 +1,337 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.scripting.capability.CapabilityBroker; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.capability.RecordReader; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The capability broker: opaque scoped tokens, read-only, host-verified table, + ** field, criteria and budgets, bounded responses, and no leaked failure detail. + *******************************************************************************/ +class CapabilityBrokerTest +{ + private final AtomicInteger readerCalls = new AtomicInteger(); + private int lastLimit; + private int rowsToReturn = 3; + private int rowWidth = 10; + private RuntimeException readerFailure; + private MutableClock clock; + private CapabilityBroker broker; + private String token; + + + + /******************************************************************************* + ** A clock a test can advance. + *******************************************************************************/ + private static class MutableClock extends Clock + { + private Instant now = Instant.parse("2026-10-30T00:00:00Z"); + + + + /*************************************************************************** + ** Move time forward. + ***************************************************************************/ + void advance(Duration duration) + { + now = now.plus(duration); + } + + + + /*************************************************************************** + ** UTC. + ***************************************************************************/ + @Override + public ZoneId getZone() + { + return (ZoneOffset.UTC); + } + + + + /*************************************************************************** + ** Zone changes are irrelevant here. + ***************************************************************************/ + @Override + public Clock withZone(ZoneId zone) + { + return (this); + } + + + + /*************************************************************************** + ** The current test instant. + ***************************************************************************/ + @Override + public Instant instant() + { + return (now); + } + } + + + + /*************************************************************************** + ** A broker over a fake reader and one grant (purchase_order, two fields, + ** 3 rows per call, 2 calls). + ***************************************************************************/ + @BeforeEach + void open() throws Exception + { + clock = new MutableClock(); + RecordReader reader = (table, fields, criteria, limit) -> + { + readerCalls.incrementAndGet(); + lastLimit = limit; + if(readerFailure != null) + { + throw readerFailure; + } + List> rows = new ArrayList<>(); + for(int i = 0; i < rowsToReturn; i++) + { + Map row = new LinkedHashMap<>(); + for(String field : fields) + { + row.put(field, "v".repeat(rowWidth)); + } + rows.add(row); + } + return (rows); + }; + broker = new CapabilityBroker(reader, clock); + token = broker.open(List.of(new CapabilityDeclaration("purchase_order", Set.of("po_number", "status"), 3, 2)), Duration.ofSeconds(10)); + } + + + + /*************************************************************************** + ** A broker request for the granted table. + ***************************************************************************/ + private String request(String tokenValue, String table, String fields, String criteria, int limit) + { + return (new JSONObject().put("protocol", WorkerProtocol.VERSION).put("type", "broker").put("requestId", "r1").put("operation", "query").put("token", tokenValue) + .put("table", table).put("fields", new org.json.JSONArray(fields)).put("criteria", new org.json.JSONArray(criteria)).put("limit", limit).toString()); + } + + + + /*************************************************************************** + ** A request that should succeed, parsed. + ***************************************************************************/ + private JSONObject ask(String fields, int limit) + { + return (new JSONObject(broker.handle(request(token, "purchase_order", fields, "[]", limit)))); + } + + + + /*************************************************************************** + ** The token is opaque random text; a valid read returns rows within budget. + ***************************************************************************/ + @Test + void validReadReturnsRows() throws Exception + { + assertTrue(token.startsWith("qct_")); + assertFalse(token.contains("purchase_order")); + + JSONObject response = ask("[\"po_number\"]", 100); + + assertTrue(response.getBoolean("ok")); + assertEquals(3, response.getJSONArray("rows").length()); + assertEquals(3, lastLimit); + } + + + + /*************************************************************************** + ** Unknown, closed and expired tokens are denied without reaching the reader. + ***************************************************************************/ + @Test + void unknownClosedAndExpiredTokensAreDenied() throws Exception + { + assertEquals("DENIED", new JSONObject(broker.handle(request("qct_guess", "purchase_order", "[\"po_number\"]", "[]", 1))).getString("code")); + + clock.advance(Duration.ofMinutes(5)); + assertEquals("DENIED", ask("[\"po_number\"]", 1).getString("code")); + + String fresh = broker.open(List.of(new CapabilityDeclaration("purchase_order", Set.of("status"), 1, 1)), Duration.ofSeconds(10)); + broker.close(fresh); + assertEquals("DENIED", new JSONObject(broker.handle(request(fresh, "purchase_order", "[\"status\"]", "[]", 1))).getString("code")); + assertEquals(0, readerCalls.get()); + } + + + + /*************************************************************************** + ** Tables, selected fields and filtered fields must all be granted. + ***************************************************************************/ + @Test + void tableAndFieldsMustBeGranted() throws Exception + { + assertEquals("DENIED", new JSONObject(broker.handle(request(token, "employee", "[\"po_number\"]", "[]", 1))).getString("code")); + assertEquals("DENIED", ask("[\"amount\"]", 1).getString("code")); + String filtered = request(token, "purchase_order", "[\"po_number\"]", "[{\"field\":\"amount\",\"operator\":\"GREATER_THAN\",\"values\":[1]}]", 1); + assertEquals("DENIED", new JSONObject(broker.handle(filtered)).getString("code")); + assertEquals(0, readerCalls.get()); + } + + + + /*************************************************************************** + ** Anything but a query is not even representable, so a write is refused. + ***************************************************************************/ + @Test + void onlyReadsAreAvailable() throws Exception + { + String update = request(token, "purchase_order", "[\"status\"]", "[]", 1).replace("\"query\"", "\"update\""); + JSONObject response = new JSONObject(broker.handle(update)); + assertFalse(response.getBoolean("ok")); + assertEquals("DENIED", response.getString("code")); + assertEquals(0, readerCalls.get()); + } + + + + /*************************************************************************** + ** Call and row budgets are enforced per lease. + ***************************************************************************/ + @Test + void callAndRowBudgetsAreEnforced() throws Exception + { + assertTrue(ask("[\"po_number\"]", 3).getBoolean("ok")); + assertTrue(ask("[\"po_number\"]", 3).getBoolean("ok")); + JSONObject third = ask("[\"po_number\"]", 3); + assertFalse(third.getBoolean("ok")); + assertEquals("LIMIT", third.getString("code")); + assertEquals(2, readerCalls.get()); + + String rowLimited = broker.open(List.of(new CapabilityDeclaration("purchase_order", Set.of("status"), 2, 1)), Duration.ofSeconds(10)); + broker.handle(request(rowLimited, "purchase_order", "[\"status\"]", "[]", 50)); + assertEquals(2, lastLimit); + } + + + + /*************************************************************************** + ** An oversized answer is truncated to the response bound and flagged. + ***************************************************************************/ + @Test + void responseSizeIsBounded() throws Exception + { + String big = broker.open(List.of(new CapabilityDeclaration("purchase_order", Set.of("po_number"), CapabilityDeclaration.ABSOLUTE_MAX_ROWS, 1)), Duration.ofSeconds(10)); + rowsToReturn = CapabilityDeclaration.ABSOLUTE_MAX_ROWS; + rowWidth = 4000; + + String raw = broker.handle(request(big, "purchase_order", "[\"po_number\"]", "[]", CapabilityDeclaration.ABSOLUTE_MAX_ROWS)); + JSONObject response = new JSONObject(raw); + + assertTrue(response.getBoolean("truncated")); + assertTrue(response.getJSONArray("rows").length() < CapabilityDeclaration.ABSOLUTE_MAX_ROWS); + assertTrue(raw.getBytes(java.nio.charset.StandardCharsets.UTF_8).length <= CapabilityBroker.MAX_RESPONSE_BYTES); + } + + + + /*************************************************************************** + ** Garbage and reader failures produce generic error answers, never throws. + ***************************************************************************/ + @Test + void failuresAreGenericAndNeverThrow() throws Exception + { + JSONObject garbage = new JSONObject(broker.handle("not json")); + assertFalse(garbage.getBoolean("ok")); + + readerFailure = new IllegalStateException("jdbc:mysql://prod-db password=hunter2"); + JSONObject failed = ask("[\"po_number\"]", 1); + assertFalse(failed.getBoolean("ok")); + assertFalse(failed.toString().contains("hunter2")); + } + + + + /*************************************************************************** + ** Declarations validate, grants must be within declarations, and only READ + ** parses back. + ***************************************************************************/ + @Test + void declarationsAreValidatedAndOrdered() throws Exception + { + CapabilityDeclaration declared = new CapabilityDeclaration("purchase_order", Set.of("status", "po_number"), 10, 2); + assertTrue(new CapabilityDeclaration("purchase_order", Set.of("status"), 5, 1).isWithin(declared)); + assertFalse(new CapabilityDeclaration("purchase_order", Set.of("amount"), 5, 1).isWithin(declared)); + assertFalse(new CapabilityDeclaration("purchase_order", Set.of("status"), 11, 1).isWithin(declared)); + assertFalse(new CapabilityDeclaration("other", Set.of("status"), 5, 1).isWithin(declared)); + + assertEquals(List.of(declared), CapabilityDeclaration.listFromJson(CapabilityDeclaration.listToJson(List.of(declared)))); + assertKind(ScriptingException.Kind.INVALID, () -> CapabilityDeclaration.listFromJson(CapabilityDeclaration.listToJson(List.of(declared)).replace("READ", "WRITE"))); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, () -> new CapabilityDeclaration("t", Set.of("a"), CapabilityDeclaration.ABSOLUTE_MAX_ROWS + 1, 1)); + org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class, () -> new CapabilityDeclaration("t; drop", Set.of("a"), 1, 1)); + } + + + + /*************************************************************************** + ** Container setup fits a bounded transport lease; expiry and close still + ** revoke its capability even when the script itself has a shorter budget. + ***************************************************************************/ + @Test + void transportLeaseIncludesSetupButIsBounded() throws Exception + { + List grants = List.of(new CapabilityDeclaration("purchase_order", Set.of("status"), 1, 3)); + assertKind(ScriptingException.Kind.LIMIT, () -> broker.open(grants, Duration.ofSeconds(121))); + assertKind(ScriptingException.Kind.LIMIT, () -> broker.open(grants, Duration.ZERO)); + String transportToken = broker.open(grants, Duration.ofSeconds(90)); + clock.advance(Duration.ofSeconds(30)); + assertTrue(new JSONObject(broker.handle(request(transportToken, "purchase_order", "[\"status\"]", "[]", 1))).getBoolean("ok")); + clock.advance(Duration.ofSeconds(60)); + assertFalse(new JSONObject(broker.handle(request(transportToken, "purchase_order", "[\"status\"]", "[]", 1))).getBoolean("ok")); + broker.close(transportToken); + assertFalse(new JSONObject(broker.handle(request(transportToken, "purchase_order", "[\"status\"]", "[]", 1))).getBoolean("ok")); + } +} + diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/CoordinatorWorkerProviderTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/CoordinatorWorkerProviderTest.java new file mode 100644 index 0000000..fe183b1 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/CoordinatorWorkerProviderTest.java @@ -0,0 +1,195 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.scripting.worker.CoordinatorWorkerProvider; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Real process pipes exercise transport trust, framing, deadlines and cleanup. + ** The child is a fixed trusted fixture, never an untrusted-script interpreter. + *******************************************************************************/ +class CoordinatorWorkerProviderTest +{ + @TempDir + Path directory; + + + /*************************************************************************** + ** Compile result needs matching identity and a clean coordinator exit. + ** Child also verifies ambient home/Java settings were not inherited. + ***************************************************************************/ + @Test + void compileUsesFixedProcessAndScrubbedEnvironment() throws Exception + { + var provider = provider(Duration.ofSeconds(15)); + assertEquals("fixture-evidence", provider.qualificationId()); + assertEquals(Duration.ofSeconds(15), provider.executionTransportBudget()); + JSONObject answer = new JSONObject(provider.compile(request("compile", "success"))); + assertEquals("compileResult", answer.getString("type")); + assertEquals("success", answer.getString("requestId")); + } + + + + /*************************************************************************** + ** Broker runs on the authenticated calling thread and consumes its reply. + ***************************************************************************/ + @Test + void brokerStaysOnCallerThreadAndRejectsForgedToken() throws Exception + { + var provider = provider(Duration.ofSeconds(15)); + Thread caller = Thread.currentThread(); + AtomicInteger calls = new AtomicInteger(); + String result = provider.execute(request("execute", "success"), json -> + { + assertEquals(caller, Thread.currentThread()); + assertEquals("fixture-token", new JSONObject(json).getString("token")); + calls.incrementAndGet(); + return (WorkerProtocol.encodeBrokerResponse(List.of(), false)); + }); + assertEquals("executeResult", new JSONObject(result).getString("type")); + assertEquals(1, calls.get()); + ScriptingException failure = assertThrows(ScriptingException.class, () -> provider.execute(request("execute", "forged"), json -> + { + calls.incrementAndGet(); + return ("{}"); + })); + assertEquals(ScriptingException.Kind.INTEGRITY, failure.getKind()); + assertEquals(1, calls.get()); + } + + + + /*************************************************************************** + ** Final output alone cannot override wrong identity, duplicate frames or + ** failed cleanup represented by coordinator nonzero exit. + ***************************************************************************/ + @Test + void malformedAndUnsuccessfulExitsFailClosed() + { + var provider = provider(Duration.ofSeconds(15)); + for(String mode : List.of("wrong-id", "duplicate", "nonzero", "partial", "huge", "diagnostics")) + { + ScriptingException failure = assertThrows(ScriptingException.class, () -> provider.compile(request("compile", mode)), mode); + assertFalse(failure.getMessage().contains("private-fixture")); + assertEquals(0, failure.getSuppressed().length); + } + assertThrows(ScriptingException.class, () -> provider.compile("not json")); + assertThrows(ScriptingException.class, () -> provider.compile(request("execute", "success"))); + } + + + + /*************************************************************************** + ** A silent process cannot hold a generation/hook request indefinitely. + ***************************************************************************/ + @Test + void silentProcessIsTerminatedWithinDeadline() + { + var provider = provider(Duration.ofMillis(200)); + long start = System.nanoTime(); + assertThrows(ScriptingException.class, () -> provider.compile(request("compile", "hang"))); + assertTrue(Duration.ofNanos(System.nanoTime() - start).compareTo(Duration.ofSeconds(5)) < 0); + } + + + + /*************************************************************************** + ** Callback errors and invalid replies cause cleanup, never success. + ***************************************************************************/ + @Test + void brokerFailureDoesNotReleaseWorkerResult() + { + var provider = provider(Duration.ofSeconds(15)); + assertThrows(ScriptingException.class, () -> provider.execute(request("execute", "success"), json -> + { + throw new IllegalStateException("private-fixture-diagnostic"); + })); + assertThrows(ScriptingException.class, () -> provider.execute(request("execute", "success"), json -> "{}")); + } + + + + /*************************************************************************** + ** Trust configuration must be pinned and bounded before process launch. + ***************************************************************************/ + @Test + void rejectsMutableImageAndInvalidConfiguration() + { + Path executable = executable(); + Path cli = Path.of("src/test/resources/FixtureCoordinator.java").toAbsolutePath(); + assertThrows(IllegalArgumentException.class, () -> new CoordinatorWorkerProvider.Configuration(executable, cli, + "worker:latest", "fixture-evidence", "fixture-evidence", "fixture", directory, "", Duration.ofSeconds(1), Duration.ZERO)); + assertThrows(IllegalArgumentException.class, () -> new CoordinatorWorkerProvider.Configuration(executable, cli, + "worker@sha256:" + "a".repeat(64), "fixture-evidence", "different", "fixture", directory, "", Duration.ofSeconds(1), Duration.ZERO)); + assertThrows(IllegalArgumentException.class, () -> new CoordinatorWorkerProvider.Configuration(executable, cli, + "worker@sha256:" + "a".repeat(64), "fixture-evidence", "fixture-evidence", "fixture", directory, "", Duration.ZERO, Duration.ZERO)); + } + + + + /*************************************************************************** + ** Configure only operator-owned fixture paths, with no user arguments. + ***************************************************************************/ + private CoordinatorWorkerProvider provider(Duration deadline) + { + return (new CoordinatorWorkerProvider(new CoordinatorWorkerProvider.Configuration(executable(), + Path.of("src/test/resources/FixtureCoordinator.java").toAbsolutePath(), "worker@sha256:" + "a".repeat(64), + "fixture-evidence", "fixture-evidence", "fixture", directory, System.getenv().getOrDefault("PATH", ""), + deadline, Duration.ofMillis(200)))); + } + + + + /*************************************************************************** + ** Use this test JVM's installed Java, portable across supported build hosts. + ***************************************************************************/ + private static Path executable() + { + String name = System.getProperty("os.name").startsWith("Windows") ? "java.exe" : "java"; + return (Path.of(System.getProperty("java.home"), "bin", name).toAbsolutePath()); + } + + + + /*************************************************************************** + ** The fixture accepts only a small envelope; it never evaluates user code. + ***************************************************************************/ + private static String request(String type, String mode) + { + return (new JSONObject().put("protocol", WorkerProtocol.VERSION).put("type", type).put("requestId", mode) + .put("capabilityToken", "fixture-token").toString()); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/FakeWorker.java b/base/src/test/java/com/kingsrook/qbits/scripting/FakeWorker.java new file mode 100644 index 0000000..7eaddcb --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/FakeWorker.java @@ -0,0 +1,124 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.function.Consumer; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import com.kingsrook.qbits.scripting.worker.WorkerProvider; +import org.json.JSONArray; +import org.json.JSONObject; + + +/******************************************************************************* + ** A protocol-level test double. It NEVER evaluates any script text: it returns + ** canned protocol JSON, like a stub HTTP server would. compile "succeeds" by + ** echoing a deterministic placeholder bundle derived from the source hash; run + ** returns whichever hook decision the test configured. + *******************************************************************************/ +class FakeWorker implements WorkerProvider +{ + String qualification = "qual-1"; + String toolchain = ScriptingTestBase.TOOLCHAIN; + boolean compileOk = true; + boolean strict = true; + boolean unavailable = false; + String hookJson = "{\"action\":\"ALLOW\"}"; + String failureCode; + String failureMessage; + String lastExecuteRequest; + int executeCalls = 0; + Consumer onExecute; + + + + /*************************************************************************** + ** Configured qualification evidence id. + ***************************************************************************/ + @Override + public String qualificationId() + { + return (qualification); + } + + + + /*************************************************************************** + ** Canned compile answer. + ***************************************************************************/ + @Override + public String compile(String compileRequestJson) throws ScriptingException + { + if(unavailable) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "worker down"); + } + JSONObject request = new JSONObject(compileRequestJson); + JSONObject response = new JSONObject() + .put("protocol", WorkerProtocol.VERSION) + .put("type", "compileResult") + .put("requestId", request.getString("requestId")) + .put("strict", strict) + .put("toolchainId", toolchain); + if(!compileOk) + { + return (response.put("success", false) + .put("diagnostics", new JSONArray().put(new JSONObject().put("file", "src/index.ts").put("line", 1).put("column", 2).put("code", "TS2322").put("message", "Type 'string' is not assignable to type 'number'."))) + .toString()); + } + String bundle = "/* placeholder bundle for " + request.getString("sourceSha256") + " */"; + return (response.put("success", true).put("bundle", bundle).put("bundleSha256", BuildCoordinator.sha256Hex(bundle)).toString()); + } + + + + /*************************************************************************** + ** Canned run answer. Like a real worker it first verifies the bundle hash. + ***************************************************************************/ + @Override + public String execute(String executeRequestJson, BrokerChannel broker) throws ScriptingException + { + if(unavailable) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "worker down"); + } + executeCalls++; + lastExecuteRequest = executeRequestJson; + if(onExecute != null) + { + onExecute.accept(broker); + } + JSONObject request = new JSONObject(executeRequestJson); + JSONObject response = new JSONObject().put("protocol", WorkerProtocol.VERSION).put("type", "executeResult").put("requestId", request.getString("requestId")); + if(!BuildCoordinator.sha256Hex(request.getString("bundle")).equals(request.getString("artifactSha256"))) + { + return (response.put("outcome", "CRASHED").put("error", new JSONObject().put("code", "HASH").put("message", "bundle hash mismatch")).toString()); + } + if(failureCode != null) + { + return (response.put("outcome", "SCRIPT_ERROR").put("error", new JSONObject().put("code", failureCode).put("message", failureMessage)).toString()); + } + return (response.put("outcome", "OK").put("result", new JSONObject(hookJson)).toString()); + } +} + + diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/HookAuthoringTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/HookAuthoringTest.java new file mode 100644 index 0000000..828478b --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/HookAuthoringTest.java @@ -0,0 +1,228 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHooks; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Semantic hook controls use native target/source authorization and real rows. + *******************************************************************************/ +class HookAuthoringTest extends ScriptingTestBase +{ + private Integer project; + private Integer publication; + private Integer buildId; + + /*************************************************************************** + ** A pinned fixture publication with the host's ordinary insert hook installed. + ***************************************************************************/ + @BeforeEach + void prepare() throws Exception + { + project = newProject(); + publication = publishNew(project, "hook-admin"); + buildId = rows(ScriptingTables.PUBLICATION).getLast().getValueInteger("build_id"); + } + + /*************************************************************************** + ** A single bounded public binding request. + ***************************************************************************/ + private BindingSpec spec(HookEvent event) + { + return (new BindingSpec("hook", HOOK_TABLE, event, publication, declared(), Set.of("status"))); + } + + /*************************************************************************** + ** Only explicitly installed phases and current editable fields are selectable. + ***************************************************************************/ + @Test + void targetsRequireHostOptInAndCurrentNativeAuthority() throws Exception + { + var view = service.hooks().view(everything(), project); + assertEquals(1, view.targets().size()); + assertEquals(List.of(HookEvent.PRE_INSERT), view.targets().getFirst().phases().stream().map(phase -> phase.event()).toList()); + assertFalse(view.targets().getFirst().phases().getFirst().patchableFields().contains("created_by")); + assertThrows(ScriptingException.class, () -> service.hooks().bind(everything(), project, spec(HookEvent.PRE_UPDATE))); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + assertTrue(service.hooks().view(everything(), project).targets().isEmpty()); + assertThrows(ScriptingException.class, () -> service.hooks().bind(everything(), project, spec(HookEvent.PRE_INSERT))); + assertThrows(ScriptingException.class, () -> service.bind(everything(), spec(HookEvent.PRE_INSERT))); + } + + /*************************************************************************** + ** Create and state changes have atomic attributed activity receipts. + ***************************************************************************/ + @Test + void bindingControlsArePinnedScopedAndAudited() throws Exception + { + int before = rows(ScriptingTables.EXECUTION).size(); + Integer id = service.hooks().bind(everything(), project, spec(HookEvent.PRE_INSERT)); + assertEquals(before + 1, rows(ScriptingTables.EXECUTION).size()); + assertEquals("BINDING_CHANGE", rows(ScriptingTables.EXECUTION).getLast().getValueString("kind")); + assertEquals("CREATED", rows(ScriptingTables.EXECUTION).getLast().getValueString("status")); + service.hooks().setEnabled(everything(), project, id, false); + assertFalse(service.hooks().view(everything(), project).bindings().getFirst().enabled()); + assertEquals("DISABLED", rows(ScriptingTables.EXECUTION).getLast().getValueString("status")); + service.hooks().setEnabled(everything(), project, id, true); + assertEquals(publication, service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).orElseThrow().publicationId()); + assertThrows(ScriptingException.class, () -> service.hooks().setEnabled(only(ScriptingPermissions.VIEW), project, id, false)); + Integer other = newProject(); + assertThrows(ScriptingException.class, () -> service.hooks().setEnabled(everything(), other, id, false)); + assertThrows(ScriptingException.class, () -> service.hooks().bind(everything(), other, spec(HookEvent.PRE_INSERT))); + assertTrue(service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).isPresent()); + } + + /*************************************************************************** + ** A built decision is tested with real prior values but never applies a patch. + ***************************************************************************/ + @Test + void selectedRecordTestIsReadOnlyAndAudited() throws Exception + { + ScriptHooks.install(instance.getTable(HOOK_TABLE), HookEvent.PRE_UPDATE); + QRecord stored = new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", "OLD")))).getRecords().getFirst(); + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\"RESULT\"}}"; + var result = service.hooks().test(everything(), project, buildId, HOOK_TABLE, HookEvent.PRE_UPDATE, stored.getValue("id"), Map.of("status", "PROPOSED")); + assertTrue(result.succeeded()); + assertEquals("OLD", rows(HOOK_TABLE).getFirst().getValueString("status")); + JSONObject payload = new JSONObject(worker.lastExecuteRequest).getJSONObject("payload"); + assertEquals("OLD", payload.getJSONObject("prior").getString("status")); + assertEquals("PROPOSED", payload.getJSONObject("proposed").getString("status")); + assertEquals("TEST", rows(ScriptingTables.EXECUTION).getLast().getValueString("kind")); + assertThrows(ScriptingException.class, () -> service.hooks().test(everything(), project, buildId, HOOK_TABLE, HookEvent.PRE_UPDATE, stored.getValue("id"), Map.of("created_by", "forged"))); + assertThrows(ScriptingException.class, () -> service.hooks().test(only(ScriptingPermissions.VIEW), project, buildId, HOOK_TABLE, HookEvent.PRE_UPDATE, stored.getValue("id"), Map.of())); + assertThrows(ScriptingException.class, () -> service.hooks().test(everything(), project, buildId, HOOK_TABLE, HookEvent.PRE_UPDATE, 999, Map.of())); + } + + /*************************************************************************** + ** Worker failures cannot copy record data into project-visible history. + ***************************************************************************/ + @Test + void workerFailureDetailsNeverReachHistoryOrTestSummary() throws Exception + { + QRecord stored = new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", "PRIVATE-RECORD-VALUE")))).getRecords().getFirst(); + worker.failureCode = "PRIVATE-CODE"; + worker.failureMessage = "PRIVATE-RECORD-VALUE"; + var result = service.hooks().test(everything(), project, buildId, HOOK_TABLE, HookEvent.PRE_INSERT, stored.getValue("id"), Map.of()); + assertEquals("SCRIPT_ERROR", result.status()); + assertEquals("The isolated worker reported SCRIPT_ERROR", result.message()); + assertEquals(result.message(), rows(ScriptingTables.EXECUTION).getLast().getValueString("message")); + assertFalse(rows(ScriptingTables.EXECUTION).toString().contains("PRIVATE-CODE")); + assertFalse(rows(ScriptingTables.EXECUTION).toString().contains("PRIVATE-RECORD-VALUE")); + worker.failureCode = null; + worker.onExecute = channel -> + { + throw new RuntimeException("PRIVATE-TRANSPORT-VALUE"); + }; + var transport = service.hooks().test(everything(), project, buildId, HOOK_TABLE, HookEvent.PRE_INSERT, stored.getValue("id"), Map.of()); + assertEquals("UNAVAILABLE", transport.status()); + assertFalse(transport.message().contains("PRIVATE")); + assertFalse(rows(ScriptingTables.EXECUTION).toString().contains("PRIVATE-TRANSPORT-VALUE")); + } + + /*************************************************************************** + ** Native read policy is rechecked after execution before any decision is returned. + ***************************************************************************/ + @Test + void currentReadRevocationRefusesTestOutput() throws Exception + { + QRecord stored = new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", "PRIVATE")))).getRecords().getFirst(); + worker.onExecute = channel -> instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED) + .withCustomPermissionChecker(new QCodeReference(NativeHookReadSecurityTest.DenyReads.class))); + assertThrows(ScriptingException.class, () -> service.hooks().test(everything(), project, buildId, HOOK_TABLE, HookEvent.PRE_INSERT, stored.getValue("id"), Map.of())); + } + + /*************************************************************************** + ** A removed phase can be disabled but cannot be enabled again through this API. + ***************************************************************************/ + @Test + void removedPhaseCannotBeActivated() throws Exception + { + Integer id = service.hooks().bind(everything(), project, spec(HookEvent.PRE_INSERT)); + instance.getTable(HOOK_TABLE).getCustomizers().remove(TableCustomizers.PRE_INSERT_RECORD.getRole()); + service.hooks().setEnabled(everything(), project, id, false); + assertThrows(ScriptingException.class, () -> service.hooks().setEnabled(everything(), project, id, true)); + } + + /*************************************************************************** + ** Public Java state changes cannot bypass target authorization in the facade. + ***************************************************************************/ + @Test + void directServiceActivationRequiresCurrentTargetPermission() throws Exception + { + Integer id = service.bind(everything(), spec(HookEvent.PRE_INSERT)); + service.setBindingEnabled(everything(), id, false); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + assertThrows(ScriptingException.class, () -> service.setBindingEnabled(everything(), id, true)); + assertThrows(ScriptingException.class, () -> service.setBindingEnabled(everything(), id, false)); + assertTrue(service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).isEmpty()); + } + + + /*************************************************************************** + ** Active bindings remain administrable after source history and binding pages grow. + ***************************************************************************/ + @Test + void oldPublicationBindingRemainsVisibleAcrossPages() throws Exception + { + Integer active = service.hooks().bind(everything(), project, spec(HookEvent.PRE_INSERT)); + for(int i = 0; i < 101; i++) + { + service.publish(everything(), buildId, "later"); + } + assertEquals(active, service.hooks().view(everything(), project).bindings().getFirst().id()); + ScriptHooks.install(instance.getTable(HOOK_TABLE), HookEvent.PRE_UPDATE); + for(int i = 0; i < 101; i++) + { + Integer id = service.hooks().bind(everything(), project, spec(HookEvent.PRE_UPDATE)); + service.hooks().setEnabled(everything(), project, id, false); + } + var first = service.hooks().view(everything(), project); + assertEquals(100, first.bindings().size()); + assertTrue(first.historyLimited()); + var second = service.hooks().view(everything(), project, first.nextBindingId()); + assertTrue(second.bindings().stream().anyMatch(row -> row.id().equals(active) && row.enabled())); + service.hooks().setEnabled(everything(), project, active, false); + assertTrue(service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).isEmpty()); + } + +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/IsolationIntegrationTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/IsolationIntegrationTest.java new file mode 100644 index 0000000..1205ad3 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/IsolationIntegrationTest.java @@ -0,0 +1,210 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.List; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.scripts.ScriptExecutionIsolation; +import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptExecutionMode; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Reviewed core marker integration, immutable source and script-origin guard. + *******************************************************************************/ +class IsolationIntegrationTest extends ScriptingTestBase +{ + /*************************************************************************** + ** Saving never activates; only tested publication advances the core pointer. + ***************************************************************************/ + @Test + void creationAndPublicationUseCoreAuthority() throws Exception + { + Integer project = newProject(); + assertEquals(ScriptExecutionMode.ISOLATED, ScriptExecutionIsolation.getExecutionMode(project, null)); + Integer revision = draft(project, "first"); + assertNull(rows("script").getFirst().getValueInteger("currentScriptRevisionId")); + var built = build(revision); + assertTrue(service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + service.publish(everything(), built.buildId(), "reviewed"); + assertEquals(revision, rows("script").getFirst().getValueInteger("currentScriptRevisionId")); + draft(project, "second"); + assertEquals(revision, rows("script").getFirst().getValueInteger("currentScriptRevisionId")); + assertKind(ScriptingException.Kind.INVALID, () -> service.createProject(everything(), " ")); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW)); + assertKind(ScriptingException.Kind.DENIED, () -> service.createProject(everything(), "Denied")); + } + + + + /*************************************************************************** + ** Generic writes cannot insert, edit or move isolated sources into legacy. + ***************************************************************************/ + @Test + void immutableSourcesCannotBeReparentedOrDeleted() throws Exception + { + Integer project = newProject(); + Integer revision = draft(project, "first"); + Integer legacy = newLegacyProject(); + assertThrows(QException.class, () -> new InsertAction().execute(new InsertInput().withTableName("scriptRevision") + .withRecords(List.of(new QRecord().withValue("scriptId", project).withValue("sequenceNo", 9))))); + assertThrows(QException.class, () -> new UpdateAction().execute(new UpdateInput("scriptRevision") + .withRecords(List.of(new QRecord().withValue("id", revision).withValue("scriptId", legacy))))); + QRecord file = rows("scriptRevisionFile").getFirst(); + assertThrows(QException.class, () -> new UpdateAction().execute(new UpdateInput("scriptRevisionFile") + .withRecords(List.of(new QRecord().withValue("id", file.getValueInteger("id")).withValue("contents", "changed"))))); + DeleteInput delete = new DeleteInput(); + delete.setTableName("scriptRevision"); + delete.setPrimaryKeys(List.of(revision)); + assertThrows(QException.class, () -> new DeleteAction().execute(delete)); + assertEquals(project, rows("scriptRevision").getFirst().getValueInteger("scriptId")); + assertEquals(file.getValueString("contents"), rows("scriptRevisionFile").getFirst().getValueString("contents")); + } + + + + /*************************************************************************** + ** Orphan intake and indirect source disclosure both fail closed. + ***************************************************************************/ + @Test + void sourceReadsRequireParentAndOrphansCannotBeInserted() throws Exception + { + draft(newProject(), "private"); + assertThrows(QException.class, () -> new InsertAction().execute(new InsertInput().withTableName("scriptRevisionFile") + .withRecords(List.of(new QRecord().withValue("scriptRevisionId", 99999).withValue("fileName", "x.ts").withValue("contents", ""))))); + instance.getTable("script").setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + assertTrue(rows("scriptRevision").isEmpty()); + assertTrue(rows("scriptRevisionFile").isEmpty()); + } + + + + /*************************************************************************** + ** Neither aggregate endpoint can infer source rows before parent filtering. + ** Metadata also avoids offering generic totals to the frontend. + ***************************************************************************/ + @Test + void genericSourceAggregatesAreUnavailableToOrdinaryCallers() throws Exception + { + draft(newProject(), "private"); + for(String table : List.of("scriptRevision", "scriptRevisionFile")) + { + CountInput count = new CountInput(); + count.setTableName(table); + AggregateInput aggregate = new AggregateInput(table); + aggregate.setAggregates(List.of(new Aggregate("id", AggregateOperator.COUNT))); + assertThrows(QPermissionDeniedException.class, () -> new CountAction().execute(count)); + assertThrows(QPermissionDeniedException.class, () -> new AggregateAction().execute(aggregate)); + assertFalse(PermissionsHelper.hasTablePermission(new QueryInput().withTableName(HOOK_TABLE), table, TablePermissionSubType.READ)); + assertThrows(QPermissionDeniedException.class, () -> new QueryAction().execute(new QueryInput().withTableName(table).withQueryJoin(new QueryJoin(table).withAlias("other")))); + assertTrue(instance.getTable(table).getDisabledCapabilities().containsAll(List.of(Capability.TABLE_COUNT, Capability.QUERY_STATS))); + } + var caller = QContext.getQSession(); + try + { + QContext.setQSession(new QSystemUserSession()); + CountInput count = new CountInput(); + count.setTableName("scriptRevision"); + assertEquals(1, new CountAction().execute(count).getCount()); + } + finally + { + QContext.setQSession(caller); + } + } + + + + /*************************************************************************** + ** The decorator preserves host read/write levels and specialized checkers. + ***************************************************************************/ + @Test + void aggregateGuardDoesNotBroadenHostSourcePermissions() throws Exception + { + draft(newProject(), "private"); + var table = instance.getTable("scriptRevision"); + table.getPermissionRules().setLevel(PermissionLevel.READ_WRITE_PERMISSIONS); + QContext.setQSession(sessionWith("scriptRevision.write")); + assertThrows(QException.class, () -> rows("scriptRevision")); + QContext.setQSession(sessionWith("scriptRevision.read")); + assertEquals(1, rows("scriptRevision").size()); + var wrapped = SourceAggregatePermission.wrap(UseOtherPermissionNameCustomPermissionChecker.build("host.source")); + table.getPermissionRules().setCustomPermissionChecker(wrapped); + assertThrows(QException.class, () -> rows("scriptRevision")); + QContext.setQSession(sessionWith("host.source")); + assertEquals(1, rows("scriptRevision").size()); + CustomPermissionChecker checker = QCodeLoader.getAdHoc(CustomPermissionChecker.class, wrapped); + assertTrue(checker.handlesBuildAvailablePermission()); + assertEquals("host.source", checker.buildAvailablePermission(table.getPermissionRules(), TablePermissionSubType.READ, + "scriptRevision", table, "Table").getName()); + } + + + + /*************************************************************************** + ** Script-origin broker callbacks cannot acquire trusted isolation authority. + ***************************************************************************/ + @Test + void workerHostCallbackCannotAcquireCreationAuthority() throws Exception + { + var built = build(draft(newProject(), "first")); + worker.onExecute = broker -> assertThrows(IllegalStateException.class, () -> ScriptExecutionIsolation.runAsIsolationAuthority("Forbidden callback", () -> true)); + assertTrue(service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + assertTrue(ScriptExecutionIsolation.runAsIsolationAuthority("Outside execution", () -> true)); + Integer legacy = newLegacyProject(); + assertKind(ScriptingException.Kind.DENIED, () -> service.saveDraft(everything(), legacy, "legacy", source("no"))); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/JdbcScriptingTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/JdbcScriptingTest.java new file mode 100644 index 0000000..3cb4d93 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/JdbcScriptingTest.java @@ -0,0 +1,511 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHooks; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Real JDBC commit/rollback and independently committed history. H2 exercises + ** core RDBMS actions on separate connections; PostgreSQL qualification is separate. + *******************************************************************************/ +class JdbcScriptingTest extends ScriptingTestBase +{ + private String jdbcUrl; + + + + /******************************************************************************* + ** Hold two publishers immediately before their inserts so the real database + ** must arbitrate the same project/version key. + *******************************************************************************/ + public static class PublicationRaceGate implements TableCustomizerInterface + { + static CyclicBarrier gate; + + + + /*************************************************************************** + ** Wait only in the controlled race test. + ***************************************************************************/ + @Override + public List preInsert(InsertInput input, List records, boolean preview) throws QException + { + try + { + gate.await(5, TimeUnit.SECONDS); + return (records); + } + catch(Exception e) + { + throw new QException("Publication race fixture timed out", e); + } + } + } + + + + /*************************************************************************** + ** Concurrent max-plus-one numbering cannot create duplicate publications. + ** One caller gets a committed publication and the conflicting caller fails. + ***************************************************************************/ + @Test + void databaseUniqueKeyArbitratesConcurrentPublish() throws Exception + { + var built = build(draft(newProject(), "race")); + service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null); + PublicationRaceGate.gate = new CyclicBarrier(2); + instance.getTable(ScriptingTables.PUBLICATION).withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(PublicationRaceGate.class)); + var session = QContext.getQSession(); + try(var pool = Executors.newFixedThreadPool(2)) + { + java.util.concurrent.Callable publish = () -> + { + QContext.init(instance, session); + try + { + service.publish(everything(), built.buildId(), "race"); + return (true); + } + catch(ScriptingException e) + { + return (false); + } + finally + { + QContext.clear(); + } + }; + var first = pool.submit(publish); + var second = pool.submit(publish); + int successes = (first.get(10, TimeUnit.SECONDS) ? 1 : 0) + (second.get(10, TimeUnit.SECONDS) ? 1 : 0); + assertEquals(1, successes); + } + assertEquals(1, count(ScriptingTables.PUBLICATION)); + } + + + + /*************************************************************************** + ** Replace the fixture's volatile backend with isolated JDBC tables and actual + ** database uniqueness constraints derived from the public metadata. + ***************************************************************************/ + @BeforeEach + void useJdbc() throws Exception + { + createJdbc("jdbc:h2:mem:scripting_" + UUID.randomUUID() + ";DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE"); + } + + + /*************************************************************************** + ** Shared real JDBC fixture schema for the optional qualified file adapter. + ***************************************************************************/ + void createJdbc(String url) throws Exception + { + jdbcUrl = url; + String backendName = "jdbc_" + UUID.randomUUID().toString().replace("-", ""); + instance.addBackend(new RDBMSBackendMetaData().withName(backendName).withVendor("h2").withJdbcUrl(jdbcUrl).withUsername("sa")); + try(Connection connection = DriverManager.getConnection(jdbcUrl, "sa", ""); Statement statement = connection.createStatement()) + { + for(QTableMetaData table : instance.getTables().values()) + { + table.setBackendName(backendName); + table.setBackendDetails(new RDBMSTableBackendDetails().withTableName(table.getName())); + List columns = new ArrayList<>(); + for(QFieldMetaData field : table.getFields().values()) + { + field.setBackendName(field.getName()); + String type = switch(field.getType()) + { + case INTEGER -> "INTEGER"; + case LONG -> "BIGINT"; + case BOOLEAN -> "BOOLEAN"; + case DECIMAL -> "DECIMAL(30,10)"; + case DATE_TIME -> "TIMESTAMP"; + case DATE -> "DATE"; + case TIME -> "TIME"; + case TEXT, HTML -> "CLOB"; + default -> "VARCHAR(2000)"; + }; + columns.add(field.getName() + " " + (field.getName().equals("id") ? "INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY" : type)); + } + if(table.getUniqueKeys() != null) + { + for(UniqueKey key : table.getUniqueKeys()) + { + columns.add("UNIQUE (" + String.join(",", key.getFieldNames()) + ")"); + } + } + statement.execute("CREATE TABLE " + table.getName() + " (" + String.join(",", columns) + ")"); + } + } + } + + + + /*************************************************************************** + ** Execute fixture DDL only, never application/user SQL. + ***************************************************************************/ + void sql(String statement) throws Exception + { + try(Connection connection = jdbcConnection(); Statement sql = connection.createStatement()) + { + sql.execute(statement); + } + } + + + + /*************************************************************************** + ** Observe committed rows using a separate JDBC connection. + ***************************************************************************/ + int count(String table) throws Exception + { + try(Connection connection = jdbcConnection(); Statement statement = connection.createStatement(); + var result = statement.executeQuery("SELECT COUNT(*) FROM " + table)) + { + result.next(); + return (result.getInt(1)); + } + } + + + + /*************************************************************************** + ** Separate observer connection; subclasses exercise the same native behavior + ** against their real database engine and migration. + ***************************************************************************/ + Connection jdbcConnection() throws Exception + { + return (DriverManager.getConnection(jdbcUrl, "sa", "")); + } + + + + /*************************************************************************** + ** Failure after the first file rolls the entire revision and its files back. + ***************************************************************************/ + @Test + void draftRollbackRemovesPartialRevisionAndFiles() throws Exception + { + Integer project = newProject(); + sql("ALTER TABLE scriptRevisionFile ADD CONSTRAINT reject_file CHECK (contents NOT LIKE '%reject-file%')"); + assertThrows(ScriptingException.class, () -> service.saveDraft(everything(), project, "test", List.of( + new SourceFile("first.ts", "export const first = 1;"), new SourceFile("second.ts", "// reject-file")))); + assertEquals(0, count("scriptRevision")); + assertEquals(0, count("scriptRevisionFile")); + assertNull(QContext.getQBackendTransaction()); + } + + + + /*************************************************************************** + ** An artifact failure cannot leave a successful orphan build behind. + ***************************************************************************/ + @Test + void buildAndArtifactCommitTogether() throws Exception + { + Integer revision = draft(newProject(), "build"); + sql("ALTER TABLE scripting_artifact ADD CONSTRAINT reject_bundle CHECK (bundle NOT LIKE '%placeholder%')"); + assertThrows(ScriptingException.class, () -> build(revision)); + assertEquals(0, count(ScriptingTables.BUILD)); + assertEquals(0, count(ScriptingTables.ARTIFACT)); + } + + + /*************************************************************************** + ** A failed retained import receipt rolls back source revision and all files. + ***************************************************************************/ + @Test + void sourceImportAndReceiptCommitTogether() throws Exception + { + Integer project = newProject(); + var runtime = ((ScriptingQBitConfig) instance.getQBits().values().iterator().next().getConfig()).getRuntime(); + runtime.withImporter((actor, id, ref) -> com.kingsrook.qbits.scripting.source.SourceSnapshot.of(ref.origin(), source("imported"))); + var ref = new com.kingsrook.qbits.scripting.source.GitCommitRef("acme/scripts", "a".repeat(40), null); + sql("ALTER TABLE scripting_import ADD CONSTRAINT reject_import CHECK (repository <> 'acme/scripts')"); + assertThrows(ScriptingException.class, () -> service.importDraft(everything(), project, null, ref)); + assertEquals(0, count("scriptRevision")); + assertEquals(0, count("scriptRevisionFile")); + assertEquals(0, count(ScriptingTables.IMPORT)); + sql("ALTER TABLE scripting_import DROP CONSTRAINT reject_import"); + service.importDraft(everything(), project, null, ref); + assertEquals(1, count("scriptRevision")); + assertEquals(1, count(ScriptingTables.IMPORT)); + } + + + + /*************************************************************************** + ** A failed publication is absent; a successful publication is committed and + ** visible from a separate connection before the service returns. + ***************************************************************************/ + @Test + void publicationOwnsItsCommitBoundary() throws Exception + { + var built = build(draft(newProject(), "publish")); + service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null); + sql("ALTER TABLE scripting_publication ADD CONSTRAINT reject_note CHECK (note <> 'reject')"); + assertThrows(ScriptingException.class, () -> service.publish(everything(), built.buildId(), "reject")); + assertEquals(0, count(ScriptingTables.PUBLICATION)); + sql("ALTER TABLE script ADD CONSTRAINT reject_activation CHECK (currentScriptRevisionId IS NULL)"); + assertThrows(ScriptingException.class, () -> service.publish(everything(), built.buildId(), "accepted")); + assertEquals(0, count(ScriptingTables.PUBLICATION)); + sql("ALTER TABLE script DROP CONSTRAINT reject_activation"); + service.publish(everything(), built.buildId(), "accepted"); + assertEquals(1, count(ScriptingTables.PUBLICATION)); + } + + + + /*************************************************************************** + ** Rollback of the writer's transaction does not erase script attempts or + ** outcomes; the service never commits the writer's transaction. + ***************************************************************************/ + @Test + void rejectedHookHistorySurvivesCallerRollback() throws Exception + { + bindHook(publishNew(newProject(), "hook"), "status"); + int before = count(ScriptingTables.EXECUTION); + worker.hookJson = "{\"action\":\"REJECT\",\"message\":\"denied\"}"; + InsertInput input = new InsertInput().withTableName(HOOK_TABLE); + try(QBackendTransaction transaction = QBackendTransaction.openFor(input)) + { + QContext.setTransaction(transaction); + var output = new InsertAction().execute(input.withTransaction(transaction).withRecords(List.of(new QRecord().withValue("status", "NEW")))); + assertFalse(output.getRecords().get(0).getErrors().isEmpty()); + assertSame(transaction, QContext.getQBackendTransaction()); + transaction.rollback(); + } + finally + { + QContext.clearTransaction(); + } + assertEquals(0, count(HOOK_TABLE)); + assertEquals(before + 2, count(ScriptingTables.EXECUTION)); + } + + + + /*************************************************************************** + ** If the independently committed request cannot be recorded, the worker is + ** never called and there is no trusted output. + ***************************************************************************/ + @Test + void unavailableHistoryPreventsWorkerCall() throws Exception + { + var built = build(draft(newProject(), "history")); + sql("ALTER TABLE scripting_execution ADD CONSTRAINT reject_start CHECK (status <> 'STARTED')"); + assertThrows(ScriptingException.class, () -> service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null)); + assertEquals(0, worker.executeCalls); + assertEquals(0, count(ScriptingTables.EXECUTION)); + } + + /*************************************************************************** + ** Post observation executes before caller commit. Native insert/update/delete + ** roll back together while independent script audit remains truthful. + ***************************************************************************/ + @Test + void nativeActionsAndPostObservationsRespectCallerRollback() throws Exception + { + Integer publication = publishNew(newProject(), "native-transaction"); + ScriptHooks.install(instance.getTable(HOOK_TABLE), synchronousEvents()); + for(HookEvent event : synchronousEvents()) + { + service.bind(everything(), new BindingSpec("native", HOOK_TABLE, event, publication, declared(), event.permitsPatch() ? Set.of("status") : Set.of())); + } + int before = count(ScriptingTables.EXECUTION); + worker.onExecute = channel -> + { + String event = new JSONObject(worker.lastExecuteRequest).getString("event"); + worker.hookJson = event.equals("PRE_INSERT") || event.equals("PRE_UPDATE") + ? new JSONObject().put("action", "PATCH").put("patch", new JSONObject().put("status", event)).toString() + : new JSONObject().put("action", "ALLOW").toString(); + }; + InsertInput input = new InsertInput().withTableName(HOOK_TABLE); + try(QBackendTransaction transaction = QBackendTransaction.openFor(input)) + { + QContext.setTransaction(transaction); + QRecord row = new InsertAction().execute(input.withTransaction(transaction).withRecords(List.of(new QRecord().withValue("status", "NEW")))).getRecords().getFirst(); + assertTrue(row.getErrors().isEmpty()); + assertEquals("POST_INSERT", new JSONObject(worker.lastExecuteRequest).getString("event")); + var updated = new UpdateAction().execute(new UpdateInput().withTableName(HOOK_TABLE).withTransaction(transaction) + .withRecords(List.of(new QRecord().withValue("id", row.getValueInteger("id")).withValue("status", "CHANGE")))); + assertTrue(updated.getRecords().getFirst().getErrors().isEmpty()); + assertFalse(new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").has("prior")); + assertEquals("PRE_UPDATE", new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("proposed").getString("status")); + assertEquals("POST_UPDATE", new JSONObject(worker.lastExecuteRequest).getString("event")); + assertEquals(1, new DeleteAction().execute(new DeleteInput().withTableName(HOOK_TABLE).withTransaction(transaction) + .withPrimaryKeys(List.of(row.getValueInteger("id")))).getDeletedRecordCount()); + assertEquals("POST_DELETE", new JSONObject(worker.lastExecuteRequest).getString("event")); + assertSame(transaction, QContext.getQBackendTransaction()); + transaction.rollback(); + } + finally + { + QContext.clearTransaction(); + } + assertEquals(0, count(HOOK_TABLE)); + assertEquals(before + 12, count(ScriptingTables.EXECUTION)); + } + + /*************************************************************************** + ** A successful observed insert is still rolled back by its caller; its + ** audit reports execution success, not a committed business record. + ***************************************************************************/ + @Test + void successfulObservedInsertCanStillRollBack() throws Exception + { + Integer publication = publishNew(newProject(), "observed-rollback"); + ScriptHooks.install(instance.getTable(HOOK_TABLE), HookEvent.POST_INSERT); + service.bind(everything(), new BindingSpec("post", HOOK_TABLE, HookEvent.POST_INSERT, publication, declared(), Set.of())); + int before = count(ScriptingTables.EXECUTION); + InsertInput input = new InsertInput().withTableName(HOOK_TABLE); + try(QBackendTransaction transaction = QBackendTransaction.openFor(input)) + { + QContext.setTransaction(transaction); + QRecord row = new InsertAction().execute(input.withTransaction(transaction).withRecords(List.of(new QRecord().withValue("status", "NEW")))).getRecords().getFirst(); + assertTrue(row.getErrors().isEmpty()); + assertTrue(row.getWarnings().isEmpty()); + assertEquals("POST_INSERT", new JSONObject(worker.lastExecuteRequest).getString("event")); + transaction.rollback(); + } + finally + { + QContext.clearTransaction(); + } + assertEquals(0, count(HOOK_TABLE)); + assertEquals(before + 2, count(ScriptingTables.EXECUTION)); + } + + /*************************************************************************** + ** Context admission refusals commit independently, contain no record values, + ** and do not pretend the worker started or the caller transaction committed. + ***************************************************************************/ + @Test + void contextRefusalSurvivesBusinessRollbackAndAuditFailureIsVisible() throws Exception + { + Integer publication = publishNew(newProject(), "context-refusal"); + ScriptHooks.install(instance.getTable(HOOK_TABLE), HookEvent.POST_INSERT); + service.bind(everything(), new BindingSpec("post", HOOK_TABLE, HookEvent.POST_INSERT, publication, declared(), Set.of())); + instance.getTable(HOOK_TABLE).setPermissionRules(new com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules() + .withLevel(com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel.NOT_PROTECTED) + .withCustomPermissionChecker(new QCodeReference(NativeHookReadSecurityTest.DenyReads.class))); + int before = count(ScriptingTables.EXECUTION); + int workerBefore = worker.executeCalls; + InsertInput input = new InsertInput(HOOK_TABLE); + try(QBackendTransaction transaction = QBackendTransaction.openFor(input)) + { + QContext.setTransaction(transaction); + QRecord row = new InsertAction().execute(input.withTransaction(transaction).withRecords(List.of(new QRecord().withValue("status", "PRIVATE")))).getRecords().getFirst(); + assertTrue(row.getErrors().isEmpty()); + assertFalse(row.getWarnings().isEmpty()); + assertSame(transaction, QContext.getQBackendTransaction()); + transaction.rollback(); + } + finally + { + QContext.clearTransaction(); + } + assertEquals(0, count(HOOK_TABLE)); + assertEquals(before + 1, count(ScriptingTables.EXECUTION)); + assertEquals(workerBefore, worker.executeCalls); + QRecord receipt = rows(ScriptingTables.EXECUTION).getLast(); + assertEquals("HOOK_REFUSAL", receipt.getValueString("kind")); + assertEquals("DENIED", receipt.getValueString("status")); + assertEquals("CONTEXT_REFUSED", receipt.getValueString("message")); + assertEquals("alice@example.com", receipt.getValueString("actor_id")); + assertFalse(receipt.getValues().toString().contains("PRIVATE")); + sql("ALTER TABLE scripting_execution ADD CONSTRAINT reject_refusal CHECK (kind <> 'HOOK_REFUSAL' OR id <= " + receipt.getValueInteger("id") + ")"); + QRecord result = new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", "PRIVATE")))).getRecords().getFirst(); + assertTrue(result.getErrors().isEmpty()); + assertTrue(result.getWarnings().toString().contains("refusal audit could not be stored")); + assertEquals(1, count(HOOK_TABLE)); + assertEquals(before + 1, count(ScriptingTables.EXECUTION)); + assertEquals(workerBefore, worker.executeCalls); + } + + + /*************************************************************************** + ** Binding state cannot commit when its same-transaction activity receipt fails. + ***************************************************************************/ + @Test + void bindingAuditFailureRollsBackCreateAndStateChange() throws Exception + { + Integer project = newProject(); + Integer publication = publishNew(project, "binding-audit"); + BindingSpec spec = new BindingSpec("audited", HOOK_TABLE, HookEvent.PRE_INSERT, publication, declared(), Set.of("status")); + int before = count(ScriptingTables.EXECUTION); + sql("ALTER TABLE scripting_execution ADD CONSTRAINT reject_binding CHECK (kind <> 'BINDING_CHANGE')"); + assertThrows(ScriptingException.class, () -> service.hooks().bind(everything(), project, spec)); + assertEquals(0, count(ScriptingTables.BINDING)); + assertEquals(before, count(ScriptingTables.EXECUTION)); + sql("ALTER TABLE scripting_execution DROP CONSTRAINT reject_binding"); + Integer id = service.hooks().bind(everything(), project, spec); + assertEquals(id, service.hooks().view(everything(), project).bindings().getFirst().id()); + sql("ALTER TABLE scripting_execution ADD CONSTRAINT reject_disable CHECK (status <> 'DISABLED')"); + assertThrows(ScriptingException.class, () -> service.hooks().setEnabled(everything(), project, id, false)); + assertTrue(service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).isPresent()); + assertEquals(before + 1, count(ScriptingTables.EXECUTION)); + } + +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/NativeHookReadSecurityTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/NativeHookReadSecurityTest.java new file mode 100644 index 0000000..d3734a4 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/NativeHookReadSecurityTest.java @@ -0,0 +1,417 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHookCustomizer; +import com.kingsrook.qbits.scripting.hooks.ScriptHooks; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.InputSource; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Genuine native READ and bounded delete snapshot regressions, with no code execution. + *******************************************************************************/ +class NativeHookReadSecurityTest extends ScriptingTestBase +{ + private static final String SNAPSHOTS = "com.kingsrook.qbits.scripting.hooks.DeleteHookSnapshots"; + private Integer publication; + + /*************************************************************************** + ** Native actions with a pinned protocol fixture. + ***************************************************************************/ + @BeforeEach + void prepare() throws Exception + { + publication = publishNew(newProject(), "read-security"); + ScriptHooks.install(instance.getTable(HOOK_TABLE), synchronousEvents()); + NestedRead.nestedWasDenied = false; + } + + /*************************************************************************** + ** Bind one read-only hook. + ***************************************************************************/ + private void bind(HookEvent event) throws Exception + { + service.bind(everything(), new BindingSpec("read-security", HOOK_TABLE, event, publication, declared(), Set.of())); + } + + /*************************************************************************** + ** Persist a fixture row through native insertion. + ***************************************************************************/ + private QRecord insert(String value) throws QException + { + return (new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", value)))).getRecords().getFirst()); + } + + /*************************************************************************** + ** Real custom permission dispatch sees a read input, never the original write. + ***************************************************************************/ + @Test + void customReadCheckerDeniesStoredContextBeforeWorker() throws Exception + { + QRecord stored = insert("PRIVATE"); + bind(HookEvent.PRE_UPDATE); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED) + .withCustomPermissionChecker(new QCodeReference(DenyReads.class))); + int before = worker.executeCalls; + QRecord result = new UpdateAction().execute(new UpdateInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("id", stored.getValue("id")).withValue("status", "NEW")))).getRecords().getFirst(); + assertFalse(result.getErrors().isEmpty()); + assertEquals(before, worker.executeCalls); + assertFalse(result.getErrors().toString().contains("PRIVATE")); + QRecord receipt = rows(ScriptingTables.EXECUTION).getLast(); + assertEquals("HOOK_REFUSAL", receipt.getValueString("kind")); + assertEquals("CONTEXT_REFUSED", receipt.getValueString("message")); + assertEquals("alice@example.com", receipt.getValueString("actor_id")); + assertFalse(receipt.getValues().toString().contains("PRIVATE")); + int count = rows(ScriptingTables.EXECUTION).size(); + new UpdateAction().execute(new UpdateInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("id", stored.getValue("id")).withValue("status", "RETRY")))); + assertEquals(count + 1, rows(ScriptingTables.EXECUTION).size()); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** A write-only null lock permits insert/update but forbids every post snapshot. + ***************************************************************************/ + @Test + void writeOnlyRecordCannotReachPostWorker() throws Exception + { + bind(HookEvent.POST_INSERT); + bind(HookEvent.POST_UPDATE); + instance.addSecurityKeyType(new QSecurityKeyType().withName("owner")); + instance.getTable(HOOK_TABLE).withRecordSecurityLock(new RecordSecurityLock().withFieldName("created_by") + .withSecurityKeyType("owner").withNullValueBehavior(RecordSecurityLock.NullValueBehavior.ALLOW_WRITE_ONLY)); + int before = worker.executeCalls; + QRecord stored = insert("PRIVATE"); + assertTrue(stored.getErrors().isEmpty()); + assertFalse(stored.getWarnings().isEmpty()); + assertNull(new GetAction().execute(new GetInput(HOOK_TABLE).withPrimaryKey(stored.getValue("id"))).getRecord()); + QContext.getQSession().setSecurityKeyValues(java.util.Map.of("owner", List.of("alice"))); + QRecord readable = new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", "READABLE").withValue("created_by", "alice")))).getRecords().getFirst(); + before = worker.executeCalls; + QRecord updated = new UpdateAction().execute(new UpdateInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("id", readable.getValue("id")).withValue("created_by", null).withValue("status", "PRIVATE-UPDATED")))).getRecords().getFirst(); + assertTrue(updated.getErrors().isEmpty()); + assertFalse(updated.getWarnings().isEmpty()); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** Read customizers cannot recursively run hooks while building hook context. + ***************************************************************************/ + @Test + void nativeReadCallbackCannotRecursivelyExecuteScript() throws Exception + { + QRecord stored = insert("OLD"); + bind(HookEvent.PRE_INSERT); + bind(HookEvent.PRE_UPDATE); + instance.getTable(HOOK_TABLE).withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(NestedRead.class)); + int before = worker.executeCalls; + QRecord result = new ScriptHookCustomizer().preUpdate(new UpdateInput(HOOK_TABLE), + List.of(new QRecord().withValue("id", stored.getValue("id")).withValue("status", "NEW")), false, Optional.empty()).getFirst(); + assertTrue(result.getErrors().isEmpty()); + assertTrue(NestedRead.nestedWasDenied); + assertEquals(before + 1, worker.executeCalls); + } + + /*************************************************************************** + ** Installing POST_DELETE alone installs its required authorized capture phase. + ***************************************************************************/ + @Test + void postDeleteUsesAuthorizedSnapshotAndConsumesReceipt() throws Exception + { + instance.getTable(HOOK_TABLE).getCustomizers().clear(); + ScriptHooks.install(instance.getTable(HOOK_TABLE), HookEvent.POST_DELETE); + bind(HookEvent.POST_DELETE); + QRecord stored = insert("DELETED"); + DeleteInput input = new DeleteInput(HOOK_TABLE).withPrimaryKeys(List.of(stored.getValue("id"))); + int before = worker.executeCalls; + var output = new DeleteAction().execute(input); + assertEquals(1, output.getDeletedRecordCount()); + assertTrue(output.getRecordsWithWarnings().isEmpty()); + assertEquals(before + 1, worker.executeCalls); + assertEquals("DELETED", new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("proposed").getString("status")); + assertNull(QContext.getObject(SNAPSHOTS)); + new ScriptHookCustomizer().postDelete(input, List.of(new QRecord(stored))); + assertEquals(before + 1, worker.executeCalls); + } + + /*************************************************************************** + ** Preview, a different request, or a changed principal cannot reuse a receipt. + ***************************************************************************/ + @Test + void deleteCaptureIsExactInputAndIdentityBound() throws Exception + { + bind(HookEvent.POST_DELETE); + QRecord stored = insert("PRIVATE"); + ScriptHookCustomizer customizer = new ScriptHookCustomizer(); + DeleteInput input = new DeleteInput(HOOK_TABLE).withPrimaryKeys(List.of(stored.getValue("id"))); + int before = worker.executeCalls; + customizer.preDelete(input, List.of(new QRecord(stored)), false); + assertNotNull(QContext.getObject(SNAPSHOTS)); + customizer.postDelete(new DeleteInput(HOOK_TABLE), List.of(new QRecord(stored))); + assertEquals(before, worker.executeCalls); + customizer.preDelete(input, List.of(new QRecord(stored)), true); + assertNull(QContext.getObject(SNAPSHOTS)); + customizer.postDelete(input, List.of(new QRecord(stored))); + assertEquals(before, worker.executeCalls); + customizer.preDelete(input, List.of(new QRecord(stored)), false); + var original = QContext.getQSession(); + QContext.setQSession(sessionWith(ScriptingPermissions.EXECUTE)); + customizer.postDelete(input, List.of(new QRecord(stored))); + assertEquals(before, worker.executeCalls); + assertNull(QContext.getObject(SNAPSHOTS)); + QContext.setQSession(original); + } + + /*************************************************************************** + ** Abandoned actions have a strict aggregate count and byte bound; teardown clears. + ***************************************************************************/ + @Test + void pendingDeleteCapturesAreBoundedAndContextScoped() throws Exception + { + bind(HookEvent.POST_DELETE); + QRecord stored = insert("PRIVATE"); + ScriptHookCustomizer customizer = new ScriptHookCustomizer(); + List pending = new ArrayList<>(); + for(int i = 0; i < 8; i++) + { + DeleteInput input = new DeleteInput(HOOK_TABLE); + pending.add(input); + assertTrue(customizer.preDelete(input, List.of(new QRecord(stored)), false).getFirst().getErrors().isEmpty()); + } + DeleteInput overflow = new DeleteInput(HOOK_TABLE); + assertTrue(customizer.preDelete(overflow, List.of(new QRecord(stored)), false).getFirst().getErrors().isEmpty()); + assertFalse(customizer.postDelete(overflow, List.of(new QRecord(stored))).getFirst().getWarnings().isEmpty()); + for(DeleteInput input : pending) + { + customizer.postDelete(input, List.of()); + } + assertNull(QContext.getObject(SNAPSHOTS)); + QRecord large = insert("x".repeat(140000)); + assertTrue(customizer.preDelete(new DeleteInput(HOOK_TABLE), List.of(new QRecord(large)), false).getFirst().getErrors().isEmpty()); + DeleteInput oversized = new DeleteInput(HOOK_TABLE); + assertTrue(customizer.preDelete(oversized, List.of(new QRecord(large)), false).getFirst().getErrors().isEmpty()); + assertFalse(customizer.postDelete(oversized, List.of(new QRecord(large))).getFirst().getWarnings().isEmpty()); + var session = QContext.getQSession(); + QContext.clear(); + QContext.init(instance, session); + assertNull(QContext.getObject(SNAPSHOTS)); + } + + /*************************************************************************** + ** A post-only read refusal warns after deletion, while a real pre hook vetoes. + ***************************************************************************/ + @Test + void postOnlyCaptureRefusalDoesNotVetoDeletion() throws Exception + { + QRecord first = insert("PRIVATE"); + QRecord second = insert("PRIVATE"); + bind(HookEvent.POST_DELETE); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED) + .withCustomPermissionChecker(new QCodeReference(DenyReads.class))); + int before = worker.executeCalls; + var deleted = new DeleteAction().execute(new DeleteInput(HOOK_TABLE).withPrimaryKeys(List.of(first.getValue("id")))); + assertEquals(1, deleted.getDeletedRecordCount()); + assertFalse(deleted.getRecordsWithWarnings().isEmpty()); + assertEquals(before, worker.executeCalls); + assertEquals("HOOK_REFUSAL", rows(ScriptingTables.EXECUTION).getLast().getValueString("kind")); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)); + bind(HookEvent.PRE_DELETE); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED) + .withCustomPermissionChecker(new QCodeReference(DenyReads.class))); + assertEquals(0, new DeleteAction().execute(new DeleteInput(HOOK_TABLE).withPrimaryKeys(List.of(second.getValue("id")))).getDeletedRecordCount()); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** Only successful native deletion IDs are observed; all snapshots are consumed. + ***************************************************************************/ + @Test + void partialDeleteObservesOnlySuccessfulRows() throws Exception + { + QRecord held = insert("HELD"); + QRecord allowed = insert("ALLOWED"); + bind(HookEvent.PRE_DELETE); + bind(HookEvent.POST_DELETE); + worker.onExecute = channel -> + { + JSONObject request = new JSONObject(worker.lastExecuteRequest); + worker.hookJson = "PRE_DELETE".equals(request.getString("event")) + && "HELD".equals(request.getJSONObject("payload").getJSONObject("proposed").getString("status")) + ? "{\"action\":\"REJECT\",\"message\":\"held\"}" : "{\"action\":\"ALLOW\"}"; + }; + int before = worker.executeCalls; + var output = new DeleteAction().execute(new DeleteInput(HOOK_TABLE).withPrimaryKeys(List.of(held.getValue("id"), allowed.getValue("id")))); + assertEquals(1, output.getDeletedRecordCount()); + assertEquals(before + 3, worker.executeCalls); + assertEquals("ALLOWED", new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("proposed").getString("status")); + assertNull(QContext.getObject(SNAPSHOTS)); + } + + /*************************************************************************** + ** The current credential's custom source is preserved on the enforced read. + ***************************************************************************/ + @Test + void customInputOriginIsPreserved() throws Exception + { + QRecord stored = insert("OLD"); + bind(HookEvent.PRE_UPDATE); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED) + .withCustomPermissionChecker(new QCodeReference(RequireOrigin.class))); + QRecord result = new UpdateAction().execute(new UpdateInput(HOOK_TABLE).withInputSource(Origin.TRUSTED_API) + .withRecords(List.of(new QRecord().withValue("id", stored.getValue("id")).withValue("status", "NEW")))).getRecords().getFirst(); + assertTrue(result.getErrors().isEmpty()); + } + + /*************************************************************************** + ** Ambient transaction changes invalidate a receipt even with a null input tx. + ***************************************************************************/ + @Test + void deleteCaptureBindsEffectiveAmbientTransaction() throws Exception + { + bind(HookEvent.POST_DELETE); + QRecord stored = insert("PRIVATE"); + DeleteInput input = new DeleteInput(HOOK_TABLE); + ScriptHookCustomizer customizer = new ScriptHookCustomizer(); + int before = worker.executeCalls; + QContext.setTransaction(new QBackendTransaction()); + customizer.preDelete(input, List.of(new QRecord(stored)), false); + assertNotNull(QContext.getObject(SNAPSHOTS)); + QContext.setTransaction(new QBackendTransaction()); + assertFalse(customizer.postDelete(input, List.of(new QRecord(stored))).getFirst().getWarnings().isEmpty()); + assertEquals(before, worker.executeCalls); + assertNull(QContext.getObject(SNAPSHOTS)); + QContext.clearTransaction(); + } + + /*************************************************************************** + ** Host-defined source identities are supported by native inputs. + ***************************************************************************/ + private enum Origin implements InputSource + { + TRUSTED_API; + + /*********************************************************************** + ** This source uses ordinary required-field checks. + ***********************************************************************/ + @Override + public boolean shouldValidateRequiredFields() + { + return (true); + } + } + + /*************************************************************************** + ** A real host read policy requiring the originating trusted API marker. + ***************************************************************************/ + public static class RequireOrigin implements CustomPermissionChecker + { + /*********************************************************************** + ** Reads must retain the host's extensible input-source value. + ***********************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException + { + if(input instanceof GetInput get && get.getInputSource() != Origin.TRUSTED_API) + { + throw new QPermissionDeniedException("Wrong originating channel"); + } + } + } + + /*************************************************************************** + ** Host policy intentionally allows mutation inputs while forbidding reads. + ***************************************************************************/ + public static class DenyReads implements CustomPermissionChecker + { + /*********************************************************************** + ** Input class is the host's real dispatch contract. + ***********************************************************************/ + @Override + public void checkPermissionsThrowing(AbstractActionInput input, MetaDataWithPermissionRules metadata) throws QPermissionDeniedException + { + if(input instanceof GetInput || input instanceof QueryInput) + { + throw new QPermissionDeniedException("Reading is forbidden"); + } + } + } + + /*************************************************************************** + ** A trusted read customizer attempts a nested insertion into the hooked table. + ***************************************************************************/ + public static class NestedRead implements TableCustomizerInterface + { + private static boolean nestedWasDenied; + + /*********************************************************************** + ** The nested native operation must fail before contacting the worker. + ***********************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface input, List records) throws QException + { + if(input.getShouldGenerateDisplayValues()) + { + return (records); + } + nestedWasDenied = !new InsertAction().execute(new InsertInput(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", "NESTED")))).getRecords().getFirst().getErrors().isEmpty(); + return (records); + } + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/NativeScriptHookTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/NativeScriptHookTest.java new file mode 100644 index 0000000..d3dcedc --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/NativeScriptHookTest.java @@ -0,0 +1,409 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHooks; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer.WhenToRun; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface.PreUpdatePhase; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Native action tests with a protocol-only worker. No script source executes. + *******************************************************************************/ +class NativeScriptHookTest extends ScriptingTestBase +{ + private Integer publication; + + /*************************************************************************** + ** Every role is explicitly opted in; installation is idempotent. + ***************************************************************************/ + @BeforeEach + void prepare() throws Exception + { + publication = publishNew(newProject(), "native-hooks"); + ScriptHooks.install(instance.getTable(HOOK_TABLE), synchronousEvents()); + ScriptHooks.install(instance.getTable(HOOK_TABLE), synchronousEvents()); + Host.calls = 0; + Host.phase = WhenToRun.BEFORE_ALL_VALIDATIONS; + Host.updatePhase = PreUpdatePhase.BEFORE_FIELD_VALIDATIONS; + } + + /*************************************************************************** + ** Bind a pinned fixture publication for one event. + ***************************************************************************/ + private void bind(HookEvent event, String... fields) throws Exception + { + service.bind(everything(), new BindingSpec("native", HOOK_TABLE, event, publication, declared(), Set.of(fields))); + } + + /*************************************************************************** + ** Run a native insertion. + ***************************************************************************/ + private QRecord insert(String status) throws Exception + { + return (new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE) + .withRecords(List.of(new QRecord().withValue("status", status)))).getRecords().getFirst()); + } + + /*************************************************************************** + ** Run a native update. + ***************************************************************************/ + private QRecord update(Integer id) throws Exception + { + return (new UpdateAction().execute(new UpdateInput().withTableName(HOOK_TABLE) + .withRecords(List.of(new QRecord().withValue("id", id).withValue("status", "EDIT")))).getRecords().getFirst()); + } + + /*************************************************************************** + ** Native update patch applies, delete veto refuses, then allow deletes. + ***************************************************************************/ + @Test + void updateAndDeleteUsePinnedNativeHooks() throws Exception + { + QRecord row = insert("NEW"); + bind(HookEvent.PRE_UPDATE, "status"); + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\"APPROVED\"}}"; + assertTrue(update(row.getValueInteger("id")).getErrors().isEmpty()); + assertEquals("NEW", new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("prior").getString("status")); + assertEquals("APPROVED", rows(HOOK_TABLE).getFirst().getValueString("status")); + bind(HookEvent.PRE_DELETE); + worker.hookJson = "{\"action\":\"REJECT\",\"message\":\"hold\"}"; + assertEquals(0, new DeleteAction().execute(new DeleteInput().withTableName(HOOK_TABLE).withPrimaryKeys(List.of(row.getValueInteger("id")))).getDeletedRecordCount()); + worker.hookJson = "{\"action\":\"ALLOW\"}"; + assertEquals(1, new DeleteAction().execute(new DeleteInput().withTableName(HOOK_TABLE).withPrimaryKeys(List.of(row.getValueInteger("id")))).getDeletedRecordCount()); + } + + /*************************************************************************** + ** Post callbacks cannot reject or mutate completed backend actions. + ***************************************************************************/ + @Test + void postPhasesAreObservationsWithVisibleDurableFailure() throws Exception + { + bind(HookEvent.POST_INSERT); + bind(HookEvent.POST_UPDATE); + bind(HookEvent.POST_DELETE); + worker.hookJson = "{\"action\":\"REJECT\",\"message\":\"too late\"}"; + QRecord row = insert("NEW"); + assertTrue(row.getErrors().isEmpty()); + assertFalse(row.getWarnings().isEmpty()); + assertEquals(1, rows(HOOK_TABLE).size()); + assertEquals("INVALID", rows(ScriptingTables.EXECUTION).getLast().getValueString("status")); + assertFalse(update(row.getValueInteger("id")).getWarnings().isEmpty()); + var deleted = new DeleteAction().execute(new DeleteInput().withTableName(HOOK_TABLE).withPrimaryKeys(List.of(row.getValueInteger("id")))); + assertEquals(1, deleted.getDeletedRecordCount()); + assertFalse(deleted.getRecordsWithWarnings().isEmpty()); + } + + /*************************************************************************** + ** Preserve a host phase and call it only once before the script sees values. + ***************************************************************************/ + @Test + void compositionPreservesOrderAndRefusesLatePatchBinding() throws Exception + { + var table = instance.getTable(HOOK_TABLE); + table.getCustomizers().put(TableCustomizers.PRE_INSERT_RECORD.getRole(), new QCodeReference(Host.class)); + ScriptHooks.install(table, HookEvent.PRE_INSERT); + ScriptHooks.install(table, HookEvent.PRE_INSERT); + bind(HookEvent.PRE_INSERT, "status"); + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\"SCRIPT\"}}"; + assertEquals("SCRIPT", insert("NEW").getValueString("status")); + assertEquals(1, Host.calls); + assertEquals("HOST", new JSONObject(worker.lastExecuteRequest).getJSONObject("payload").getJSONObject("proposed").getString("status")); + Host.phase = WhenToRun.AFTER_ALL_VALIDATIONS; + assertFalse(insert("NEW").getErrors().isEmpty()); + service.setBindingEnabled(everything(), service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).orElseThrow().id(), false); + assertThrows(ScriptingException.class, () -> bind(HookEvent.PRE_INSERT, "status")); + bind(HookEvent.PRE_INSERT); + worker.hookJson = "{\"action\":\"ALLOW\"}"; + assertTrue(insert("NEW").getErrors().isEmpty()); + } + + /*************************************************************************** + ** Revocation during worker execution prevents applying its proposed change. + ***************************************************************************/ + @Test + void currentWritePermissionIsRecheckedAfterWorker() throws Exception + { + bind(HookEvent.PRE_INSERT, "status"); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + QContext.getQSession().getPermissions().add(HOOK_TABLE + ".write"); + worker.onExecute = channel -> QContext.getQSession().getPermissions().remove(HOOK_TABLE + ".write"); + assertFalse(insert("NEW").getErrors().isEmpty()); + worker.onExecute = null; + QContext.getQSession().getPermissions().add(HOOK_TABLE + ".read"); + assertTrue(rows(HOOK_TABLE).isEmpty()); + } + + /*************************************************************************** + ** Even trusted worker callbacks cannot recursively enter a native script hook. + ***************************************************************************/ + @Test + void nestedInsertAndUpdateAreRefusedAndGuardUnwinds() throws Exception + { + QRecord row = insert("NEW"); + bind(HookEvent.PRE_INSERT); + bind(HookEvent.PRE_UPDATE); + AtomicBoolean denied = new AtomicBoolean(); + worker.onExecute = channel -> + { + try + { + denied.set(!insert("NESTED").getErrors().isEmpty() && !update(row.getValueInteger("id")).getErrors().isEmpty()); + } + catch(Exception e) + { + throw new IllegalStateException(e); + } + }; + assertTrue(insert("OUTER").getErrors().isEmpty()); + assertTrue(denied.get()); + worker.onExecute = null; + assertTrue(update(row.getValueInteger("id")).getErrors().isEmpty()); + assertEquals(2, rows(HOOK_TABLE).size()); + } + + /*************************************************************************** + ** Delete/post bindings cannot claim mutable patch capabilities. + ***************************************************************************/ + @Test + void nonMutatingEventsRefusePatchFields() throws Exception + { + assertThrows(ScriptingException.class, () -> bind(HookEvent.PRE_DELETE, "status")); + assertThrows(ScriptingException.class, () -> bind(HookEvent.POST_INSERT, "status")); + } + + /*************************************************************************** + ** Controlled host customization makes native ordering directly observable. + ***************************************************************************/ + public static class Host implements TableCustomizerInterface + { + static int calls; + static WhenToRun phase; + static PreUpdatePhase updatePhase; + + /*********************************************************************** + ** Preserve the host's configured native phase. + ***********************************************************************/ + @Override + public WhenToRun whenToRunPreInsert(InsertInput input, boolean preview) + { + return (phase); + } + + /*********************************************************************** + ** Preserve host update scheduling when composing. + ***********************************************************************/ + @Override + public PreUpdatePhase whenToRunPreUpdate(UpdateInput input, boolean preview) + { + return (updatePhase); + } + + /*********************************************************************** + ** Set a value the following script must see. + ***********************************************************************/ + @Override + public List preInsert(InsertInput input, List records, boolean preview) + { + calls++; + records.forEach(record -> record.setValue("status", "HOST")); + return (records); + } + + /*********************************************************************** + ** Record host callback order for this native phase. + ***********************************************************************/ + @Override + public List preUpdate(UpdateInput input, List records, boolean preview, Optional> prior) + { + calls++; + return (records); + } + + /*********************************************************************** + ** Record host callback order for this native phase. + ***********************************************************************/ + @Override + public List preDelete(DeleteInput input, List records, boolean preview) + { + calls++; + return (records); + } + + /*********************************************************************** + ** Record host callback order for this native phase. + ***********************************************************************/ + @Override + public List postInsert(InsertInput input, List records) + { + calls++; + return (records); + } + + /*********************************************************************** + ** Record host callback order for this native phase. + ***********************************************************************/ + @Override + public List postUpdate(UpdateInput input, List records, Optional> prior) + { + calls++; + return (records); + } + + /*********************************************************************** + ** Record host callback order for this native phase. + ***********************************************************************/ + @Override + public List postDelete(DeleteInput input, List records) + { + calls++; + return (records); + } + + } + + /*************************************************************************** + ** Native patches respect required/length constraints and stored unique keys. + ***************************************************************************/ + @Test + void invalidPatchedFieldsNeverReachStorage() throws Exception + { + var table = instance.getTable(HOOK_TABLE); + table.getFields().get("status").withIsRequired(true).withMaxLength(8).withBehavior(com.kingsrook.qqq.backend.core.model.metadata.fields.ValueTooLongBehavior.ERROR); + table.withUniqueKey(new com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey("status")); + insert("EXISTS"); + bind(HookEvent.PRE_INSERT, "status"); + for(String invalid : List.of("", "way-too-long", "EXISTS")) + { + worker.hookJson = new JSONObject().put("action", "PATCH").put("patch", new JSONObject().put("status", invalid)).toString(); + assertFalse(insert("OK").getErrors().isEmpty(), invalid); + } + assertEquals(1, rows(HOOK_TABLE).size()); + bind(HookEvent.PRE_UPDATE, "status"); + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\" \"}}"; + assertFalse(update(rows(HOOK_TABLE).getFirst().getValueInteger("id")).getErrors().isEmpty()); + assertEquals("EXISTS", rows(HOOK_TABLE).getFirst().getValueString("status")); + } + + /*************************************************************************** + ** Oversized native action batches are refused before contacting the worker. + ***************************************************************************/ + @Test + void nativeBatchBoundRejectsWholeInput() throws Exception + { + bind(HookEvent.PRE_INSERT); + int before = worker.executeCalls; + List records = java.util.stream.IntStream.range(0, service.getMaxRecordsPerHook() + 1) + .mapToObj(i -> new QRecord().withValue("status", "NEW")).toList(); + var output = new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE).withRecords(records)); + assertTrue(output.getRecords().stream().allMatch(record -> !record.getErrors().isEmpty())); + assertTrue(rows(HOOK_TABLE).isEmpty()); + assertEquals(before, worker.executeCalls); + } + + /*************************************************************************** + ** Write-only actors cannot send stored values to a worker or leak them in a + ** script rejection message. Native read permission is required for context. + ***************************************************************************/ + @Test + void writeOnlyActorCannotExposePriorContext() throws Exception + { + QRecord row = insert("PRIVATE-STORED-CONTEXT"); + bind(HookEvent.PRE_UPDATE); + instance.getTable(HOOK_TABLE).setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + QContext.getQSession().getPermissions().add(HOOK_TABLE + ".write"); + int before = worker.executeCalls; + QRecord denied = update(row.getValueInteger("id")); + assertFalse(denied.getErrors().isEmpty()); + assertEquals(before, worker.executeCalls); + assertFalse(denied.getErrors().toString().contains("PRIVATE-STORED-CONTEXT")); + QContext.getQSession().getPermissions().add(HOOK_TABLE + ".read"); + assertTrue(update(row.getValueInteger("id")).getErrors().isEmpty()); + } + + /*************************************************************************** + ** Composition preserves every native host callback exactly once per phase. + ***************************************************************************/ + @Test + void everyHostPhaseComposesOnceAndLateUpdateRefusesPatching() throws Exception + { + var table = instance.getTable(HOOK_TABLE); + for(HookEvent event : synchronousEvents()) + { + table.getCustomizers().put(TableCustomizers.valueOf(event.name() + "_RECORD").getRole(), new QCodeReference(Host.class)); + } + ScriptHooks.install(table, synchronousEvents()); + ScriptHooks.install(table, synchronousEvents()); + for(HookEvent event : synchronousEvents()) + { + bind(event); + } + QRecord row = insert("NEW"); + assertTrue(row.getErrors().isEmpty()); + assertTrue(update(row.getValueInteger("id")).getErrors().isEmpty()); + assertEquals(1, new DeleteAction().execute(new DeleteInput().withTableName(HOOK_TABLE).withPrimaryKeys(List.of(row.getValueInteger("id")))).getDeletedRecordCount()); + assertEquals(6, Host.calls); + Host.updatePhase = PreUpdatePhase.AFTER_ALL_VALIDATIONS; + service.setBindingEnabled(everything(), service.findBinding(HOOK_TABLE, HookEvent.PRE_UPDATE).orElseThrow().id(), false); + assertThrows(ScriptingException.class, () -> bind(HookEvent.PRE_UPDATE, "status")); + } + + /*************************************************************************** + ** A non-Java host reference cannot partially install an isolated hook set. + ***************************************************************************/ + @Test + void nonJavaMetadataIsRefusedBeforeMutation() + { + var table = instance.getTable(HOOK_TABLE); + var code = new QCodeReference().withCodeType(com.kingsrook.qqq.backend.core.model.metadata.code.QCodeType.JAVA_SCRIPT).withName("untrusted"); + table.getCustomizers().put(TableCustomizers.PRE_DELETE_RECORD.getRole(), code); + var before = new java.util.HashMap<>(table.getCustomizers()); + assertThrows(IllegalArgumentException.class, () -> ScriptHooks.install(table, HookEvent.PRE_INSERT, HookEvent.PRE_DELETE)); + assertEquals(before, table.getCustomizers()); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/PublishedContextTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/PublishedContextTest.java new file mode 100644 index 0000000..395c027 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/PublishedContextTest.java @@ -0,0 +1,171 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.service.ScriptingResults.PublishedContext; +import com.kingsrook.qqq.backend.core.context.QContext; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Published context execution is a narrow public boundary, separate from hook + ** decisions and never resolved through a mutable current-revision pointer. + *******************************************************************************/ +class PublishedContextTest extends ScriptingTestBase +{ + /*************************************************************************** + ** Discovery never executes and selects only exact context-tested builds. + ***************************************************************************/ + @Test + void chooserExcludesHookOnlyBuildsAndPreservesPublishedPin() throws Exception + { + Integer project = service.createProject(everything(), "Packing context"); + Integer hookOnly = publishNew(project, "hook only"); + assertKind(ScriptingException.Kind.INVALID, () -> service.selectPublishedContext(everything(), hookOnly)); + assertTrue(service.searchPublishedContexts(everything(), "Packing", 10).isEmpty()); + var first = build(draft(project, "context")); + service.testContext(everything(), first.buildId(), "{}", List.of()); + Integer publication = service.publish(everything(), first.buildId(), "context tested"); + int executions = worker.executeCalls; + var selected = service.selectPublishedContext(everything(), publication); + assertEquals(first.artifactId(), selected.pin().artifactId()); + assertTrue(selected.label().contains("Packing context")); + assertEquals(List.of(selected), service.searchPublishedContexts(everything(), "Packing", 10)); + assertTrue(service.searchPublishedContexts(everything(), "missing", 10).isEmpty()); + assertEquals(executions, worker.executeCalls); + publishNew(project, "later hook publication"); + assertEquals(selected, service.selectPublishedContext(everything(), publication)); + } + + /*************************************************************************** + ** Current named/native project access and bounded requests are mandatory. + ***************************************************************************/ + @Test + void chooserReauthorizesProjectAndRejectsUnboundedDiscovery() throws Exception + { + Integer project = newProject(); + var built = build(draft(project, "context")); + service.testContext(everything(), built.buildId(), "{}", List.of()); + Integer publication = service.publish(everything(), built.buildId(), "tested"); + assertKind(ScriptingException.Kind.INVALID, () -> service.searchPublishedContexts(everything(), "x".repeat(129), 1)); + assertKind(ScriptingException.Kind.INVALID, () -> service.searchPublishedContexts(everything(), "", 51)); + assertKind(ScriptingException.Kind.DENIED, () -> service.selectPublishedContext(only(ScriptingPermissions.VIEW), publication)); + instance.getTable("script").setPermissionRules(new com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules() + .withLevel(com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel.READ_WRITE_PERMISSIONS)); + assertKind(ScriptingException.Kind.DENIED, () -> service.selectPublishedContext(everything(), publication)); + assertKind(ScriptingException.Kind.DENIED, () -> service.searchPublishedContexts(everything(), "", 10)); + } + + /*************************************************************************** + ** A context test on another build never qualifies the selected publication. + ***************************************************************************/ + @Test + void chooserRequiresContextSuccessOnTheExactPublishedBuild() throws Exception + { + Integer project = newProject(); + Integer hookOnly = publishNew(project, "hook"); + var other = build(draft(project, "different context")); + service.testContext(everything(), other.buildId(), "{}", List.of()); + assertKind(ScriptingException.Kind.INVALID, () -> service.selectPublishedContext(everything(), hookOnly)); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW)); + assertKind(ScriptingException.Kind.DENIED, () -> service.searchPublishedContexts(everything(), "", 10)); + } + + /*************************************************************************** + ** Test a context build, publish it, and keep the old publication pinned after + ** a newer publication exists. The worker still receives only bounded JSON. + ***************************************************************************/ + @Test + void contextIsTestedPublishedAndPinned() throws Exception + { + Integer project = newProject(); + var first = build(draft(project, "one")); + worker.hookJson = "{\"greeting\":\"hello\",\"items\":[1,2],\"empty\":null}"; + assertTrue(service.testContext(everything(), first.buildId(), "{\"record\":{\"id\":7}}", List.of()).succeeded()); + Integer firstPublication = service.publish(everything(), first.buildId(), "first"); + var second = build(draft(project, "two")); + service.testContext(everything(), second.buildId(), "{}", List.of()); + service.publish(everything(), second.buildId(), "second"); + var pinned = service.validatePublishedContext(everything(), firstPublication); + assertEquals(first.artifactId(), pinned.artifactId()); + var result = service.executePublishedContext(everything(), pinned, "{\"record\":{\"id\":7}}", List.of()); + assertTrue(result.succeeded()); + assertEquals("hello", new JSONObject(result.contextJson()).getString("greeting")); + JSONObject message = new JSONObject(worker.lastExecuteRequest); + assertEquals("CONTEXT", message.getString("event")); + assertEquals("a-" + first.artifactId(), message.getString("artifactId")); + assertEquals(7, message.getJSONObject("payload").getJSONObject("context").getJSONObject("record").getInt("id")); + assertFalse(message.has("session")); + assertEquals("CONTEXT", rows(ScriptingTables.EXECUTION).getLast().getValueString("kind")); + } + + + + /*************************************************************************** + ** A context consumer cannot widen grants, skip execute permission, or rely + ** on stale permission captured before a user loses access. + ***************************************************************************/ + @Test + void contextRequiresCurrentPermissionAndNarrowedGrants() throws Exception + { + Integer publication = publishNew(newProject(), "context"); + var pinned = service.validatePublishedContext(everything(), publication); + int before = worker.executeCalls; + assertKind(ScriptingException.Kind.INTEGRITY, () -> service.executePublishedContext(everything(), + new PublishedContext(pinned.publicationId(), pinned.artifactId(), "0".repeat(64), pinned.capabilities()), "{}", List.of())); + assertKind(ScriptingException.Kind.DENIED, () -> service.executePublishedContext(everything(), pinned, "{}", List.of( + new CapabilityDeclaration(HOOK_TABLE, Set.of("amount"), 1, 1)))); + assertKind(ScriptingException.Kind.DENIED, () -> service.validatePublishedContext(only(ScriptingPermissions.VIEW), publication)); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW)); + assertKind(ScriptingException.Kind.DENIED, () -> service.executePublishedContext(everything(), pinned, "{}", List.of())); + assertEquals(before, worker.executeCalls); + } + + + + /*************************************************************************** + ** Context results cannot be confused with hook decisions; malformed context + ** input is refused and its failure is recorded. + ***************************************************************************/ + @Test + void contextInputIsBoundedAndSeparateFromHookShape() throws Exception + { + Integer publication = publishNew(newProject(), "context"); + var pinned = service.validatePublishedContext(everything(), publication); + worker.hookJson = "{\"action\":\"arbitrary application field\",\"data\":{\"nested\":true}}"; + var result = service.executePublishedContext(everything(), pinned, "{}", List.of()); + assertTrue(result.succeeded()); + assertEquals("arbitrary application field", new JSONObject(result.contextJson()).getString("action")); + var invalid = service.executePublishedContext(everything(), pinned, "[]", List.of()); + assertFalse(invalid.succeeded()); + assertEquals("INVALID", invalid.status()); + assertEquals("INVALID", rows(ScriptingTables.EXECUTION).getLast().getValueString("status")); + } +} + diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/QqqRecordReaderTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/QqqRecordReaderTest.java new file mode 100644 index 0000000..7d64fe3 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/QqqRecordReaderTest.java @@ -0,0 +1,88 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.scripting.capability.BrokerRequest; +import com.kingsrook.qbits.scripting.capability.QqqRecordReader; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + + +/******************************************************************************* + ** The host-side reader over the real QQQ QueryAction on the memory backend: + ** projection to JSON-safe values, filtering, limits, and metadata checks. + *******************************************************************************/ +class QqqRecordReaderTest extends ScriptingTestBase +{ + /*************************************************************************** + ** Three purchase orders, two of them OPEN. + ***************************************************************************/ + @BeforeEach + void orders() throws Exception + { + new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE).withRecords(List.of( + new QRecord().withValue("po_number", "PO-1").withValue("status", "OPEN").withValue("quantity", 1), + new QRecord().withValue("po_number", "PO-2").withValue("status", "OPEN").withValue("quantity", 2), + new QRecord().withValue("po_number", "PO-3").withValue("status", "CLOSED").withValue("quantity", 3)))); + } + + + + /*************************************************************************** + ** Only the requested fields come back, filtered and limited. + ***************************************************************************/ + @Test + void readsProjectedFilteredAndLimitedRows() throws Exception + { + QqqRecordReader reader = new QqqRecordReader(); + BrokerRequest.Criterion open = new BrokerRequest.Criterion("status", "EQUALS", List.of("OPEN")); + + List> rows = reader.read(HOOK_TABLE, List.of("po_number"), List.of(open), 10); + assertEquals(2, rows.size()); + assertEquals(Map.of("po_number", "PO-1"), rows.get(0)); + assertFalse(rows.get(0).containsKey("status")); + + assertEquals(1, reader.read(HOOK_TABLE, List.of("po_number"), List.of(open), 1).size()); + } + + + + /*************************************************************************** + ** Unknown tables and fields (selected or filtered) are refused. + ***************************************************************************/ + @Test + void unknownTablesAndFieldsAreRefused() throws Exception + { + QqqRecordReader reader = new QqqRecordReader(); + assertKind(ScriptingException.Kind.NOT_FOUND, () -> reader.read("no_such_table", List.of("a"), List.of(), 1)); + assertKind(ScriptingException.Kind.DENIED, () -> reader.read(HOOK_TABLE, List.of("no_such_field"), List.of(), 1)); + assertKind(ScriptingException.Kind.DENIED, () -> reader.read(HOOK_TABLE, List.of("po_number"), List.of(new BrokerRequest.Criterion("no_such_field", "EQUALS", List.of("x"))), 1)); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptHookCustomizerTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptHookCustomizerTest.java new file mode 100644 index 0000000..584d5a0 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptHookCustomizerTest.java @@ -0,0 +1,299 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHookCustomizer; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The record hook: ALLOW / REJECT / PATCH, patch confinement, fail-closed + ** behaviour, update prior values, and wiring through the real InsertAction. + *******************************************************************************/ +class ScriptHookCustomizerTest extends ScriptingTestBase +{ + private Integer publicationId; + + + + /*************************************************************************** + ** A published, bound script allowed to patch status, amount and quantity. + ***************************************************************************/ + @BeforeEach + void bound() throws Exception + { + publicationId = publishNew(newProject(), "hook"); + bindHook(publicationId, "status", "amount", "quantity"); + } + + + + /*************************************************************************** + ** Run the insert hook over fresh records. + ***************************************************************************/ + private List insert(QRecord... records) throws Exception + { + return (new ScriptHookCustomizer().preInsert(new InsertInput().withTableName(HOOK_TABLE), new ArrayList<>(List.of(records)), false)); + } + + + + /*************************************************************************** + ** A purchase order record. + ***************************************************************************/ + private static QRecord order() + { + return (new QRecord().withValue("po_number", "PO-1").withValue("status", "NEW")); + } + + + + /*************************************************************************** + ** ALLOW leaves the record alone and writes bounded, attributed history. + ***************************************************************************/ + @Test + void allowLeavesRecordUnchangedAndRecordsHistory() throws Exception + { + QRecord record = insert(order()).get(0); + + assertTrue(record.getErrors().isEmpty()); + assertEquals("NEW", record.getValueString("status")); + QRecord history = rows(ScriptingTables.EXECUTION).getLast(); + assertEquals("HOOK", history.getValueString("kind")); + assertEquals("OK", history.getValueString("status")); + assertEquals("ALLOW", history.getValueString("result_action")); + assertEquals("alice@example.com", history.getValueString("actor_id")); + } + + + + /*************************************************************************** + ** The worker sees proposed values as JSON only, never the record object. + ***************************************************************************/ + @Test + void workerReceivesJsonOfProposedValuesOnly() throws Exception + { + insert(order()); + + JSONObject request = new JSONObject(worker.lastExecuteRequest); + assertEquals("NEW", request.getJSONObject("payload").getJSONObject("proposed").getString("status")); + assertTrue(request.getJSONObject("payload").isNull("prior") || !request.getJSONObject("payload").has("prior")); + assertEquals("PRE_INSERT", request.getString("event")); + } + + + + /*************************************************************************** + ** REJECT becomes a record error with the script's reason, sanitized. + ***************************************************************************/ + @Test + void rejectAddsRecordError() throws Exception + { + worker.hookJson = "{\"action\":\"REJECT\",\"message\":\"PO total is too high\"}"; + + QRecord record = insert(order()).get(0); + + assertEquals(1, record.getErrors().size()); + assertTrue(record.getErrors().get(0).getMessage().contains("PO total is too high")); + } + + + + /*************************************************************************** + ** A patch to listed fields is type-checked and applied. + ***************************************************************************/ + @Test + void patchAppliesToListedFields() throws Exception + { + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\"APPROVED\",\"amount\":12.5,\"quantity\":3}}"; + + QRecord record = insert(order()).get(0); + + assertTrue(record.getErrors().isEmpty()); + assertEquals("APPROVED", record.getValueString("status")); + assertEquals(new BigDecimal("12.5"), record.getValue("amount")); + assertEquals(3, record.getValueInteger("quantity")); + } + + + + /*************************************************************************** + ** A patch touching an unlisted field is refused whole: nothing is applied. + ***************************************************************************/ + @Test + void patchOutsideListedFieldsIsRefusedAllOrNothing() throws Exception + { + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\"APPROVED\",\"po_number\":\"HIJACK\"}}"; + + QRecord record = insert(order()).get(0); + + assertEquals(1, record.getErrors().size()); + assertEquals("NEW", record.getValueString("status")); + assertEquals("PO-1", record.getValueString("po_number")); + } + + + + /*************************************************************************** + ** Wrong-typed or non-whole numbers are refused. + ***************************************************************************/ + @Test + void patchWithWrongTypeIsRefused() throws Exception + { + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"quantity\":\"many\"}}"; + assertEquals(1, insert(order()).get(0).getErrors().size()); + + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"quantity\":1.5}}"; + assertEquals(1, insert(order()).get(0).getErrors().size()); + } + + + + /*************************************************************************** + ** If the script cannot run, the write is refused, not allowed through. + ***************************************************************************/ + @Test + void unavailableOrUnqualifiedWorkerRejectsTheWrite() throws Exception + { + worker.unavailable = true; + QRecord down = insert(order()).get(0); + assertEquals(1, down.getErrors().size()); + assertTrue(down.getErrors().get(0).getMessage().startsWith("The script hook could not be completed")); + assertEquals("UNAVAILABLE", rows(ScriptingTables.EXECUTION).getLast().getValueString("status")); + + worker.unavailable = false; + worker.qualification = "revoked"; + assertEquals(1, insert(order()).get(0).getErrors().size()); + } + + + + /*************************************************************************** + ** A worker that crashes or answers badly also refuses the write. + ***************************************************************************/ + @Test + void badWorkerAnswerRejectsTheWrite() throws Exception + { + worker.hookJson = "{\"action\":\"EXPLODE\"}"; + assertEquals(1, insert(order()).get(0).getErrors().size()); + + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":{\"nested\":true}}}"; + assertEquals(1, insert(order()).get(0).getErrors().size()); + } + + + + /*************************************************************************** + ** An oversized batch is refused without running any script. + ***************************************************************************/ + @Test + void oversizedBatchIsRefused() throws Exception + { + MemoryRecordStore.getInstance().reset(); + install(new ScriptingRuntime().withWorker(worker).withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN)).withRequiredQualificationId("qual-1").withMaxRecordsPerHook(2)); + bindHook(publishNew(newProject(), "batch"), "status"); + int callsBefore = worker.executeCalls; + + List out = insert(order(), order(), order()); + + assertTrue(out.stream().allMatch(r -> r.getErrors().size() == 1)); + assertEquals(callsBefore, worker.executeCalls); + } + + + + /*************************************************************************** + ** No enabled binding for the event means the hook does nothing. + ***************************************************************************/ + @Test + void eventWithoutBindingPassesThrough() throws Exception + { + int callsBefore = worker.executeCalls; + List out = new ScriptHookCustomizer().preUpdate(new UpdateInput().withTableName(HOOK_TABLE), new ArrayList<>(List.of(order())), false, Optional.empty()); + + assertTrue(out.get(0).getErrors().isEmpty()); + assertEquals(callsBefore, worker.executeCalls); + } + + + + /*************************************************************************** + ** An update hook receives prior values next to the proposed ones. + ***************************************************************************/ + @Test + void updateHookReceivesPriorValues() throws Exception + { + com.kingsrook.qbits.scripting.hooks.ScriptHooks.install(instance.getTable(HOOK_TABLE), HookEvent.PRE_UPDATE); + service.bind(everything(), new BindingSpec("po-update", HOOK_TABLE, HookEvent.PRE_UPDATE, publicationId, + List.of(new CapabilityDeclaration(HOOK_TABLE, Set.of("status"), 5, 1)), Set.of("status"))); + QRecord stored = new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", "A")))).getRecords().getFirst(); + QRecord proposed = new QRecord().withValue("id", stored.getValue("id")).withValue("status", "B"); + QRecord prior = new QRecord().withValue("id", stored.getValue("id")).withValue("status", "FORGED"); + + List out = new ScriptHookCustomizer().preUpdate(new UpdateInput().withTableName(HOOK_TABLE), new ArrayList<>(List.of(proposed)), false, Optional.of(List.of(prior))); + + assertTrue(out.get(0).getErrors().isEmpty()); + JSONObject payload = new JSONObject(worker.lastExecuteRequest).getJSONObject("payload"); + assertEquals("B", payload.getJSONObject("proposed").getString("status")); + assertEquals("A", payload.getJSONObject("prior").getString("status")); + assertEquals("PRE_UPDATE", new JSONObject(worker.lastExecuteRequest).getString("event")); + } + + + + /*************************************************************************** + ** Through the real InsertAction a rejected record is not stored and a patched + ** record is stored with the patched value. + ***************************************************************************/ + @Test + void insertActionHonoursRejectAndPatch() throws Exception + { + worker.hookJson = "{\"action\":\"REJECT\",\"message\":\"no\"}"; + QRecord rejected = new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE).withRecords(List.of(order()))).getRecords().get(0); + assertEquals(1, rejected.getErrors().size()); + assertTrue(rows(HOOK_TABLE).isEmpty()); + + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\"APPROVED\"}}"; + QRecord patched = new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE).withRecords(List.of(order()))).getRecords().get(0); + assertTrue(patched.getErrors().isEmpty()); + assertEquals("APPROVED", rows(HOOK_TABLE).get(0).getValueString("status")); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptTimingModuleProbe.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptTimingModuleProbe.java new file mode 100644 index 0000000..cefe1b7 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptTimingModuleProbe.java @@ -0,0 +1,51 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.kingsrook.qbits.scripting; + +import com.kingsrook.qbits.scripting.observability.ScriptTiming; + +/******************************************************************************* + ** Trusted no-JFR child fixture, with no QQQ context or worker execution. + *******************************************************************************/ +public final class ScriptTimingModuleProbe +{ + /*************************************************************************** + ** Utility fixture. + ***************************************************************************/ + private ScriptTimingModuleProbe() {} + + /*************************************************************************** + ** Prove missing JFR does not affect either repeated calls or business code. + ***************************************************************************/ + public static void main(String[] args) + { + var first = ScriptTiming.start(ScriptTiming.Phase.EXECUTION_TOTAL); + if(first != ScriptTiming.start(ScriptTiming.Phase.WORKER_DISPATCH)) + { + throw new IllegalStateException("Expected shared disabled span"); + } + first.failed(); + first.close(); + System.out.println("NO_JFR_OK"); + } +} \ No newline at end of file diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptTimingTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptTimingTest.java new file mode 100644 index 0000000..db9d6f0 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptTimingTest.java @@ -0,0 +1,235 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + + +package com.kingsrook.qbits.scripting; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHooks; +import com.kingsrook.qbits.scripting.observability.ScriptTiming; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import jdk.jfr.Recording; +import jdk.jfr.consumer.RecordedEvent; +import jdk.jfr.consumer.RecordingFile; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/******************************************************************************* + ** Actual JFR recording over native host routing with the existing protocol-only + ** FakeWorker. These are correctness/privacy tests, never a runtime benchmark. + *******************************************************************************/ +class ScriptTimingTest extends ScriptingTestBase +{ + @TempDir + Path directory; + + /*************************************************************************** + ** Default recordings leave this event disabled and reuse one no-op object. + ***************************************************************************/ + @Test + void disabledByDefaultUsesSharedNoop() throws Exception + { + try(var recording = new Recording()) + { + recording.start(); + var first = ScriptTiming.start(ScriptTiming.Phase.EXECUTION_TOTAL); + assertSame(first, ScriptTiming.start(ScriptTiming.Phase.WORKER_DISPATCH)); + first.failed(); + first.close(); + first.close(); + recording.stop(); + Path file = directory.resolve("disabled.jfr"); + recording.dump(file); + assertTrue(events(file).isEmpty()); + } + } + + /*************************************************************************** + ** Native insert/update exercise every declared stage without recording data. + ***************************************************************************/ + @Test + void actualHookPhasesContainOnlyClosedLabels() throws Exception + { + String sentinel = "private-source-record-token-message-marker"; + Integer publication = publishNew(newProject(), sentinel); + ScriptHooks.install(instance.getTable(HOOK_TABLE), HookEvent.PRE_INSERT, HookEvent.PRE_UPDATE); + bindHook(publication); + service.bind(everything(), new BindingSpec("update", HOOK_TABLE, HookEvent.PRE_UPDATE, publication, declared(), Set.of())); + worker.onExecute = channel -> channel.handle(sentinel); + int before = rows(ScriptingTables.EXECUTION).size(); + Path file = directory.resolve("enabled.jfr"); + try(var recording = recording()) + { + recording.start(); + QRecord inserted = new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE) + .withRecords(List.of(new QRecord().withValue("po_number", sentinel).withValue("status", "NEW")))).getRecords().getFirst(); + assertTrue(inserted.getErrors().isEmpty()); + QRecord updated = new UpdateAction().execute(new UpdateInput().withTableName(HOOK_TABLE) + .withRecords(List.of(new QRecord().withValue("id", inserted.getValueInteger("id")).withValue("status", "UPDATED")))).getRecords().getFirst(); + assertTrue(updated.getErrors().isEmpty()); + recording.stop(); + recording.dump(file); + } + assertEquals(before + 4, rows(ScriptingTables.EXECUTION).size(), "Timing preserves required STARTED and outcome histories"); + List events = events(file); + assertEquals(Set.of(ScriptTiming.Phase.values()).stream().map(Enum::name).collect(Collectors.toSet()), + events.stream().map(event -> event.getString("phase")).collect(Collectors.toSet())); + for(RecordedEvent event : events) + { + assertEquals("RETURNED", event.getString("outcome")); + assertEquals(Set.of("startTime", "duration", "eventThread", "stackTrace", "phase", "outcome"), + event.getEventType().getFields().stream().map(field -> field.getName()).collect(Collectors.toSet())); + assertEquals(null, event.getStackTrace()); + assertFalse(event.toString().contains(sentinel)); + assertFalse(event.getDuration().isNegative()); + } + } + + /*************************************************************************** + ** A real dispatch failure still vetoes native DML and records its outcome. + ***************************************************************************/ + @Test + void throwingWorkerPreservesVetoAndAudit() throws Exception + { + Integer publication = publishNew(newProject(), "private-worker-source-marker"); + ScriptHooks.install(instance.getTable(HOOK_TABLE), HookEvent.PRE_INSERT); + bindHook(publication); + worker.onExecute = channel -> + { + throw new IllegalStateException("private-worker-exception-marker"); + }; + int before = rows(ScriptingTables.EXECUTION).size(); + Path file = directory.resolve("worker-failure.jfr"); + try(var recording = recording()) + { + recording.start(); + QRecord refused = new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE) + .withRecords(List.of(new QRecord().withValue("po_number", "PRIVATE").withValue("status", "NEW")))).getRecords().getFirst(); + assertFalse(refused.getErrors().isEmpty()); + recording.stop(); + recording.dump(file); + } + assertTrue(rows(HOOK_TABLE).isEmpty()); + assertEquals(before + 2, rows(ScriptingTables.EXECUTION).size()); + assertEquals("UNAVAILABLE", rows(ScriptingTables.EXECUTION).getLast().getValueString("status")); + List events = events(file); + assertTrue(events.stream().anyMatch(event -> "WORKER_DISPATCH".equals(event.getString("phase")) && "THREW".equals(event.getString("outcome")))); + assertTrue(events.stream().anyMatch(event -> "EXECUTION_TOTAL".equals(event.getString("phase")) && "RETURNED".equals(event.getString("outcome")))); + assertEquals(2, events.stream().filter(event -> "AUDIT_PERSISTENCE".equals(event.getString("phase"))).count()); + assertFalse(events.toString().contains("private-worker")); + } + + /*************************************************************************** + ** A stopped recording cannot replace the original application exception. + ***************************************************************************/ + @Test + void recordingLifecycleAndFailureDoNotChangeBusinessException() throws Exception + { + Path file = directory.resolve("throw.jfr"); + var original = new IllegalStateException("private-exception-message-marker"); + try(var recording = recording()) + { + recording.start(); + var thrown = assertThrows(IllegalStateException.class, () -> + { + try(var timing = ScriptTiming.start(ScriptTiming.Phase.EXECUTION_TOTAL)) + { + try + { + throw original; + } + catch(Throwable failure) + { + timing.failed(); + throw failure; + } + } + }); + assertSame(original, thrown); + var unfinished = ScriptTiming.start(ScriptTiming.Phase.WORKER_DISPATCH); + recording.stop(); + recording.dump(file); + unfinished.close(); + unfinished.close(); + } + assertTrue(events(file).stream().anyMatch(event -> "THREW".equals(event.getString("outcome")))); + assertFalse(events(file).toString().contains(original.getMessage())); + } + + /*************************************************************************** + ** A trusted child JVM with jdk.jfr absent must still execute ordinary code. + ***************************************************************************/ + @Test + void missingJfrModuleIsNoop() throws Exception + { + String executable = Path.of(System.getProperty("java.home"), "bin", System.getProperty("os.name").startsWith("Windows") ? "java.exe" : "java").toString(); + String classpath = Path.of(ScriptTiming.class.getProtectionDomain().getCodeSource().getLocation().toURI()) + + java.io.File.pathSeparator + Path.of(ScriptTimingModuleProbe.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + Process process = new ProcessBuilder(executable, "--limit-modules", "java.base", "-cp", classpath, ScriptTimingModuleProbe.class.getName()).redirectErrorStream(true).start(); + try + { + assertTrue(process.waitFor(10, TimeUnit.SECONDS)); + assertEquals(0, process.exitValue()); + assertEquals("NO_JFR_OK", new String(process.getInputStream().readAllBytes(), java.nio.charset.StandardCharsets.UTF_8).trim()); + } + finally + { + if(process.isAlive()) + { + process.destroyForcibly(); + } + } + } + + /*************************************************************************** + ** Enable only the opt-in event, preserving its no-stack default. + ***************************************************************************/ + private Recording recording() + { + var recording = new Recording(); + recording.enable(ScriptTiming.EVENT_NAME).withThreshold(Duration.ZERO); + return (recording); + } + + /*************************************************************************** + ** Inspect only this component's recorded events. + ***************************************************************************/ + private List events(Path file) throws Exception + { + return (RecordingFile.readAllEvents(file).stream().filter(event -> ScriptTiming.EVENT_NAME.equals(event.getEventType().getName())).toList()); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingAuthoringProcessesTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingAuthoringProcessesTest.java new file mode 100644 index 0000000..4ed16e7 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingAuthoringProcessesTest.java @@ -0,0 +1,128 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting; + +import java.util.List; + +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.metadata.fields.AdornmentType; +import com.kingsrook.qqq.backend.core.model.session.QCredentialScope; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/*************************************************************************** + ** Native process clients create server-marked projects through the public + ** service, without a browser-specific API or worker availability. + ***************************************************************************/ +class ScriptingAuthoringProcessesTest extends ScriptingTestBase +{ + /*************************************************************************** + ** The app advertises the standard form and its native record-result link. + ***************************************************************************/ + @Test + void nativeMetadataAndCreationNeedNoWorker() throws Exception + { + worker.unavailable = true; + var process = instance.getProcess(ScriptingAuthoringProcesses.CREATE_PROJECT); + assertEquals(0, process.getMinInputRecords()); + assertEquals("name", process.getFrontendStep("details").getFormFields().getFirst().getName()); + assertTrue(instance.getApp("scripting").getSections().getFirst().getProcesses().contains(process.getName())); + assertTrue(process.getFrontendStep("result").getViewFields().getFirst().getAdornments().stream() + .anyMatch(adornment -> adornment.getType().equals(AdornmentType.LINK))); + var output = create("Native script"); + Integer id = output.getValueInteger("scriptId"); + assertNotNull(id); + var row = new GetAction().execute(new GetInput("script").withPrimaryKey(id)).getRecord(); + assertEquals("ISOLATED", row.getValueString("executionMode")); + assertEquals("Native script", row.getValueString("name")); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** Repeated registration preserves host fields and the existing native widget. + ***************************************************************************/ + @Test + void viewerMetadataIsDeclarativeAndDoesNotReplaceHostSections() throws Exception + { + var script = instance.getTable("script"); + var sections = List.copyOf(script.getSections()); + var widget = instance.getWidget(ScriptingNavigation.VIEWER); + ScriptingNavigation.register(instance); + assertEquals(sections, script.getSections()); + assertEquals(widget, instance.getWidget(ScriptingNavigation.VIEWER)); + assertEquals(1, script.getSections().stream().filter(section -> ScriptingNavigation.VIEWER.equals(section.getWidgetName())).count()); + assertTrue(script.getSections().getFirst().getFieldNames().contains("executionMode")); + } + + /*************************************************************************** + ** A previously visible form does not retain a revoked author grant. + ***************************************************************************/ + @Test + void revokedGrantAndReadOnlyCredentialRefuseCreate() throws Exception + { + QContext.getQSession().getPermissions().remove(ScriptingPermissions.DRAFT); + assertThrows(QException.class, () -> create("Denied")); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT) + .withCredentialScope(new QCredentialScope().withReadOnly(true))); + assertThrows(QException.class, () -> create("Read only")); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** A form cannot set legacy mode, publication or caller identity. + ***************************************************************************/ + @Test + void invalidNameAndExtraAuthorityFieldsDoNotBypassService() throws Exception + { + assertThrows(QException.class, () -> create(" ")); + var input = new RunBackendStepInput().withProcessName(ScriptingAuthoringProcesses.CREATE_PROJECT); + input.addValue("name", "Bounded"); + input.addValue("executionMode", "LEGACY"); + input.addValue("actor", "somebody-else"); + var output = new RunBackendStepOutput(); + new ScriptingAuthoringProcesses.Create().run(input, output); + var row = new GetAction().execute(new GetInput("script").withPrimaryKey(output.getValueInteger("scriptId"))).getRecord(); + assertEquals("ISOLATED", row.getValueString("executionMode")); + assertFalse(output.getValues().containsKey("artifact")); + } + + /*************************************************************************** + ** Same native backend step invoked by ordinary process execution. + ***************************************************************************/ + private RunBackendStepOutput create(String name) throws Exception + { + var input = new RunBackendStepInput().withProcessName(ScriptingAuthoringProcesses.CREATE_PROJECT); + input.addValue("name", name); + var output = new RunBackendStepOutput(); + new ScriptingAuthoringProcesses.Create().run(input, output); + return (output); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingAuthoringTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingAuthoringTest.java new file mode 100644 index 0000000..f9474fb --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingAuthoringTest.java @@ -0,0 +1,116 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting; + +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Native authoring views and optimistic editing without parallel source state. + *******************************************************************************/ +class ScriptingAuthoringTest extends ScriptingTestBase +{ + /*************************************************************************** + ** History stays separate from source and excludes private artifact fields. + ***************************************************************************/ + @Test + void viewsLoadOnlyAuthorizedProjectAndRevision() throws Exception + { + Integer project = newProject(); + assertNull(service.authoring().project(everything(), project).latestRevisionId()); + Integer revision = draft(project, "one"); + var built = build(revision); + service.testContext(everything(), built.buildId(), "{}", List.of()); + service.publish(everything(), built.buildId(), "one"); + var view = service.authoring().project(everything(), project); + assertEquals(revision, view.latestRevisionId()); + assertEquals(1, view.builds().size()); + assertEquals(1, view.publications().size()); + assertTrue(view.workerAvailable()); + assertTrue(view.actions().contains("publish")); + assertEquals(source("one"), service.authoring().draft(everything(), project, revision).files()); + assertKind(ScriptingException.Kind.DENIED, () -> service.authoring().draft(everything(), newProject(), revision)); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW)); + assertTrue(service.authoring().project(only(ScriptingPermissions.VIEW), project).actions().isEmpty()); + } + + /*************************************************************************** + ** Legacy source is never implicitly opted into the new editor save route. + ***************************************************************************/ + @Test + void optimisticSaveRequiresIsolatedProjectAndCurrentBase() throws Exception + { + Integer legacy = newLegacyProject(); + assertKind(ScriptingException.Kind.DENIED, () -> service.saveDraft(everything(), legacy, null, "draft", source("one"))); + Integer project = newProject(); + Integer first = service.saveDraft(everything(), project, null, "first", source("one")); + assertKind(ScriptingException.Kind.INVALID, () -> service.saveDraft(everything(), project, null, "stale", source("two"))); + Integer second = service.saveDraft(everything(), project, first, "second", source("two")); + assertEquals(second, service.authoring().project(everything(), project).latestRevisionId()); + assertEquals(2, service.authoring().project(everything(), project).revisions().size()); + var built = build(second); + service.authoring().requireBuild(everything(), project, built.buildId()); + assertTrue(service.authoring().build(everything(), project, built.buildId()).success()); + assertKind(ScriptingException.Kind.DENIED, () -> service.authoring().requireBuild(everything(), newProject(), built.buildId())); + } + + /*************************************************************************** + ** API assistance names only explicitly requested currently readable fields. + ***************************************************************************/ + @Test + void scopedTypingsRespectFieldAndSessionPolicy() throws Exception + { + Integer project = newProject(); + String types = service.authoring().typings(everything(), project, declared()); + assertTrue(types.contains("query(table: \"purchase_order\"")); + assertTrue(types.contains("\"status\"")); + assertFalse(types.contains("created_by")); + for(var event : com.kingsrook.qbits.scripting.hooks.HookEvent.values()) + { + assertTrue(types.contains("\"" + event.name() + "\"")); + } + assertTrue(types.contains("record?: never")); + assertTrue(types.contains("readonly event?: CommittedEvent")); + instance.getTable(HOOK_TABLE).addField(new QFieldMetaData("private", QFieldType.PASSWORD)); + assertKind(ScriptingException.Kind.DENIED, () -> service.authoring().typings(everything(), project, + List.of(new CapabilityDeclaration(HOOK_TABLE, Set.of("private"), 1, 1)))); + assertKind(ScriptingException.Kind.DENIED, () -> service.authoring().typings(everything(), project, + List.of(new CapabilityDeclaration("missing", Set.of("id"), 1, 1)))); + assertKind(ScriptingException.Kind.LIMIT, () -> service.authoring().typings(everything(), project, null)); + QContext.setQSession(sessionWith()); + assertKind(ScriptingException.Kind.DENIED, () -> service.authoring().project(everything(), project)); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingFileActionsProcessTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingFileActionsProcessTest.java new file mode 100644 index 0000000..fcdba53 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingFileActionsProcessTest.java @@ -0,0 +1,257 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting; + +import java.util.HashMap; +import java.util.List; +import com.kingsrook.qbits.scripting.ScriptingLifecycleProcesses.Operation; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QCredentialScope; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/*************************************************************************** + ** Native file operations append drafts through the existing owner service. + ** No worker, filesystem source mutation or publication is involved. + ***************************************************************************/ +class ScriptingFileActionsProcessTest extends ScriptingTestBase +{ + private static final SourceFile MAIN = new SourceFile("main.ts", "import { value } from './lib/value.js';\nexport const main = () => value;"); + private static final SourceFile HELPER = new SourceFile("lib/value.ts", "export const value = 1;"); + private static final List ORIGINAL = List.of(HELPER, MAIN); + + /*************************************************************************** + ** Exact source bytes and earlier revisions survive rename and later removal. + ***************************************************************************/ + @Test + void renameAndRemoveAppendImmutableDraftsWithoutRewritingImports() throws Exception + { + int project = newProject(); + int first = seed(project); + var rename = load(project, Operation.RENAME_FILE, HELPER.path()); + rename.addValue("newPath", "lib/renamed.ts"); + rename.addValue("contents", "must not replace retained source"); + rename.addValue("selectedPath", MAIN.path()); + var renamed = apply(rename); + int second = renamed.getValueInteger("revisionId"); + assertEquals("DRAFT_FILE_RENAMED", renamed.getValueString("resultStatus")); + assertEquals(List.of(new SourceFile("lib/renamed.ts", HELPER.content()), MAIN), + service.authoring().draft(everything(), project, second).files()); + var removed = apply(load(project, Operation.REMOVE_FILE, "lib/renamed.ts")); + int third = removed.getValueInteger("revisionId"); + assertEquals("DRAFT_FILE_REMOVED", removed.getValueString("resultStatus")); + assertEquals(List.of(MAIN), service.authoring().draft(everything(), project, third).files()); + assertEquals(ORIGINAL, service.authoring().draft(everything(), project, first).files()); + assertEquals(List.of(new SourceFile("lib/renamed.ts", HELPER.content()), MAIN), + service.authoring().draft(everything(), project, second).files()); + assertTrue(service.authoring().project(everything(), project).builds().isEmpty()); + assertTrue(service.authoring().project(everything(), project).publications().isEmpty()); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** Generic native clients discover both screens; selected identity is readonly. + ***************************************************************************/ + @Test + void metadataExposesReadonlySelectionAndPerRequestChoices() throws Exception + { + int project = newProject(); + seed(project); + for(Operation operation : List.of(Operation.RENAME_FILE, Operation.REMOVE_FILE)) + { + var process = instance.getProcess(ScriptingLifecycleProcesses.name(operation)); + assertNotNull(process); + var prepared = prepare(project, operation); + assertNotNull(prepared.getValueInteger("expectedRevisionId")); + var details = process.getFrontendStep("details").getFormFields(); + assertFalse(details.stream().filter(field -> field.getName().equals("path")).findFirst().orElseThrow().getIsEditable()); + assertEquals(operation == Operation.RENAME_FILE, + details.stream().anyMatch(field -> field.getName().equals("newPath"))); + assertNull(process.getFrontendStep("choose").getFormFields().getFirst().getInlinePossibleValueSource()); + var input = load(project, operation, HELPER.path()); + assertEquals(HELPER.path(), input.getValueString("path")); + assertNull(input.getValueString("contents")); + assertNotNull(input.getValueString("sourceActionNotice")); + } + } + + /*************************************************************************** + ** Existing authoritative snapshot validators reject collisions and unsafe paths. + ***************************************************************************/ + @Test + void invalidRenameDoesNotCreateRevision() throws Exception + { + int project = newProject(); + int revision = seed(project); + for(String path : List.of("main.ts", "MAIN.ts", "../escape.ts", "/absolute.ts", "lib\\bad.ts", "file.js", HELPER.path())) + { + var input = load(project, Operation.RENAME_FILE, HELPER.path()); + input.addValue("newPath", path); + assertThrows(QException.class, () -> apply(input), path); + assertEquals(revision, service.authoring().project(everything(), project).latestRevisionId()); + } + assertEquals(ORIGINAL, service.authoring().draft(everything(), project, revision).files()); + } + + /*************************************************************************** + ** Case-only rename is valid for the same file, never an alias for another file. + ***************************************************************************/ + @Test + void caseOnlyRenamePreservesContentsAndExactSelection() throws Exception + { + int project = newProject(); + seed(project); + var wrongCase = prepare(project, Operation.RENAME_FILE); + wrongCase.addValue("selectedPath", "LIB/value.ts"); + assertThrows(QException.class, () -> new ScriptingLifecycleSteps.LoadFile().run(wrongCase, new RunBackendStepOutput())); + var rename = load(project, Operation.RENAME_FILE, HELPER.path()); + rename.addValue("newPath", "lib/Value.ts"); + int revision = apply(rename).getValueInteger("revisionId"); + assertEquals(List.of(new SourceFile("lib/Value.ts", HELPER.content()), MAIN), + service.authoring().draft(everything(), project, revision).files()); + } + + /*************************************************************************** + ** Drift before selection or final submit cannot overwrite the current draft. + ***************************************************************************/ + @Test + void staleSelectionAndSubmitBothRefuse() throws Exception + { + for(Operation operation : List.of(Operation.RENAME_FILE, Operation.REMOVE_FILE)) + { + int project = newProject(); + int first = seed(project); + var beforeLoad = prepare(project, operation); + beforeLoad.addValue("selectedPath", HELPER.path()); + var input = load(project, operation, HELPER.path()); + input.addValue("newPath", "renamed.ts"); + int current = service.saveDraft(everything(), project, first, "concurrent", List.of(MAIN, new SourceFile(HELPER.path(), "export const value = 2;"))); + assertThrows(QException.class, () -> new ScriptingLifecycleSteps.LoadFile().run(beforeLoad, new RunBackendStepOutput())); + assertThrows(QException.class, () -> apply(input)); + assertEquals(current, service.authoring().project(everything(), project).latestRevisionId()); + assertEquals(ORIGINAL, service.authoring().draft(everything(), project, first).files()); + } + } + + /*************************************************************************** + ** A displayed form cannot retain revoked action rights or a write-capable token. + ***************************************************************************/ + @Test + void currentAuthorityAndCredentialCeilingAreRequiredOnSubmit() throws Exception + { + int project = newProject(); + int revision = seed(project); + var rename = load(project, Operation.RENAME_FILE, HELPER.path()); + rename.addValue("newPath", "renamed.ts"); + var remove = load(project, Operation.REMOVE_FILE, HELPER.path()); + QContext.getQSession().getPermissions().remove(ScriptingPermissions.DRAFT); + for(var input : List.of(rename, remove)) + { + assertThrows(QException.class, () -> apply(input)); + } + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT) + .withCredentialScope(new QCredentialScope().withReadOnly(true))); + for(var input : List.of(rename, remove)) + { + assertThrows(QException.class, () -> apply(input)); + } + assertEquals(revision, service.authoring().project(everything(), project).latestRevisionId()); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** No empty project or missing selected path is manufactured by removal. + ***************************************************************************/ + @Test + void removalCannotEmptyProjectOrSkipValidSelection() throws Exception + { + int empty = newProject(); + assertThrows(QException.class, () -> prepare(empty, Operation.REMOVE_FILE)); + assertThrows(QException.class, () -> prepare(empty, Operation.RENAME_FILE)); + int project = newProject(); + seed(project); + var forged = prepare(project, Operation.REMOVE_FILE); + forged.addValue("path", "missing.ts"); + assertThrows(QException.class, () -> apply(forged)); + apply(load(project, Operation.REMOVE_FILE, HELPER.path())); + assertThrows(QException.class, () -> prepare(project, Operation.REMOVE_FILE)); + var direct = new RunBackendStepInput().withProcessName(ScriptingLifecycleProcesses.name(Operation.REMOVE_FILE)); + direct.addValue("scriptId", project); + direct.addValue("expectedRevisionId", service.authoring().project(everything(), project).latestRevisionId()); + direct.addValue("path", MAIN.path()); + assertThrows(QException.class, () -> apply(direct)); + assertEquals(List.of(MAIN), service.authoring().draft(everything(), project, direct.getValueInteger("expectedRevisionId")).files()); + } + + /*************************************************************************** + ** Use real immutable owner storage for the two-file base. + ***************************************************************************/ + private int seed(Integer project) throws Exception + { + return (service.saveDraft(everything(), project, null, "initial", ORIGINAL)); + } + + /*************************************************************************** + ** Preserve server-produced identity and expected revision between native steps. + ***************************************************************************/ + private RunBackendStepInput prepare(Integer project, Operation operation) throws Exception + { + String process = ScriptingLifecycleProcesses.name(operation); + var input = new RunBackendStepInput().withProcessName(process).withRecords(List.of(new QRecord().withValue("id", project))); + var output = new RunBackendStepOutput(); + new ScriptingLifecycleSteps.Prepare().run(input, output); + return (new RunBackendStepInput().withProcessName(process).withValues(new HashMap<>(output.getValues()))); + } + + /*************************************************************************** + ** Simulate the declared choose screen and retain only backend-loaded identity. + ***************************************************************************/ + private RunBackendStepInput load(Integer project, Operation operation, String path) throws Exception + { + var input = prepare(project, operation); + input.addValue("selectedPath", path); + var output = new RunBackendStepOutput(); + new ScriptingLifecycleSteps.LoadFile().run(input, output); + output.getValues().forEach(input::addValue); + return (input); + } + + /*************************************************************************** + ** Call the same action as any standard native process client. + ***************************************************************************/ + private RunBackendStepOutput apply(RunBackendStepInput input) throws Exception + { + var output = new RunBackendStepOutput(); + new ScriptingLifecycleSteps.Apply().run(input, output); + return (output); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingGitImportProcessTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingGitImportProcessTest.java new file mode 100644 index 0000000..ac45465 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingGitImportProcessTest.java @@ -0,0 +1,213 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting; + +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.scripting.ScriptingLifecycleProcesses.Operation; +import com.kingsrook.qbits.scripting.source.GitCommitRef; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QCredentialScope; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/*************************************************************************** + ** Native forms use the existing importer/service; trusted source fixtures never + ** contact GitHub, compile code, execute a script or supply provider credentials. + ***************************************************************************/ +class ScriptingGitImportProcessTest extends ScriptingTestBase +{ + private static final GitCommitRef REF = new GitCommitRef("example/scripts", "a".repeat(40), "source"); + + /*************************************************************************** + ** Native metadata exposes import and returns the retained exact receipt. + ***************************************************************************/ + @Test + void importsThroughNativeFormsWithoutBuildingOrPublishing() throws Exception + { + AtomicInteger fetches = importer(); + int project = newProject(); + int previous = draft(project, "previous"); + var input = prepare(project); + var form = instance.getProcess(input.getProcessName()).getFrontendStep("details"); + assertEquals(List.of("scriptId", "repository", "commitSha", "subdirectory", "importNotice"), + form.getFormFields().stream().map(field -> field.getName()).toList()); + assertNotNull(input.getValueString("importNotice")); + var result = apply(input); + int revision = result.getValueInteger("revisionId"); + var imported = service.authoring().draft(everything(), project, revision); + assertEquals("DRAFT_IMPORTED", result.getValueString("resultStatus")); + assertEquals(source("imported"), imported.files()); + assertEquals(source("previous"), service.authoring().draft(everything(), project, previous).files()); + JSONObject receipt = new JSONObject(result.getValueString("resultJson")); + assertEquals(REF.repository(), receipt.getString("repository")); + assertEquals(REF.commitSha(), receipt.getString("commitSha")); + assertEquals(REF.subdirectory(), receipt.getString("subdirectory")); + assertEquals(imported.sourceSha256(), receipt.getString("sourceSha256")); + assertEquals(1, fetches.get()); + assertTrue(service.authoring().project(everything(), project).builds().isEmpty()); + assertTrue(service.authoring().project(everything(), project).publications().isEmpty()); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** Existing draft drift is refused before either native or direct retrieval. + ***************************************************************************/ + @Test + void staleBaseNeverFetchesOrOverwrites() throws Exception + { + AtomicInteger fetches = importer(); + int project = newProject(); + var input = prepare(project); + int latest = draft(project, "newer"); + assertThrows(QException.class, () -> apply(input)); + assertThrows(ScriptingException.class, () -> service.importDraft(everything(), project, null, REF)); + assertEquals(0, fetches.get()); + assertEquals(latest, service.authoring().project(everything(), project).latestRevisionId()); + } + + /*************************************************************************** + ** Missing configuration, moving refs and unsafe paths never reach a fetch. + ***************************************************************************/ + @Test + void unavailableAndMalformedImportsFailClosed() throws Exception + { + int project = newProject(); + assertThrows(QException.class, () -> prepare(project)); + AtomicInteger fetches = importer(); + for(String sha : List.of("main", "A".repeat(40), "a".repeat(39))) + { + var input = prepare(project); + input.addValue("commitSha", sha); + assertThrows(QException.class, () -> apply(input)); + } + var unsafe = prepare(project); + unsafe.addValue("subdirectory", "../secret"); + assertThrows(QException.class, () -> apply(unsafe)); + assertEquals(0, fetches.get()); + } + + /*************************************************************************** + ** Revoke a rendered form; a read-only token cannot trigger external retrieval. + ***************************************************************************/ + @Test + void revokedActionAndReadOnlyCredentialDoNotFetch() throws Exception + { + AtomicInteger fetches = importer(); + int project = newProject(); + var input = prepare(project); + QContext.getQSession().getPermissions().remove(ScriptingPermissions.DRAFT); + assertThrows(QException.class, () -> apply(input)); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT) + .withCredentialScope(new QCredentialScope().withReadOnly(true))); + assertThrows(QException.class, () -> apply(input)); + assertThrows(ScriptingException.class, () -> service.importDraft(everything(), project, null, REF)); + assertEquals(0, fetches.get()); + } + + /*************************************************************************** + ** Preflight does not replace the final atomic stale-base check after retrieval. + ***************************************************************************/ + @Test + void concurrentDraftDuringFetchRemainsCurrent() throws Exception + { + importer(); + int project = newProject(); + var input = prepare(project); + runtime.withImporter((actor, id, ref) -> + { + service.saveDraft(actor, id, "concurrent", source("concurrent")); + return (SourceSnapshot.of(ref.origin(), source("imported"))); + }); + assertThrows(QException.class, () -> apply(input)); + int latest = service.authoring().project(everything(), project).latestRevisionId(); + assertEquals(source("concurrent"), service.authoring().draft(everything(), project, latest).files()); + assertTrue(rows(ScriptingTables.IMPORT).isEmpty()); + } + + /*************************************************************************** + ** An empty optional directory means repository root, without alias coercion. + ***************************************************************************/ + @Test + void emptyOptionalDirectoryUsesExactRepositoryRoot() throws Exception + { + importer(); + runtime.withImporter((actor, id, ref) -> + { + assertEquals(null, ref.subdirectory()); + return (SourceSnapshot.of(ref.origin(), source("root"))); + }); + var input = prepare(newProject()); + input.addValue("subdirectory", ""); + assertEquals("DRAFT_IMPORTED", apply(input).getValueString("resultStatus")); + } + + /*************************************************************************** + ** Trusted source-only importer; all authority and persistence stay real. + ***************************************************************************/ + private AtomicInteger importer() + { + AtomicInteger calls = new AtomicInteger(); + runtime.withImporter((actor, id, ref) -> + { + calls.incrementAndGet(); + return (SourceSnapshot.of(ref.origin(), source("imported"))); + }); + return (calls); + } + + /*************************************************************************** + ** Preserve server-produced revision state through the standard form step. + ***************************************************************************/ + private RunBackendStepInput prepare(Integer project) throws Exception + { + String process = ScriptingLifecycleProcesses.name(Operation.IMPORT_COMMIT); + var selected = new RunBackendStepInput().withProcessName(process).withRecords(List.of(new QRecord().withValue("id", project))); + var output = new RunBackendStepOutput(); + new ScriptingLifecycleSteps.Prepare().run(selected, output); + var input = new RunBackendStepInput().withProcessName(process).withValues(new HashMap<>(output.getValues())); + input.addValue("repository", REF.repository()); + input.addValue("commitSha", REF.commitSha()); + input.addValue("subdirectory", REF.subdirectory()); + return (input); + } + + /*************************************************************************** + ** Same backend step used by every generic native process client. + ***************************************************************************/ + private RunBackendStepOutput apply(RunBackendStepInput input) throws Exception + { + var output = new RunBackendStepOutput(); + new ScriptingLifecycleSteps.Apply().run(input, output); + return (output); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingImportTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingImportTest.java new file mode 100644 index 0000000..db1ae86 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingImportTest.java @@ -0,0 +1,117 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qbits.scripting.source.GitCommitRef; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/******************************************************************************* + ** Import is ordinary authorized draft intake with retained exact provenance. + *******************************************************************************/ +class ScriptingImportTest extends ScriptingTestBase +{ + private static final GitCommitRef REF = new GitCommitRef("acme/scripts", "a".repeat(40), "src"); + + /*************************************************************************** + ** Persist exact commit/hash with draft; importing never calls the worker. + ***************************************************************************/ + @Test + void importRetainsProvenanceAndRequiresExpectedBase() throws Exception + { + AtomicInteger calls = new AtomicInteger(); + install(new ScriptingRuntime().withImporter((actor, project, ref) -> + { + calls.incrementAndGet(); + return (SourceSnapshot.of(ref.origin(), source("imported"))); + })); + Integer project = isolated(); + Integer revision = service.importDraft(everything(), project, null, REF); + var draft = service.authoring().draft(everything(), project, revision); + assertNotNull(draft.provenance()); + assertEquals(REF.commitSha(), draft.provenance().commitSha()); + assertEquals(draft.sourceSha256(), draft.provenance().sourceSha256()); + assertEquals(1, rows(ScriptingTables.IMPORT).size()); + assertEquals(0, rows(ScriptingTables.BUILD).size()); + assertKind(ScriptingException.Kind.INVALID, () -> service.importDraft(everything(), project, null, REF)); + assertEquals(1, rows(ScriptingTables.IMPORT).size()); + service.importDraft(everything(), project, REF); + assertEquals(2, rows(ScriptingTables.IMPORT).size()); + assertEquals(2, calls.get()); + QContext.setQSession(sessionWith()); + assertKind(ScriptingException.Kind.DENIED, () -> service.importDraft(everything(), project, revision, REF)); + assertEquals(2, calls.get()); + } + + /*************************************************************************** + ** Missing or misassociated provider output cannot be saved as source. + ***************************************************************************/ + @Test + void refusesUnavailableAndForgedImports() throws Exception + { + Integer project = isolated(); + assertKind(ScriptingException.Kind.UNAVAILABLE, () -> service.importDraft(everything(), project, null, REF)); + install(new ScriptingRuntime().withImporter((actor, id, ref) -> SourceSnapshot.of("wrong-origin", source("bad")))); + Integer wrongProject = isolated(); + assertKind(ScriptingException.Kind.INTEGRITY, () -> service.importDraft(everything(), wrongProject, null, REF)); + assertEquals(0, rows("scriptRevision").size()); + } + + /*************************************************************************** + ** Current actor permissions are checked again after external import returns. + ***************************************************************************/ + @Test + void revokedPermissionsDuringFetchPreventSave() throws Exception + { + install(new ScriptingRuntime().withImporter((actor, project, ref) -> + { + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW)); + return (SourceSnapshot.of(ref.origin(), source("imported"))); + })); + Integer project = isolated(); + assertKind(ScriptingException.Kind.DENIED, () -> service.importDraft(everything(), project, null, REF)); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW)); + assertEquals(0, rows("scriptRevision").size()); + } + + /*************************************************************************** + ** Fixture marker only; successful real-core authority integration is separate. + ***************************************************************************/ + private Integer isolated() throws Exception + { + Integer project = newProject(); + return (project); + } +} + diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingLifecycleProcessesTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingLifecycleProcessesTest.java new file mode 100644 index 0000000..c03a08d --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingLifecycleProcessesTest.java @@ -0,0 +1,211 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting; + +import java.util.HashMap; +import java.util.List; +import com.kingsrook.qbits.scripting.ScriptingLifecycleProcesses.Operation; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QCredentialScope; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/*************************************************************************** + ** Native fallback forms exercise public owner services on real QQQ metadata + ** with a protocol fixture. No script text is executed by these host tests. + ***************************************************************************/ +class ScriptingLifecycleProcessesTest extends ScriptingTestBase +{ + /*************************************************************************** + ** Add/edit preserves all other files and the immutable previous revision. + ***************************************************************************/ + @Test + void addAndEditPreserveOtherFilesAndUsePerRequestChoices() throws Exception + { + int project = newProject(); + var add = prepare(project, Operation.ADD_FILE); + add.addValue("contents", "export const one = 1;"); + int first = apply(add).getValueInteger("revisionId"); + var second = prepare(project, Operation.ADD_FILE); + second.addValue("path", "lib/two.ts"); + second.addValue("contents", "export const two = 2;"); + apply(second); + var edit = prepare(project, Operation.EDIT_FILE); + edit.addValue("selectedPath", "lib/two.ts"); + var loaded = new RunBackendStepOutput(); + new ScriptingLifecycleSteps.LoadFile().run(edit, loaded); + assertEquals("export const two = 2;", loaded.getValueString("contents")); + loaded.getValues().forEach(edit::addValue); + edit.addValue("contents", "export const two = 22;"); + int revision = apply(edit).getValueInteger("revisionId"); + var files = service.authoring().draft(everything(), project, revision).files(); + assertEquals(List.of(new SourceFile("lib/two.ts", "export const two = 22;"), new SourceFile("main.ts", "export const one = 1;")), files); + assertEquals(List.of(new SourceFile("main.ts", "export const one = 1;")), service.authoring().draft(everything(), project, first).files()); + assertNull(instance.getProcess(ScriptingLifecycleProcesses.name(Operation.EDIT_FILE)).getFrontendStep("choose").getFormFields().getFirst().getInlinePossibleValueSource()); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** Optimistic source pins reject stale saves and stale compilation forms. + ***************************************************************************/ + @Test + void staleEditAndCheckRefuseWithoutOverwritingCurrentSource() throws Exception + { + int project = newProject(); + int first = draft(project, "first"); + var edit = prepare(project, Operation.EDIT_FILE); + var check = prepare(project, Operation.CHECK); + int latest = draft(project, "newer"); + edit.addValue("contents", "stale"); + assertThrows(QException.class, () -> apply(edit)); + assertThrows(QException.class, () -> apply(check)); + assertEquals(latest, service.authoring().project(everything(), project).latestRevisionId()); + assertEquals(source("first"), service.authoring().draft(everything(), project, first).files()); + assertTrue(service.authoring().project(everything(), project).builds().isEmpty()); + } + + /*************************************************************************** + ** A complete native path returns exact build/test/publication record links. + ***************************************************************************/ + @Test + void checkTestAndPublishUseExactImmutableBuild() throws Exception + { + int project = newProject(); + draft(project, "context"); + var check = prepare(project, Operation.CHECK); + check.addValue("capabilitiesJson", CapabilityDeclaration.listToJson(declared())); + var checked = apply(check); + int build = checked.getValueInteger("buildId"); + assertEquals("CHECK_PASSED", checked.getValueString("resultStatus")); + var publish = prepare(project, Operation.PUBLISH); + assertEquals(build, publish.getValueInteger("buildId")); + assertThrows(QException.class, () -> apply(publish)); + var test = prepare(project, Operation.TEST_CONTEXT); + test.addValue("inputJson", "{\"input\":1}"); + test.addValue("capabilitiesJson", "[]"); + var tested = apply(test); + assertEquals("OK", tested.getValueString("resultStatus")); + assertNotNull(tested.getValueInteger("executionId")); + draft(project, "later draft does not change tested build"); + publish.addValue("note", "Native published context"); + var published = apply(publish); + assertNotNull(published.getValueInteger("publicationId")); + assertEquals(build, published.getValueInteger("buildId")); + assertEquals(1, worker.executeCalls); + } + + /*************************************************************************** + ** Caller-modified child IDs cannot cross the authorized selected project. + ***************************************************************************/ + @Test + void foreignBuildAndInvalidSourceChoiceAreRefused() throws Exception + { + int project = newProject(); + build(draft(project, "one")); + int other = newProject(); + var foreign = build(draft(other, "two")); + var test = prepare(project, Operation.TEST_CONTEXT); + test.addValue("buildId", foreign.buildId()); + assertThrows(QException.class, () -> apply(test)); + var edit = prepare(project, Operation.EDIT_FILE); + edit.addValue("selectedPath", "other.ts"); + edit.addValue("path", "other.ts"); + assertThrows(QException.class, () -> new ScriptingLifecycleSteps.LoadFile().run(edit, new RunBackendStepOutput())); + edit.addValue("contents", "not an existing file"); + assertThrows(QException.class, () -> apply(edit)); + var add = prepare(project, Operation.ADD_FILE); + add.addValue("path", ENTRY.toUpperCase().replace(".TS", ".ts")); + assertThrows(QException.class, () -> apply(add)); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** A visible form retains neither a revoked action nor a broader credential. + ***************************************************************************/ + @Test + void currentPermissionsAndCredentialCeilingsApplyOnSubmit() throws Exception + { + int project = newProject(); + draft(project, "one"); + var edit = prepare(project, Operation.EDIT_FILE); + edit.addValue("path", ENTRY); + edit.addValue("contents", "edited"); + QContext.getQSession().getPermissions().remove(ScriptingPermissions.DRAFT); + assertThrows(QException.class, () -> apply(edit)); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT).withCredentialScope(new QCredentialScope().withReadOnly(true))); + assertThrows(QException.class, () -> apply(edit)); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** Shared codecs bound JSON before parsing; unknown/coercible grants fail closed. + ***************************************************************************/ + @Test + void capabilityShapesAndMissingDraftsAreBounded() throws Exception + { + assertEquals(declared(), ScriptingProcessInputs.capabilities(CapabilityDeclaration.listToJson(declared()))); + for(String bad : List.of("{}", "[1]", "[{\"operation\":\"WRITE\"}]", "[".repeat(30), "x".repeat(262145), + "[{\"operation\":\"READ\",\"table\":\"t\",\"fields\":[\"id\"],\"maxRows\":\"1\",\"maxCalls\":1}]", + "[{\"operation\":\"READ\",\"table\":\"t\",\"fields\":[\"id\"],\"maxRows\":1,\"maxCalls\":1,\"extra\":true}]")) + { + assertThrows(ScriptingException.class, () -> ScriptingProcessInputs.capabilities(bad)); + } + int empty = newProject(); + assertThrows(QException.class, () -> prepare(empty, Operation.CHECK)); + assertThrows(QException.class, () -> prepare(empty, Operation.EDIT_FILE)); + assertThrows(QException.class, () -> prepare(empty, Operation.TEST_CONTEXT)); + assertFalse(instance.getProcess(ScriptingLifecycleProcesses.name(Operation.PUBLISH)).getFrontendStep("details").getFormFields().isEmpty()); + } + + /*************************************************************************** + ** Simulate native process state between the prepare step and rendered form. + ***************************************************************************/ + private RunBackendStepInput prepare(Integer project, Operation operation) throws Exception + { + String process = ScriptingLifecycleProcesses.name(operation); + var input = new RunBackendStepInput().withProcessName(process).withRecords(List.of(new QRecord().withValue("id", project))); + var output = new RunBackendStepOutput(); + new ScriptingLifecycleSteps.Prepare().run(input, output); + return (new RunBackendStepInput().withProcessName(process).withValues(new HashMap<>(output.getValues()))); + } + + /*************************************************************************** + ** Same apply entry as a CLI, Material or other generic process client. + ***************************************************************************/ + private RunBackendStepOutput apply(RunBackendStepInput input) throws Exception + { + var output = new RunBackendStepOutput(); + new ScriptingLifecycleSteps.Apply().run(input, output); + return (output); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingLifecycleTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingLifecycleTest.java new file mode 100644 index 0000000..e20e0bb --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingLifecycleTest.java @@ -0,0 +1,362 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BuildSummary; +import com.kingsrook.qbits.scripting.service.ScriptingResults.ExecutionSummary; +import com.kingsrook.qbits.scripting.service.ScriptingResults.HookBindingView; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.function.Executable; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Separate, explicitly authorized lifecycle steps; failed builds never publish; + ** integrity failures fail closed; drafts are immutable; versions are pinned. + *******************************************************************************/ +class ScriptingLifecycleTest extends ScriptingTestBase +{ + /*************************************************************************** + ** Assert a ScriptingException of one kind. + ***************************************************************************/ + private static void assertKind(ScriptingException.Kind kind, Executable executable) + { + assertEquals(kind, assertThrows(ScriptingException.class, executable).getKind()); + } + + + + /*************************************************************************** + ** Replace stored values as the system, simulating an attacker with + ** database access (not a path the service offers). + ***************************************************************************/ + private void tamper(String table, Integer id, Map values) throws Exception + { + QSession original = QContext.getQSession(); + QContext.setQSession(new QSystemUserSession()); + try + { + QRecord record = new QRecord().withValue("id", id); + values.forEach((k, v) -> record.setValue(k, v)); + new UpdateAction().execute(new UpdateInput().withTableName(table).withRecords(List.of(record))); + } + finally + { + QContext.setQSession(original); + } + } + + + + /*************************************************************************** + ** Each step needs its own permission, holds nothing implicitly, and a denied + ** step leaves no trace. + ***************************************************************************/ + @Test + void deniedPermissionsAreEnforcedPerStep() throws Exception + { + Integer scriptId = newProject(); + Integer publicationId = publishNew(scriptId, "seed"); + + assertKind(ScriptingException.Kind.DENIED, () -> service.saveDraft(only(ScriptingPermissions.CHECK), scriptId, "m", source("a"))); + Integer revisionId = draft(scriptId, "a"); + int buildsBefore = rows(ScriptingTables.BUILD).size(); + assertKind(ScriptingException.Kind.DENIED, () -> service.check(only(ScriptingPermissions.DRAFT), revisionId, ENTRY, declared())); + assertEquals(buildsBefore, rows(ScriptingTables.BUILD).size()); + + Integer buildId = build(revisionId).buildId(); + int executionsBefore = rows(ScriptingTables.EXECUTION).size(); + assertKind(ScriptingException.Kind.DENIED, () -> service.test(only(ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK), buildId, HookEvent.PRE_INSERT, "{}", null)); + assertEquals(executionsBefore, rows(ScriptingTables.EXECUTION).size()); + + assertKind(ScriptingException.Kind.DENIED, () -> service.publish(only(ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK, ScriptingPermissions.TEST), buildId, "n")); + assertEquals(1, rows(ScriptingTables.PUBLICATION).size()); + + assertKind(ScriptingException.Kind.DENIED, () -> service.bind(only(ScriptingPermissions.PUBLISH), new BindingSpec("b", HOOK_TABLE, HookEvent.PRE_INSERT, publicationId, List.of(), Set.of()))); + assertTrue(rows(ScriptingTables.BINDING).isEmpty()); + } + + + + /*************************************************************************** + ** Authoring needs a named person: not the system session, not anonymous. + ***************************************************************************/ + @Test + void actorIdentityIsRequired() throws Exception + { + assertKind(ScriptingException.Kind.DENIED, () -> Actor.fromSession(new QSystemUserSession(instance))); + assertKind(ScriptingException.Kind.DENIED, () -> Actor.fromSession(new QSession())); + assertKind(ScriptingException.Kind.DENIED, () -> Actor.fromSession(null)); + assertEquals("alice@example.com", Actor.fromSession(sessionWith(ScriptingPermissions.DRAFT)).id()); + assertKind(ScriptingException.Kind.DENIED, () -> Actor.fromSession(sessionWith(ScriptingPermissions.DRAFT)).require(ScriptingPermissions.PUBLISH)); + } + + + + /*************************************************************************** + ** A type error is recorded with diagnostics, produces no artifact, and can + ** never be tested or published. + ***************************************************************************/ + @Test + void failedCompileCannotBePublished() throws Exception + { + worker.compileOk = false; + BuildSummary summary = build(draft(newProject(), "bad")); + + assertFalse(summary.success()); + assertNull(summary.artifactId()); + assertEquals("TS2322", summary.diagnostics().get(0).code()); + assertEquals("FAILED", rows(ScriptingTables.BUILD).get(0).getValueString("status")); + assertTrue(rows(ScriptingTables.ARTIFACT).isEmpty()); + + assertKind(ScriptingException.Kind.INVALID, () -> service.publish(everything(), summary.buildId(), "n")); + ExecutionSummary refused = service.test(everything(), summary.buildId(), HookEvent.PRE_INSERT, "{}", null); + assertEquals("INVALID", refused.status()); + assertFalse(refused.succeeded()); + assertTrue(rows(ScriptingTables.PUBLICATION).isEmpty()); + assertEquals(0, worker.executeCalls); + } + + + + /*************************************************************************** + ** A worker that claims non-strict mode or a different toolchain is refused, + ** not trusted. + ***************************************************************************/ + @Test + void workerThatIgnoresPolicyIsRefused() throws Exception + { + Integer revisionId = draft(newProject(), "x"); + worker.strict = false; + assertKind(ScriptingException.Kind.INTEGRITY, () -> build(revisionId)); + worker.strict = true; + worker.toolchain = "some-other-toolchain"; + assertKind(ScriptingException.Kind.INTEGRITY, () -> build(revisionId)); + assertTrue(rows(ScriptingTables.ARTIFACT).isEmpty()); + } + + + + /*************************************************************************** + ** With no worker, no coordinator or an unqualified worker, nothing builds and + ** nothing runs, and no partial rows are written. + ***************************************************************************/ + @Test + void unconfiguredOrUnqualifiedWorkerFailsClosed() throws Exception + { + install(new ScriptingRuntime().withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN))); + Integer revisionId = draft(newProject(), "x"); + assertKind(ScriptingException.Kind.UNAVAILABLE, () -> build(revisionId)); + assertTrue(rows(ScriptingTables.BUILD).isEmpty()); + + install(new ScriptingRuntime()); + Integer otherRevision = draft(newProject(), "y"); + assertKind(ScriptingException.Kind.UNAVAILABLE, () -> build(otherRevision)); + + worker = new FakeWorker(); + install(new ScriptingRuntime().withWorker(worker).withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN))); + Integer blankRevision = draft(newProject(), "z"); + assertKind(ScriptingException.Kind.UNAVAILABLE, () -> build(blankRevision)); + + worker = new FakeWorker(); + worker.qualification = "different-evidence"; + install(new ScriptingRuntime().withWorker(worker).withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN)).withRequiredQualificationId("qual-1")); + Integer wrongRevision = draft(newProject(), "w"); + assertKind(ScriptingException.Kind.UNAVAILABLE, () -> build(wrongRevision)); + assertEquals(0, worker.executeCalls); + } + + + + /*************************************************************************** + ** A stored bundle edited after acceptance is refused before any run, even if + ** its hash column is edited to match; only the coordinator can attest. + ***************************************************************************/ + @Test + void tamperedStoredArtifactIsRejected() throws Exception + { + BuildSummary built = build(draft(newProject(), "t")); + assertTrue(service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + assertEquals(1, worker.executeCalls); + + String evil = "/* attacker supplied javascript */"; + tamper(ScriptingTables.ARTIFACT, built.artifactId(), Map.of("bundle", evil, "bundle_sha256", BuildCoordinator.sha256Hex(evil))); + + ExecutionSummary summary = service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null); + assertEquals("INTEGRITY", summary.status()); + assertFalse(summary.succeeded()); + assertEquals(1, worker.executeCalls); + assertKind(ScriptingException.Kind.INTEGRITY, () -> service.publish(everything(), built.buildId(), "n")); + } + + + + /*************************************************************************** + ** Source edited through generic core CRUD after the build is detected at + ** publish time. + ***************************************************************************/ + @Test + void sourceChangedAfterBuildBlocksPublish() throws Exception + { + Integer revisionId = draft(newProject(), "s"); + BuildSummary built = build(revisionId); + assertTrue(service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + + QRecord file = rows(ScriptRevisionFile.TABLE_NAME).get(0); + // Emulate out-of-band database corruption; the generic lifecycle guard is tested separately. + instance.getTable(ScriptRevisionFile.TABLE_NAME).getCustomizers().remove("preUpdateRecord"); + tamper(ScriptRevisionFile.TABLE_NAME, file.getValueInteger("id"), Map.of("contents", "export const injected = 1;")); + + assertKind(ScriptingException.Kind.INTEGRITY, () -> service.publish(everything(), built.buildId(), "n")); + assertTrue(rows(ScriptingTables.PUBLICATION).isEmpty()); + } + + + + /*************************************************************************** + ** Publishing needs a recorded passing test of that same build. + ***************************************************************************/ + @Test + void publishRequiresPassingTestOfSameBuild() throws Exception + { + BuildSummary built = build(draft(newProject(), "p")); + assertKind(ScriptingException.Kind.INVALID, () -> service.publish(everything(), built.buildId(), "n")); + + worker.unavailable = true; + ExecutionSummary failed = service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null); + assertEquals("UNAVAILABLE", failed.status()); + assertEquals(2, rows(ScriptingTables.EXECUTION).size()); + assertKind(ScriptingException.Kind.INVALID, () -> service.publish(everything(), built.buildId(), "n")); + + worker.unavailable = false; + assertTrue(service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + assertEquals(1, service.publish(everything(), built.buildId(), "n").intValue()); + } + + + + /*************************************************************************** + ** Saving a draft appends a new revision and never touches an earlier one or + ** an existing binding; a binding keeps running the publication it names. + ***************************************************************************/ + @Test + void draftsAreImmutableAndBindingStaysPinned() throws Exception + { + Integer scriptId = newProject(); + Integer revisionOne = draft(scriptId, "one"); + BuildSummary buildOne = build(revisionOne); + assertTrue(service.test(everything(), buildOne.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + Integer publicationOne = service.publish(everything(), buildOne.buildId(), "v1"); + Integer bindingId = bindHook(publicationOne, "status"); + + Integer revisionTwo = draft(scriptId, "two"); + assertNotEquals(revisionOne, revisionTwo); + assertEquals(2, rows("scriptRevision").size()); + assertEquals(1, rows("scriptRevision").get(0).getValueInteger("sequenceNo")); + assertEquals(2, rows("scriptRevision").get(1).getValueInteger("sequenceNo")); + assertTrue(rows(ScriptRevisionFile.TABLE_NAME).get(0).getValueString("contents").contains("one")); + assertEquals(1, rows(ScriptingTables.BUILD).size()); + assertEquals(1, rows(ScriptingTables.PUBLICATION).size()); + + BuildSummary buildTwo = build(revisionTwo); + assertTrue(service.test(everything(), buildTwo.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + Integer publicationTwo = service.publish(everything(), buildTwo.buildId(), "v2"); + assertNotEquals(publicationOne, publicationTwo); + assertEquals(2, rows(ScriptingTables.PUBLICATION).get(1).getValueInteger("version")); + + HookBindingView binding = service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).orElseThrow(); + assertEquals(bindingId, binding.id()); + assertEquals(publicationOne, binding.publicationId()); + + ExecutionSummary run = service.runHook(binding, "alice@example.com", "{\"status\":\"NEW\"}", null); + assertTrue(run.succeeded()); + QRecord artifactOne = rows(ScriptingTables.ARTIFACT).get(0); + assertTrue(worker.lastExecuteRequest.contains(artifactOne.getValueString("bundle_sha256"))); + assertTrue(worker.lastExecuteRequest.contains("a-" + buildOne.artifactId())); + } + + + + /*************************************************************************** + ** Grants cannot exceed what the build declared; patch targets must be + ** ordinary editable non-key fields; one enabled binding per table event. + ***************************************************************************/ + @Test + void bindingRulesAreEnforced() throws Exception + { + Integer publicationId = publishNew(newProject(), "b"); + List tooWide = List.of(new CapabilityDeclaration(HOOK_TABLE, Set.of("amount"), 5, 1)); + assertKind(ScriptingException.Kind.DENIED, () -> service.bind(everything(), new BindingSpec("x", HOOK_TABLE, HookEvent.PRE_INSERT, publicationId, tooWide, Set.of("status")))); + assertKind(ScriptingException.Kind.DENIED, () -> bindHook(publicationId, "id")); + assertKind(ScriptingException.Kind.DENIED, () -> bindHook(publicationId, "created_by")); + assertKind(ScriptingException.Kind.DENIED, () -> bindHook(publicationId, "no_such_field")); + assertKind(ScriptingException.Kind.INVALID, () -> service.bind(everything(), new BindingSpec("x", ScriptingTables.BUILD, HookEvent.PRE_INSERT, publicationId, List.of(), Set.of()))); + assertKind(ScriptingException.Kind.INVALID, () -> service.bind(everything(), new BindingSpec("x", "unknown_table", HookEvent.PRE_INSERT, publicationId, List.of(), Set.of()))); + assertTrue(rows(ScriptingTables.BINDING).isEmpty()); + + Integer bindingId = bindHook(publicationId, "status"); + assertKind(ScriptingException.Kind.INVALID, () -> bindHook(publicationId, "status")); + + service.setBindingEnabled(everything(), bindingId, false); + assertTrue(service.findBinding(HOOK_TABLE, HookEvent.PRE_INSERT).isEmpty()); + assertKind(ScriptingException.Kind.DENIED, () -> service.setBindingEnabled(only(ScriptingPermissions.PUBLISH), bindingId, true)); + } + + + + /*************************************************************************** + ** Generic CRUD cannot forge scripting rows, whoever the user is. + ***************************************************************************/ + @Test + void directWritesToScriptingTablesAreDenied() throws Exception + { + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK, ScriptingPermissions.TEST, ScriptingPermissions.PUBLISH, ScriptingPermissions.BIND)); + QRecord forged = new QRecord().withValue("bundle", "/* forged */").withValue("bundle_sha256", "x"); + assertThrows(QException.class, () -> new InsertAction().execute(new InsertInput().withTableName(ScriptingTables.ARTIFACT).withRecords(List.of(forged)))); + assertTrue(rows(ScriptingTables.ARTIFACT).isEmpty()); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingRecordSecurityTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingRecordSecurityTest.java new file mode 100644 index 0000000..0924f79 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingRecordSecurityTest.java @@ -0,0 +1,222 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.io.Serializable; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qqq.backend.core.actions.customizers.AbstractPreInsertCustomizer; +import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType; +import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.scripts.Script; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** A host may lock the script project table with native record security (for + ** example by tenant). Projects stay private to their lock value: the owner can + ** author, a different tenant cannot reach the hidden project, and the + ** configuration is refused unless the lock's key type lets the QBit's own system + ** reads through, instead of silently making even the owner's authoring "not found". + *******************************************************************************/ +class ScriptingRecordSecurityTest extends ScriptingTestBase +{ + + /*************************************************************************** + ** Stamp the caller's tenant on each new project, as a host would. + ***************************************************************************/ + public static class TenantStamp extends com.kingsrook.qqq.backend.core.actions.scripts.ScriptTableCustomizer + { + private static volatile QCodeReference existing; + + + + /*********************************************************************** + ** Run before native validation so the lock sees the value. + ***********************************************************************/ + @Override + public AbstractPreInsertCustomizer.WhenToRun whenToRunPreInsert(InsertInput insertInput, boolean isPreview) + { + return (AbstractPreInsertCustomizer.WhenToRun.BEFORE_ALL_VALIDATIONS); + } + + + + /*********************************************************************** + ** Copy the session's single tenant onto the new row. + ***********************************************************************/ + @Override + public List preInsert(InsertInput insertInput, List records, boolean isPreview) throws QException + { + if(existing != null) + { + records = QCodeLoader.getAdHoc(TableCustomizerInterface.class, existing).preInsert(insertInput, records, isPreview); + } + List values = QContext.getQSession().getSecurityKeyValues("tenant"); + records.forEach(record -> record.setValue("tenant", values.get(0))); + return (records); + } + } + + + + /*************************************************************************** + ** Build a locked-script-table instance with the production service. + ***************************************************************************/ + private void installLocked(boolean allAccessKey) throws Exception + { + MemoryRecordStore.getInstance().reset(); + runtime = new ScriptingRuntime().withWorker(worker).withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN)) + .withRequiredQualificationId("qual-1").withTransactionPolicy((transaction, tables) -> + { + // Deliberately volatile fixture: this does not attest durability. + }); + instance = new QInstance().withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS)); + instance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class)); + QSecurityKeyType tenant = new QSecurityKeyType().withName("tenant"); + instance.addSecurityKeyType(allAccessKey ? tenant.withAllAccessKeyName("tenantAllAccess") : tenant); + QTableMetaData script = core(Script.TABLE_NAME, Script.class); + script.addField(new QFieldMetaData("tenant", QFieldType.STRING)); + script.withRecordSecurityLocks(List.of(new RecordSecurityLock().withFieldName("tenant").withSecurityKeyType("tenant") + .withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE))); + instance.addTable(script); + instance.addTable(core(ScriptRevision.TABLE_NAME, ScriptRevision.class)); + instance.addTable(core(ScriptRevisionFile.TABLE_NAME, ScriptRevisionFile.class)); + instance.addTable(new QTableMetaData().withName(HOOK_TABLE).withBackendName("memory").withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("status", QFieldType.STRING))); + new ScriptingQBitProducer().withConfig(new ScriptingQBitConfig().withBackendName("memory").withRuntime(runtime)).produce(instance); + // Stamp before native validation, wrapping the guard the producer composed. + String role = TableCustomizers.PRE_INSERT_RECORD.getRole(); + TenantStamp.existing = script.getCustomizer(role).orElse(null); + script.getCustomizers().remove(role); + script.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(TenantStamp.class)); + service = ScriptingQBitConfig.serviceFor(instance); + instance.setEnforcePermissionsInCoreActions(true); + } + + + + /*************************************************************************** + ** Core table from its entity, on the memory backend. + ***************************************************************************/ + private static QTableMetaData core(String name, Class entity) throws Exception + { + QTableMetaData table = new QTableMetaData().withName(name).withBackendName("memory").withPrimaryKeyField("id").withFieldsFromEntity(entity); + table.getFields().values().forEach(field -> field.setPossibleValueSourceName(null)); + return (table); + } + + + + /*************************************************************************** + ** A tenant's session holding every lifecycle permission. + ***************************************************************************/ + private static QSession session(String user, String tenant) + { + return (new QSession().withUser(new QUser().withIdReference(user)).withSecurityKeyValue("tenant", tenant) + .withPermissions(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK, ScriptingPermissions.TEST, + ScriptingPermissions.PUBLISH, ScriptingPermissions.BIND, ScriptingPermissions.EXECUTE)); + } + + + + /*************************************************************************** + ** The matching service actor for a session user. + ***************************************************************************/ + private static Actor actor(String user) + { + return (Actor.of(user, Set.of(ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK, ScriptingPermissions.TEST, + ScriptingPermissions.PUBLISH, ScriptingPermissions.BIND, ScriptingPermissions.VIEW, ScriptingPermissions.EXECUTE))); + } + + + + /*************************************************************************** + ** Without an all-access key the lock would hide every project from the QBit + ** itself, so the configuration is refused with the exact remedy. + ***************************************************************************/ + @Test + void lockWithoutSystemAccessKeyIsRefusedAtStartup() + { + QException refused = assertThrows(QException.class, () -> installLocked(false)); + assertTrue(refused.getMessage().contains("all-access key") && refused.getMessage().contains("script") && refused.getMessage().contains("tenant")); + } + + + + /*************************************************************************** + ** The owner authors end to end; another tenant cannot reach the hidden project + ** or its revisions, even holding the same lifecycle permissions. + ***************************************************************************/ + @Test + void tenantLockedProjectIsOwnedByItsTenantAndHiddenFromOthers() throws Exception + { + installLocked(true); + QContext.init(instance, session("alice@example.com", "a")); + Integer project = service.createProject(actor("alice@example.com"), "tenant a project"); + Integer revision = service.saveDraft(actor("alice@example.com"), project, "a draft", source("a")); + assertTrue(service.check(actor("alice@example.com"), revision, ENTRY, declared()).success()); + assertEquals(1, rows(ScriptRevision.TABLE_NAME).size()); + + QContext.init(instance, session("bob@example.com", "b")); + Actor bob = actor("bob@example.com"); + assertThrows(ScriptingException.class, () -> service.saveDraft(bob, project, "intrusion", source("b"))); + assertThrows(ScriptingException.class, () -> service.check(bob, revision, ENTRY, declared())); + assertEquals(0, rows(Script.TABLE_NAME).size()); + assertEquals(0, rows(ScriptRevision.TABLE_NAME).size()); + Integer own = service.createProject(bob, "tenant b project"); + assertTrue(service.saveDraft(bob, own, "b draft", source("b")) > 0); + + QContext.init(instance, session("alice@example.com", "a")); + assertThrows(ScriptingException.class, () -> service.saveDraft(actor("alice@example.com"), own, "cross", source("a"))); + assertEquals(1, rows(Script.TABLE_NAME).size()); + } + +} \ No newline at end of file diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingStorageBoundaryTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingStorageBoundaryTest.java new file mode 100644 index 0000000..7cc02cc --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingStorageBoundaryTest.java @@ -0,0 +1,104 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.Set; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData; +import com.kingsrook.qqq.backend.core.model.session.QCredentialScope; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Semantic service entry points preserve native channel and credential ceilings + ** before privileged storage, and missing persistence qualification fails closed. + *******************************************************************************/ +class ScriptingStorageBoundaryTest extends ScriptingTestBase +{ + /*************************************************************************** + ** A named draft grant cannot override a read-only credential's ceiling. + ***************************************************************************/ + @Test + void readOnlyCredentialCannotAppendDraftOrHistory() throws Exception + { + Integer project = newProject(); + var built = build(draft(project, "baseline")); + int revisions = rows("scriptRevision").size(); + int executions = rows(ScriptingTables.EXECUTION).size(); + QContext.getQSession().withCredentialScope(new QCredentialScope().withReadOnly(true)); + assertKind(ScriptingException.Kind.DENIED, () -> draft(project, "denied")); + assertKind(ScriptingException.Kind.DENIED, () -> service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null)); + assertEquals(revisions, rows("scriptRevision").size()); + assertEquals(executions, rows(ScriptingTables.EXECUTION).size()); + assertEquals(0, worker.executeCalls); + } + + + /*************************************************************************** + ** Restricting a source or owned table remains effective under system writes. + ***************************************************************************/ + @Test + void deniedSourceAndOwnedChannelsDoNotDisappear() throws Exception + { + Integer project = newProject(); + Integer revision = draft(project, "allowed"); + ClientChannelsMetaData.ofOrWithNew(instance).withDeniedWritePattern("api", "scriptRevision"); + QContext.getQSession().withClientChannel("api"); + assertKind(ScriptingException.Kind.DENIED, () -> draft(project, "denied")); + ClientChannelsMetaData.ofOrWithNew(instance).withDeniedWritePattern("api", ScriptingTables.BUILD); + assertKind(ScriptingException.Kind.DENIED, () -> build(revision)); + assertTrue(rows(ScriptingTables.BUILD).isEmpty()); + } + + + /*************************************************************************** + ** A source-only credential cannot use a semantic route to read excluded owned + ** tables. Removing the scope restores the already-authorized normal action. + ***************************************************************************/ + @Test + void excludedOwnedTablesStayExcludedFromSemanticReads() throws Exception + { + Integer project = newProject(); + build(draft(project, "history")); + QContext.getQSession().withCredentialScope(new QCredentialScope().withTables(Set.of("script", "scriptRevision", "scriptRevisionFile"))); + assertKind(ScriptingException.Kind.DENIED, () -> service.authoring().project(everything(), project)); + QContext.getQSession().setCredentialScope(null); + assertEquals(project, service.authoring().project(everything(), project).scriptId()); + } + + + /*************************************************************************** + ** No default, memory or JDBC transaction is implicitly a qualified deployment. + ***************************************************************************/ + @Test + void absentPolicyDeniesPersistenceBeforeSourceChanges() throws Exception + { + Integer project = newProject(); + runtime.withTransactionPolicy(null); + assertKind(ScriptingException.Kind.UNAVAILABLE, () -> draft(project, "unqualified")); + assertTrue(rows("scriptRevision").isEmpty()); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingTestBase.java b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingTestBase.java new file mode 100644 index 0000000..f963faf --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/ScriptingTestBase.java @@ -0,0 +1,292 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHookCustomizer; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BuildSummary; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qbits.scripting.service.ScriptingService; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.scripts.Script; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevision; +import com.kingsrook.qqq.backend.core.model.scripts.ScriptRevisionFile; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Real QQQ instance on the core memory backend (no database, no network), the + ** real core script tables, the real QBit producer and a protocol-level fake + ** worker. Nothing here runs script text. + *******************************************************************************/ +abstract class ScriptingTestBase +{ + static final String TOOLCHAIN = "ts-test-toolchain-1"; + static final String ENTRY = "src/index.ts"; + static final String HOOK_TABLE = "purchase_order"; + static final byte[] KEY = "0123456789abcdef0123456789abcdef".getBytes(StandardCharsets.UTF_8); + + ScriptingRuntime runtime; + FakeWorker worker; + ScriptingService service; + QInstance instance; + + /*************************************************************************** + ** Synchronous fixtures intentionally exclude separately installed Jobs phases. + ***************************************************************************/ + static HookEvent[] synchronousEvents() + { + return (Arrays.stream(HookEvent.values()).filter(event -> !event.isAfterCommit()).toArray(HookEvent[]::new)); + } + + + + /*************************************************************************** + ** Fresh memory store, instance, QBit and session for every test. + ***************************************************************************/ + @BeforeEach + void setUpScripting() throws Exception + { + MemoryRecordStore.getInstance().reset(); + worker = new FakeWorker(); + install(new ScriptingRuntime() + .withWorker(worker) + .withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN)) + .withRequiredQualificationId("qual-1")); + } + + + + /*************************************************************************** + ** Clear the thread's QQQ context. + ***************************************************************************/ + @AfterEach + void tearDownScripting() + { + QContext.clear(); + } + + + + /*************************************************************************** + ** Build the instance around a runtime and make a session with every + ** scripting permission the current tests use. + ***************************************************************************/ + void install(ScriptingRuntime runtime) throws Exception + { + this.runtime = runtime; + if(runtime.getTransactionPolicy() == null) + { + // Explicit protocol/volatile fixture only: never production durability evidence. + runtime.withTransactionPolicy((transaction, tables) -> + { + // Deliberately volatile fixture: this does not attest durability. + }); + } + instance = new QInstance().withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS)); + instance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class)); + instance.addTable(core(Script.TABLE_NAME, Script.class)); + instance.addTable(core(ScriptRevision.TABLE_NAME, ScriptRevision.class)); + instance.addTable(core(ScriptRevisionFile.TABLE_NAME, ScriptRevisionFile.class)); + instance.addTable(new QTableMetaData().withName(HOOK_TABLE).withBackendName("memory").withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER)) + .withField(new QFieldMetaData("po_number", QFieldType.STRING)) + .withField(new QFieldMetaData("status", QFieldType.STRING)) + .withField(new QFieldMetaData("amount", QFieldType.DECIMAL)) + .withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) + .withField(new QFieldMetaData("created_by", QFieldType.STRING).withIsEditable(false)) + .withCustomizer(TableCustomizers.PRE_INSERT_RECORD, new QCodeReference(ScriptHookCustomizer.class))); + + new ScriptingQBitProducer().withConfig(new ScriptingQBitConfig().withBackendName("memory").withRuntime(runtime)).produce(instance); + service = ScriptingQBitConfig.serviceFor(instance); + instance.setEnforcePermissionsInCoreActions(true); + QContext.init(instance, sessionWith(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK, ScriptingPermissions.TEST, ScriptingPermissions.PUBLISH, ScriptingPermissions.BIND, ScriptingPermissions.EXECUTE)); + } + + + + /*************************************************************************** + ** A core table derived from its entity, on the memory backend. + ***************************************************************************/ + private static QTableMetaData core(String name, Class entity) throws Exception + { + QTableMetaData table = new QTableMetaData().withName(name).withBackendName("memory").withPrimaryKeyField("id").withFieldsFromEntity(entity); + table.getFields().values().forEach(field -> field.setPossibleValueSourceName(null)); + return (table); + } + + + + /*************************************************************************** + ** An identified QQQ session holding exactly these permissions. + ***************************************************************************/ + static QSession sessionWith(String... permissions) + { + return (new QSession().withUser(new QUser().withIdReference("alice@example.com")).withPermissions(permissions)); + } + + + + /*************************************************************************** + ** An actor holding exactly these scripting permissions. + ***************************************************************************/ + static Actor only(String... permissions) + { + return (Actor.of("alice@example.com", Set.copyOf(Arrays.asList(permissions)))); + } + + + + /*************************************************************************** + ** An actor holding every lifecycle permission. + ***************************************************************************/ + static Actor everything() + { + return (only(ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK, ScriptingPermissions.TEST, ScriptingPermissions.PUBLISH, ScriptingPermissions.BIND, ScriptingPermissions.VIEW, ScriptingPermissions.EXECUTE)); + } + + + + /*************************************************************************** + ** One-file project source. + ***************************************************************************/ + static List source(String body) + { + return (List.of(new SourceFile(ENTRY, "export function run(): string { return \"" + body + "\"; }"))); + } + + + + /*************************************************************************** + ** What the test scripts declare they may read. + ***************************************************************************/ + static List declared() + { + return (List.of(new CapabilityDeclaration(HOOK_TABLE, Set.of("po_number", "status"), 10, 2))); + } + + + + /*************************************************************************** + ** Insert a core script project and return its id. + ***************************************************************************/ + Integer newLegacyProject() throws Exception + { + return (new InsertAction().execute(new InsertInput().withTableName(Script.TABLE_NAME) + .withRecords(List.of(new QRecord().withValue("name", "legacy")))).getRecords().getFirst().getValueInteger("id")); + } + + + + /*************************************************************************** + ** Create a marked isolated project through the production service. + ***************************************************************************/ + Integer newProject() throws Exception + { + return (service.createProject(everything(), "po-validate")); + } + + + + /*************************************************************************** + ** Save a draft with the given body and return the revision id. + ***************************************************************************/ + Integer draft(Integer scriptId, String body) throws Exception + { + return (service.saveDraft(everything(), scriptId, "msg " + body, source(body))); + } + + + + /*************************************************************************** + ** Check a draft with the standard entry point and declarations. + ***************************************************************************/ + BuildSummary build(Integer revisionId) throws Exception + { + return (service.check(everything(), revisionId, ENTRY, declared())); + } + + + + /*************************************************************************** + ** Draft, build, pass a recorded test, publish; returns the publication id. + ***************************************************************************/ + Integer publishNew(Integer scriptId, String body) throws Exception + { + BuildSummary built = build(draft(scriptId, body)); + assertTrue(built.success()); + assertTrue(service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{\"status\":\"NEW\"}", null).succeeded()); + return (service.publish(everything(), built.buildId(), "note")); + } + + + + /*************************************************************************** + ** Bind a publication to the hook table's insert event with a narrow grant. + ***************************************************************************/ + Integer bindHook(Integer publicationId, String... patchable) throws Exception + { + return (service.bind(everything(), new BindingSpec("po-hook", HOOK_TABLE, HookEvent.PRE_INSERT, publicationId, + List.of(new CapabilityDeclaration(HOOK_TABLE, Set.of("status"), 5, 1)), Set.of(patchable)))); + } + + + + /*************************************************************************** + ** All rows of a table, read as the current (view-permitted) session. + ***************************************************************************/ + List rows(String table) throws Exception + { + return (new QueryAction().execute(new QueryInput().withTableName(table)).getRecords()); + } +} + diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/SecurityRegressionTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/SecurityRegressionTest.java new file mode 100644 index 0000000..bfe7bb5 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/SecurityRegressionTest.java @@ -0,0 +1,281 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.capability.BrokerRequest; +import com.kingsrook.qbits.scripting.capability.QqqRecordReader; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.hooks.ScriptHookCustomizer; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BuildSummary; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecutionLimits; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessDecision; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessHelper; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessResolverMetaData; +import com.kingsrook.qqq.backend.core.actions.permissions.fieldaccess.FieldAccessTier; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Security regressions through real QQQ actions and explicit worker fixtures. + *******************************************************************************/ +class SecurityRegressionTest extends ScriptingTestBase +{ + /*************************************************************************** + ** Direct views and aggregates cannot expose inaccessible projects; disabling + ** one hook does not permit enabling two competing bindings afterwards. + ***************************************************************************/ + @Test + void genericViewsRespectProjectAndBindingEnableCannotConflict() throws Exception + { + Integer publication = publishNew(newProject(), "x"); + Integer first = bindHook(publication, "status"); + service.setBindingEnabled(everything(), first, false); + bindHook(publication, "status"); + assertKind(ScriptingException.Kind.INVALID, () -> service.setBindingEnabled(everything(), first, true)); + assertFalse(rows(ScriptingTables.EXECUTION).isEmpty()); + Integer buildId = rows(ScriptingTables.BUILD).get(0).getValueInteger("id"); + instance.getTable("script").setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + GetInput denied = new GetInput(); + denied.setTableName(ScriptingTables.BUILD); + denied.setPrimaryKey(buildId); + assertNull(new GetAction().execute(denied).getRecord()); + for(String table : List.of(ScriptingTables.BUILD, ScriptingTables.ARTIFACT, ScriptingTables.PUBLICATION, ScriptingTables.BINDING, ScriptingTables.EXECUTION)) + { + assertTrue(rows(table).isEmpty()); + CountInput input = new CountInput(); + input.setTableName(table); + assertThrows(QException.class, () -> new CountAction().execute(input)); + } + } + + + + /*************************************************************************** + ** Host-side callback and output budgets remain authoritative even if a + ** provider disregards the limits sent in its worker message. + ***************************************************************************/ + @Test + void hostEnforcesTotalCallsOutputAndUnexpectedProviderFailure() throws Exception + { + install(new ScriptingRuntime().withWorker(worker).withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN)) + .withRequiredQualificationId("qual-1").withLimits(new ExecutionLimits(2000, 1000, 64L * 1024 * 1024, 128 * 1024, 0))); + BuildSummary built = build(draft(newProject(), "x")); + AtomicReference response = new AtomicReference<>(); + worker.onExecute = broker -> response.set(broker.handle("{}")); + assertTrue(service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + assertEquals("LIMIT", new JSONObject(response.get()).getString("code")); + worker.onExecute = broker -> + { + throw new IllegalStateException("secret provider failure"); + }; + var failure = service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null); + assertEquals("UNAVAILABLE", failure.status()); + assertFalse(failure.message().contains("secret")); + worker.onExecute = null; + install(new ScriptingRuntime().withWorker(worker).withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN)) + .withRequiredQualificationId("qual-1").withLimits(new ExecutionLimits(2000, 1000, 64L * 1024 * 1024, 1, 0))); + BuildSummary small = build(draft(newProject(), "y")); + assertEquals("LIMIT", service.test(everything(), small.buildId(), HookEvent.PRE_INSERT, "{}", null).status()); + } + + + + /*************************************************************************** + ** A caller-supplied Actor cannot impersonate a user or retain revoked rights. + ***************************************************************************/ + @Test + void actorMustMatchCurrentSessionAndPermissions() throws Exception + { + Integer project = newProject(); + assertKind(ScriptingException.Kind.DENIED, () -> service.saveDraft(Actor.of("other", Set.of(ScriptingPermissions.DRAFT)), project, "", source("x"))); + Actor captured = everything(); + QContext.setQSession(sessionWith(ScriptingPermissions.VIEW)); + assertKind(ScriptingException.Kind.DENIED, () -> service.saveDraft(captured, project, "", source("x"))); + assertTrue(rows("scriptRevision").isEmpty()); + } + + + + /*************************************************************************** + ** Privileged source storage never bypasses the owning project's ACL. + ***************************************************************************/ + @Test + void deniedProjectBlocksDraftCheckTestAndPublish() throws Exception + { + Integer project = newProject(); + Integer revision = draft(project, "x"); + BuildSummary built = build(revision); + instance.getTable("script").setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + assertKind(ScriptingException.Kind.DENIED, () -> draft(project, "denied")); + assertKind(ScriptingException.Kind.DENIED, () -> build(revision)); + assertKind(ScriptingException.Kind.DENIED, () -> service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null)); + assertKind(ScriptingException.Kind.DENIED, () -> service.publish(everything(), built.buildId(), "denied")); + assertEquals(0, worker.executeCalls); + } + + + + /*************************************************************************** + ** Altering signed capability JSON cannot widen an accepted build. + ***************************************************************************/ + @Test + void tamperedCapabilityManifestIsRejectedBeforeWorker() throws Exception + { + BuildSummary built = build(draft(newProject(), "x")); + QRecord row = rows(ScriptingTables.BUILD).get(0); + String manifest = row.getValueString("manifest_json").replace("po_number", "amount"); + QSession original = QContext.getQSession(); + QContext.setQSession(new QSystemUserSession()); + try + { + new UpdateAction().execute(new UpdateInput().withTableName(ScriptingTables.BUILD).withRecords(List.of(new QRecord().withValue("id", built.buildId()).withValue("manifest_json", manifest)))); + } + finally + { + QContext.setQSession(original); + } + assertKind(ScriptingException.Kind.INTEGRITY, () -> service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null)); + assertKind(ScriptingException.Kind.INTEGRITY, () -> service.publish(everything(), built.buildId(), "tampered")); + assertEquals(0, worker.executeCalls); + } + + + + /*************************************************************************** + ** Core masking and denied filter policy apply even when the host's default + ** action permission enforcement is disabled. + ***************************************************************************/ + @Test + void readerHonoursMaskedHiddenAndFilterPolicies() throws Exception + { + new InsertAction().execute(new InsertInput().withTableName(HOOK_TABLE).withRecords(List.of(new QRecord().withValue("status", "SECRET-SENTINEL")))); + AtomicReference tier = new AtomicReference<>(FieldAccessTier.MASKED); + FieldAccessResolverMetaData.ofOrWithNew(instance).withResolverInstance((table, field) -> + new FieldAccessDecision(field.getName().equals("status") ? tier.get() : FieldAccessTier.FULL, null, "test")); + instance.setEnforcePermissionsInCoreActions(false); + QqqRecordReader reader = new QqqRecordReader(); + List> rows = reader.read(HOOK_TABLE, List.of("status"), List.of(), 1); + assertEquals(1, rows.size()); + assertNotEquals("SECRET-SENTINEL", rows.get(0).get("status")); + assertKind(ScriptingException.Kind.DENIED, () -> reader.read(HOOK_TABLE, List.of("id"), List.of(new BrokerRequest.Criterion("status", "EQUALS", List.of("SECRET-SENTINEL"))), 1)); + tier.set(FieldAccessTier.HIDDEN); + QContext.removeObject(FieldAccessHelper.CACHE_KEY); + assertKind(ScriptingException.Kind.DENIED, () -> reader.read(HOOK_TABLE, List.of("status"), List.of(), 1)); + } + + + + /*************************************************************************** + ** Raw hook records do not expose hidden or restricted values; changed field + ** permission is checked again before accepting a script patch. + ***************************************************************************/ + @Test + void hookFiltersPayloadAndRechecksPatchPolicy() throws Exception + { + bindHook(publishNew(newProject(), "x"), "status"); + instance.getTable(HOOK_TABLE).addField(new QFieldMetaData("hidden", QFieldType.STRING).withIsHidden(true)); + FieldAccessResolverMetaData.ofOrWithNew(instance).withResolverInstance((table, field) -> + new FieldAccessDecision(field.getName().equals("status") ? FieldAccessTier.MASKED : FieldAccessTier.FULL, null, "test")); + QContext.removeObject(FieldAccessHelper.CACHE_KEY); + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\"changed\"}}"; + QRecord proposed = new QRecord().withValue("status", "SECRET-SENTINEL").withValue("hidden", "HIDDEN-SENTINEL"); + new ScriptHookCustomizer().preInsert(new InsertInput().withTableName(HOOK_TABLE), List.of(proposed), false); + assertFalse(proposed.getErrors().isEmpty()); + assertEquals("SECRET-SENTINEL", proposed.getValueString("status")); + assertFalse(worker.lastExecuteRequest.contains("SECRET-SENTINEL")); + assertFalse(worker.lastExecuteRequest.contains("HIDDEN-SENTINEL")); + } + + + + /*************************************************************************** + ** Worker callbacks cannot inherit a different session or switch to another + ** thread with accidental ambient authority. + ***************************************************************************/ + @Test + void brokerCallbackIsBoundToInitiatingSessionAndThread() throws Exception + { + BuildSummary built = build(draft(newProject(), "x")); + AtomicReference otherThread = new AtomicReference<>(); + AtomicReference otherSession = new AtomicReference<>(); + worker.onExecute = broker -> + { + Thread thread = new Thread(() -> otherThread.set(broker.handle("{}"))); + thread.start(); + try + { + thread.join(); + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + QSession original = QContext.getQSession(); + QContext.setQSession(new QSystemUserSession()); + try + { + otherSession.set(broker.handle("{}")); + } + finally + { + QContext.setQSession(original); + } + }; + assertTrue(service.test(everything(), built.buildId(), HookEvent.PRE_INSERT, "{}", null).succeeded()); + assertEquals("DENIED", new JSONObject(otherThread.get()).getString("code")); + assertEquals("DENIED", new JSONObject(otherSession.get()).getString("code")); + } +} + + diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/SourceAndBuildTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/SourceAndBuildTest.java new file mode 100644 index 0000000..edcfa7a --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/SourceAndBuildTest.java @@ -0,0 +1,196 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.scripting.build.ArtifactSubmission; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.build.BuildOutcome; +import com.kingsrook.qbits.scripting.source.GitCommitRef; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Source snapshots and Git commit refs (immutability, hashing, policy) and the + ** trusted build coordinator (only attested, hash-matching artifacts verify). + *******************************************************************************/ +class SourceAndBuildTest +{ + private static final String SHA = "0123456789abcdef0123456789abcdef01234567"; + private static final byte[] OTHER_KEY = "ffffffffffffffffffffffffffffffff".getBytes(java.nio.charset.StandardCharsets.UTF_8); + + + + /*************************************************************************** + ** Identity is content-derived and independent of input order. + ***************************************************************************/ + @Test + void snapshotHashIsDeterministicAndContentSensitive() throws Exception + { + SourceFile a = new SourceFile("src/a.ts", "export const a = 1;"); + SourceFile b = new SourceFile("src/b.ts", "export const b = 2;"); + + SourceSnapshot one = SourceSnapshot.of("o", List.of(a, b)); + SourceSnapshot two = SourceSnapshot.of("o", List.of(b, a)); + SourceSnapshot changed = SourceSnapshot.of("o", List.of(a, new SourceFile("src/b.ts", "export const b = 3;"))); + + assertEquals(one.sha256(), two.sha256()); + assertNotEquals(one.sha256(), changed.sha256()); + assertTrue(one.verify()); + assertFalse(new SourceSnapshot("o", one.files(), "0".repeat(64)).verify()); + assertThrows(UnsupportedOperationException.class, () -> one.files().add(a)); + } + + + + /*************************************************************************** + ** Only plain relative .ts paths are accepted; tooling files are not source. + ***************************************************************************/ + @Test + void sourcePolicyRejectsUnsafePathsAndFiles() throws Exception + { + for(String bad : List.of("../x.ts", "/abs.ts", "a\\b.ts", "x.js", "package.json", "tsconfig.json", "a/../b.ts", "a//b.ts", ".hidden.ts", "a b.ts", "x.ts\0")) + { + assertThrows(IllegalArgumentException.class, () -> new SourceFile(bad, ""), bad); + } + assertThrows(IllegalArgumentException.class, () -> new SourceFile("a.ts", "x".repeat(SourceFile.MAX_CONTENT_BYTES + 1))); + assertThrows(IllegalArgumentException.class, () -> new SourceFile("a.ts", "nul\0byte")); + + assertKind(ScriptingException.Kind.INVALID, () -> SourceSnapshot.of("o", List.of(new SourceFile("A.ts", ""), new SourceFile("a.ts", "")))); + assertKind(ScriptingException.Kind.INVALID, () -> SourceSnapshot.of("o", List.of())); + List tooMany = new ArrayList<>(); + for(int i = 0; i <= SourceSnapshot.MAX_FILES; i++) + { + tooMany.add(new SourceFile("f" + i + ".ts", "")); + } + assertKind(ScriptingException.Kind.INVALID, () -> SourceSnapshot.of("o", tooMany)); + } + + + + /*************************************************************************** + ** An import names an exact commit: branches, tags and short SHAs cannot be + ** expressed. + ***************************************************************************/ + @Test + void gitRefsNeedAnExactCommit() throws Exception + { + assertEquals("git:acme/scripts@" + SHA + ":hooks/po", new GitCommitRef("acme/scripts", SHA, "hooks/po").origin()); + assertEquals("git:acme/scripts@" + SHA, new GitCommitRef("acme/scripts", SHA, null).origin()); + assertThrows(IllegalArgumentException.class, () -> new GitCommitRef("acme/scripts", "main", null)); + assertThrows(IllegalArgumentException.class, () -> new GitCommitRef("acme/scripts", SHA.substring(0, 12), null)); + assertThrows(IllegalArgumentException.class, () -> new GitCommitRef("acme/scripts", SHA.toUpperCase(java.util.Locale.ROOT), null)); + assertThrows(IllegalArgumentException.class, () -> new GitCommitRef("https://evil.example/x", SHA, null)); + assertThrows(IllegalArgumentException.class, () -> new GitCommitRef("acme/scripts", SHA, "../etc")); + } + + + + /*************************************************************************** + ** A genuine build verifies; the verdict carries a submission only on success. + ***************************************************************************/ + @Test + void coordinatorBuildVerifies() throws Exception + { + BuildCoordinator coordinator = new BuildCoordinator("coord-1", ScriptingTestBase.KEY, ScriptingTestBase.TOOLCHAIN); + SourceSnapshot snapshot = SourceSnapshot.of("t", ScriptingTestBase.source("x")); + + BuildOutcome outcome = coordinator.build(new FakeWorker(), snapshot, ScriptingTestBase.ENTRY); + + assertTrue(outcome.success()); + coordinator.verify(outcome.submission(), snapshot.sha256()); + assertThrows(IllegalArgumentException.class, () -> new BuildOutcome(true, List.of(), null)); + assertThrows(IllegalArgumentException.class, () -> new BuildOutcome(false, List.of(), outcome.submission())); + } + + + + /*************************************************************************** + ** Arbitrary JavaScript cannot be blessed as compiled: wrong or missing + ** attestation, other coordinator, other key, altered bytes, wrong source. + ***************************************************************************/ + @Test + void forgedOrAlteredArtifactsAreRejected() throws Exception + { + BuildCoordinator coordinator = new BuildCoordinator("coord-1", ScriptingTestBase.KEY, ScriptingTestBase.TOOLCHAIN); + SourceSnapshot snapshot = SourceSnapshot.of("t", ScriptingTestBase.source("x")); + ArtifactSubmission genuine = coordinator.build(new FakeWorker(), snapshot, ScriptingTestBase.ENTRY).submission(); + + String clientJs = "globalThis.owned = true;"; + ArtifactSubmission clientMade = new ArtifactSubmission("coord-1", snapshot.sha256(), ScriptingTestBase.TOOLCHAIN, ScriptingTestBase.ENTRY, clientJs, BuildCoordinator.sha256Hex(clientJs), null, "deadbeef"); + assertKind(ScriptingException.Kind.INTEGRITY, () -> coordinator.verify(clientMade, snapshot.sha256())); + + ArtifactSubmission swapped = new ArtifactSubmission("coord-1", snapshot.sha256(), ScriptingTestBase.TOOLCHAIN, ScriptingTestBase.ENTRY, clientJs, BuildCoordinator.sha256Hex(clientJs), null, genuine.attestation()); + assertKind(ScriptingException.Kind.INTEGRITY, () -> coordinator.verify(swapped, snapshot.sha256())); + + ArtifactSubmission wrongHash = new ArtifactSubmission("coord-1", snapshot.sha256(), ScriptingTestBase.TOOLCHAIN, ScriptingTestBase.ENTRY, clientJs, genuine.bundleSha256(), null, genuine.attestation()); + assertKind(ScriptingException.Kind.INTEGRITY, () -> coordinator.verify(wrongHash, snapshot.sha256())); + + ArtifactSubmission otherCoordinator = new ArtifactSubmission("coord-2", genuine.sourceSha256(), genuine.toolchainId(), genuine.entryPoint(), genuine.bundle(), genuine.bundleSha256(), null, genuine.attestation()); + assertKind(ScriptingException.Kind.INTEGRITY, () -> coordinator.verify(otherCoordinator, snapshot.sha256())); + + ArtifactSubmission otherKey = new BuildCoordinator("coord-1", OTHER_KEY, ScriptingTestBase.TOOLCHAIN).build(new FakeWorker(), snapshot, ScriptingTestBase.ENTRY).submission(); + assertKind(ScriptingException.Kind.INTEGRITY, () -> coordinator.verify(otherKey, snapshot.sha256())); + + assertKind(ScriptingException.Kind.INTEGRITY, () -> coordinator.verify(genuine, "f".repeat(64))); + assertKind(ScriptingException.Kind.INTEGRITY, () -> coordinator.verify(null, snapshot.sha256())); + coordinator.verify(genuine, snapshot.sha256()); + } + + + + /*************************************************************************** + ** The coordinator refuses a worker bundle whose hash it cannot reproduce, and + ** refuses weak configuration. + ***************************************************************************/ + @Test + void coordinatorDoesNotTrustWorkerHashesOrWeakKeys() throws Exception + { + FakeWorker lying = new FakeWorker() + { + /*************************************************************************** + ** Answer with a bundle whose claimed hash is wrong. + ***************************************************************************/ + @Override + public String compile(String compileRequestJson) throws ScriptingException + { + return (super.compile(compileRequestJson).replaceAll("\"bundleSha256\":\"[0-9a-f]{64}\"", "\"bundleSha256\":\"" + "0".repeat(64) + "\"")); + } + }; + BuildCoordinator coordinator = new BuildCoordinator("coord-1", ScriptingTestBase.KEY, ScriptingTestBase.TOOLCHAIN); + SourceSnapshot snapshot = SourceSnapshot.of("t", ScriptingTestBase.source("x")); + + assertKind(ScriptingException.Kind.INTEGRITY, () -> coordinator.build(lying, snapshot, ScriptingTestBase.ENTRY)); + assertKind(ScriptingException.Kind.INVALID, () -> coordinator.build(new FakeWorker(), snapshot, "src/missing.ts")); + assertThrows(IllegalArgumentException.class, () -> new BuildCoordinator("coord-1", new byte[8], ScriptingTestBase.TOOLCHAIN)); + assertThrows(IllegalArgumentException.class, () -> new BuildCoordinator("", ScriptingTestBase.KEY, ScriptingTestBase.TOOLCHAIN)); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/TestAssertions.java b/base/src/test/java/com/kingsrook/qbits/scripting/TestAssertions.java new file mode 100644 index 0000000..b4150b5 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/TestAssertions.java @@ -0,0 +1,50 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import org.junit.jupiter.api.function.Executable; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + + +/******************************************************************************* + ** Shared assertion for the scripting failure taxonomy. + *******************************************************************************/ +final class TestAssertions +{ + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private TestAssertions() + { + } + + + + /*************************************************************************** + ** The executable must throw a ScriptingException of exactly this kind. + ***************************************************************************/ + static void assertKind(ScriptingException.Kind kind, Executable executable) + { + assertEquals(kind, assertThrows(ScriptingException.class, executable).getKind()); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/WarmRuntimeServiceTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/WarmRuntimeServiceTest.java new file mode 100644 index 0000000..4854a1e --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/WarmRuntimeServiceTest.java @@ -0,0 +1,917 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qbits.scripting.service.WarmPublicationSource; +import com.kingsrook.qbits.scripting.service.WarmReconciliationConfig; +import com.kingsrook.qbits.scripting.service.WarmRuntimeProbe; +import com.kingsrook.qbits.scripting.service.WarmRuntimeService; +import com.kingsrook.qbits.scripting.worker.WarmWorkerMaintenance; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import com.kingsrook.qqq.backend.core.serviceidentity.QServicePrincipalMetaData; +import com.kingsrook.qqq.backend.core.serviceidentity.ServiceIdentitiesMetaData; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Native metadata/lifecycle and real authoring pins with protocol-only warming. + ** Memory persistence and a fake transport do not qualify a deployed worker. + *******************************************************************************/ +class WarmRuntimeServiceTest extends ScriptingTestBase +{ + private MaintenanceWorker warm; + private WarmRuntimeService lifecycle; + private QServicePrincipalMetaData principal; + + /*************************************************************************** + ** Producer sees the maintenance capability before registering native hooks. + ***************************************************************************/ + @BeforeEach + void installMaintenance() throws Exception + { + warm = new MaintenanceWorker(); + install(new ScriptingRuntime().withWorker(warm).withCoordinator(new BuildCoordinator("coord-1", KEY, TOOLCHAIN)) + .withRequiredQualificationId("qual-1").withWarmReconciliation(policy(10, 1024 * 1024))); + worker = warm; + principal = new QServicePrincipalMetaData().withName("script-cache").withLocalPermissions(ScriptingPermissions.WARM, ScriptingPermissions.VIEW); + ServiceIdentitiesMetaData.ofOrWithNew(instance).withPrincipal(principal); + } + + /*************************************************************************** + ** Every test closes only its native service/provider, never a global executor. + ***************************************************************************/ + @AfterEach + void stopMaintenance() + { + warm.release.countDown(); + if(lifecycle != null) + { + lifecycle.stop(); + lifecycle.afterApplicationStop(); + } + } + + /*************************************************************************** + ** Explicit long interval lets tests trigger subsequent identical cold passes. + ***************************************************************************/ + private static WarmReconciliationConfig policy(int count, long bytes) + { + return (new WarmReconciliationConfig("script-cache", Set.of(HOOK_TABLE), count, bytes, + Duration.ofSeconds(300), Duration.ofSeconds(5), Duration.ofSeconds(600))); + } + + /*************************************************************************** + ** An immutable publication bound through normal authorized owner services. + ***************************************************************************/ + private Integer activeBinding() throws Exception + { + return (bindHook(publishNew(newProject(), "warm lifecycle"))); + } + + /*************************************************************************** + ** Start the same no-argument implementation used by QQQ runtime code loading. + ***************************************************************************/ + private void start() throws Exception + { + warm.ready.clear(); + warm.preparations.set(0); + warm.entered = new CountDownLatch(1); + lifecycle = new WarmRuntimeService(); + lifecycle.start(instance); + } + + /*************************************************************************** + ** Native health entry point resolves the instance-owned lifecycle object. + ***************************************************************************/ + private HealthStatus health() + { + return (new WarmRuntimeService().run(new HealthCheckInput().withQInstance(instance) + .withDeadline(Instant.now().plusSeconds(2))).getStatus()); + } + + /*************************************************************************** + ** Bounded synchronization on actual protocol/health observations. + ***************************************************************************/ + private static void await(BooleanSupplier condition) throws Exception + { + long stop = System.nanoTime() + Duration.ofSeconds(4).toNanos(); + while(!condition.getAsBoolean() && System.nanoTime() < stop) + { + Thread.sleep(10); + } + assertTrue(condition.getAsBoolean()); + } + + /*************************************************************************** + ** Startup restores old bindings without their author or any record data. + ***************************************************************************/ + @Test + void nativeStartupRestoresPinsAndRegistersReadiness() throws Exception + { + activeBinding(); + var caller = QContext.getQSession(); + caller.withBackendVariants(java.util.Map.of("memory", "caller-variant")); + int executions = warm.executeCalls; + runtime.withReader((table, fields, criteria, limit) -> + { + throw new AssertionError("Maintenance must not read records"); + }); + assertTrue(instance.getRuntimeServices().stream().anyMatch(ref -> WarmRuntimeService.class.getName().equals(ref.getName()))); + assertTrue(HealthChecksMetaData.ofOrWithNew(instance).getChecks().stream().anyMatch(check -> "scripting.warm".equals(check.getName()) && check.getIncludeInReadiness())); + assertEquals(HealthStatus.DOWN, health()); + start(); + await(() -> health() == HealthStatus.UP); + assertEquals(1, warm.preparations.get()); + assertEquals("service:script-cache", warm.lastActor); + assertNull(warm.lastVariants); + assertEquals(executions, warm.executeCalls); + assertSame(caller, QContext.getQSession()); + assertEquals(ENTRY, warm.last.entryPoint()); + assertEquals(TOOLCHAIN, warm.last.toolchainId()); + assertEquals(BuildCoordinator.sha256Hex(warm.last.bundle()), warm.last.sha256()); + } + + /*************************************************************************** + ** Readiness notices lost cache; next cold pass restores exact code only. + ***************************************************************************/ + @Test + void reconnectRestoresCodeWithoutExecutionOrRepeatedPrepare() throws Exception + { + activeBinding(); + start(); + await(() -> health() == HealthStatus.UP); + int before = warm.preparations.get(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(before, warm.preparations.get()); + warm.ready.clear(); + assertEquals(HealthStatus.DOWN, health()); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(before + 1, warm.preparations.get()); + assertEquals(HealthStatus.UP, health()); + } + + /*************************************************************************** + ** Native metadata permission is re-minted; old author rights never substitute. + ***************************************************************************/ + @Test + void revokedPrincipalAndProjectReadDenyWithoutPreparation() throws Exception + { + activeBinding(); + start(); + await(() -> health() == HealthStatus.UP); + int before = warm.preparations.get(); + principal.setLocalPermissions(Set.of(ScriptingPermissions.VIEW)); + assertEquals(HealthStatus.DOWN, health()); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(before, warm.preparations.get()); + principal.setLocalPermissions(Set.of(ScriptingPermissions.VIEW, ScriptingPermissions.WARM)); + instance.getTable("script").withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + assertEquals(HealthStatus.DOWN, health()); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(before, warm.preparations.get()); + } + + /*************************************************************************** + ** Missing explicit authority configuration never means an empty ready cache. + ***************************************************************************/ + @Test + void missingPolicyFailsClosed() throws Exception + { + runtime.withWarmReconciliation(null); + start(); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, warm.heartbeats.get()); + assertEquals(0, warm.preparations.get()); + assertThrows(IllegalArgumentException.class, () -> new WarmReconciliationConfig("script-cache", Set.of(), 1, 100, + Duration.ofSeconds(1), Duration.ofSeconds(1), Duration.ofSeconds(5))); + } + + /*************************************************************************** + ** Exact immutable pins are reverified before cold transfer and health success. + ***************************************************************************/ + @Test + void tamperedSignedArtifactIsNeverTransferred() throws Exception + { + activeBinding(); + start(); + await(() -> health() == HealthStatus.UP); + QRecord artifact = rows(ScriptingTables.ARTIFACT).getFirst(); + MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.ARTIFACT) + .withRecords(List.of(new QRecord().withValue("id", artifact.getValueInteger("id")).withValue("bundle", "tampered"))), false); + int before = warm.preparations.get(); + warm.ready.clear(); + assertEquals(HealthStatus.DOWN, health()); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(before, warm.preparations.get()); + } + + /*************************************************************************** + ** A callback cannot change scoped binding grants then publish READY status. + ***************************************************************************/ + @Test + void changedBindingDuringPreparationRefusesReadiness() throws Exception + { + Integer binding = activeBinding(); + warm.callback = () -> MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.BINDING) + .withRecords(List.of(new QRecord().withValue("id", binding).withValue("grants_json", "changed"))), false); + start(); + await(() -> warm.preparations.get() == 1); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(1, warm.preparations.get()); + } + + /*************************************************************************** + ** Changed session scope is refused and the caller context is fully restored. + ***************************************************************************/ + @Test + void callbackScopeMutationCannotBecomeMetadataAuthority() throws Exception + { + activeBinding(); + start(); + await(() -> health() == HealthStatus.UP); + var caller = QContext.getQSession(); + warm.ready.clear(); + warm.callback = () -> QContext.getQSession().withSecurityKeyValue("broader", true); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertSame(caller, QContext.getQSession()); + assertFalse(caller.hasSecurityKeyValue("broader", true)); + } + + /*************************************************************************** + ** A finite code budget refuses before transferring even one oversized bundle. + ***************************************************************************/ + @Test + void byteBudgetRefusesBeforeTransfer() throws Exception + { + activeBinding(); + runtime.withWarmReconciliation(policy(10, 1)); + start(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, warm.preparations.get()); + } + + /*************************************************************************** + ** Non-overlap and stop are real scheduler properties, not an executor mock. + ***************************************************************************/ + @Test + void blockedPassCoalescesAndStopKeepsReadinessDown() throws Exception + { + activeBinding(); + warm.block = true; + start(); + assertTrue(warm.entered.await(2, TimeUnit.SECONDS)); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(1, warm.preparations.get()); + lifecycle.stop(); + warm.release.countDown(); + assertEquals(HealthStatus.DOWN, health()); + assertFalse(warm.closed); + lifecycle.afterApplicationStop(); + assertTrue(warm.closed); + } + + /*************************************************************************** + ** More required rows than the explicit limit cannot become partial readiness. + ***************************************************************************/ + @Test + void inventoryOverflowRefusesBeforeColdTransfer() throws Exception + { + activeBinding(); + runtime.withWarmReconciliation(policy(1, 1024 * 1024)); + QRecord duplicate = new QRecord(rows(ScriptingTables.BINDING).getFirst()); + duplicate.setValue("id", 999); + duplicate.setValue("active_slot", "fixture-second-slot"); + MemoryRecordStore.getInstance().insert(new InsertInput().withTableName(ScriptingTables.BINDING).withRecords(List.of(duplicate)), false); + start(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, warm.preparations.get()); + } + + /*************************************************************************** + ** A declared-principal permission mutation during a cold callback is refused. + ***************************************************************************/ + @Test + void declarationRevocationDuringPrepareRefusesReadiness() throws Exception + { + activeBinding(); + start(); + await(() -> health() == HealthStatus.UP); + warm.ready.clear(); + warm.callback = () -> principal.setLocalPermissions(Set.of(ScriptingPermissions.VIEW)); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + } + + /*************************************************************************** + ** A typo in explicit target scope must not look like an empty ready inventory. + ***************************************************************************/ + @Test + void absentTargetTableCannotReportReady() throws Exception + { + runtime.withWarmReconciliation(new WarmReconciliationConfig("script-cache", Set.of("missing_table"), 10, 1024, + Duration.ofSeconds(300), Duration.ofSeconds(5), Duration.ofSeconds(600))); + start(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, warm.preparations.get()); + assertEquals(0, warm.heartbeats.get()); + } + + /*************************************************************************** + ** Even an empty inventory needs an explicitly declared metadata principal. + ***************************************************************************/ + @Test + void undeclaredPrincipalCannotReportReady() throws Exception + { + runtime.withWarmReconciliation(new WarmReconciliationConfig("absent-principal", Set.of(HOOK_TABLE), 10, 1024, + Duration.ofSeconds(300), Duration.ofSeconds(5), Duration.ofSeconds(600))); + start(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, warm.preparations.get()); + assertEquals(0, warm.heartbeats.get()); + } + + /*************************************************************************** + ** Final local readiness callbacks cannot change metadata after its last read. + ***************************************************************************/ + @Test + void readinessCallbackPinMutationRefusesHealthAndReconciliation() throws Exception + { + Integer binding = activeBinding(); + start(); + await(() -> health() == HealthStatus.UP); + String original = rows(ScriptingTables.BINDING).getFirst().getValueString("grants_json"); + AtomicInteger calls = new AtomicInteger(); + warm.readinessCallback = () -> + { + calls.incrementAndGet(); + MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.BINDING) + .withRecords(List.of(new QRecord().withValue("id", binding).withValue("grants_json", "changed-by-ready"))), false); + }; + assertEquals(HealthStatus.DOWN, health()); + assertEquals(1, calls.get()); + MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.BINDING) + .withRecords(List.of(new QRecord().withValue("id", binding).withValue("grants_json", original))), false); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(2, calls.get()); + assertEquals(HealthStatus.DOWN, health()); + } + + /*************************************************************************** + ** An unbound publication qualified by the exact CONTEXT test lifecycle. + ***************************************************************************/ + private WarmPublicationSource.Pin contextPublication(String body) throws Exception + { + var built = build(draft(newProject(), body)); + warm.hookJson = "{\"data\":{\"fixture\":true}}"; + assertTrue(service.testContext(everything(), built.buildId(), "{}", List.of()).succeeded()); + return (warmPin(service.publish(everything(), built.buildId(), "context fixture"))); + } + + /*************************************************************************** + ** Capture the consumer reference under authoring authority before maintenance. + ***************************************************************************/ + private WarmPublicationSource.Pin warmPin(Integer publicationId) throws ScriptingException + { + var pin = service.validatePublishedContext(everything(), publicationId); + return (new WarmPublicationSource.Pin(publicationId, pin.artifactId(), pin.bundleSha256())); + } + + /*************************************************************************** + ** Restart inventories unbound contexts with code-only service authority; + ** execution and health do not prepare or call the inventory on hot dispatch. + ***************************************************************************/ + @Test + void contextStartupRestoresUnboundPublicationWithoutHotPreparation() throws Exception + { + WarmPublicationSource.Pin publication = contextPublication("context restart"); + var pin = service.selectPublishedContext(everything(), publication.publicationId()).pin(); + AtomicInteger inventories = new AtomicInteger(); + runtime.withWarmPublicationSource((tables, limit) -> + { + assertEquals(Set.of(HOOK_TABLE), tables); + assertEquals(10, limit); + assertEquals("service:script-cache", QContext.getQSession().getUser().getIdReference()); + assertFalse(QContext.getQSession().hasPermission(ScriptingPermissions.EXECUTE)); + inventories.incrementAndGet(); + return (List.of(publication)); + }); + int executions = warm.executeCalls; + start(); + await(() -> health() == HealthStatus.UP); + assertEquals(1, warm.preparations.get()); + assertEquals(executions, warm.executeCalls); + warm.requirePrepared = true; + int reads = inventories.get(); + assertTrue(service.executePublishedContext(everything(), pin, "{}", List.of()).succeeded()); + assertEquals(reads, inventories.get()); + assertEquals(1, warm.preparations.get()); + warm.ready.clear(); + assertFalse(service.executePublishedContext(everything(), pin, "{}", List.of()).succeeded()); + assertEquals(reads, inventories.get()); + assertEquals(1, warm.preparations.get()); + assertEquals(HealthStatus.DOWN, health()); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.UP, health()); + assertEquals(2, warm.preparations.get()); + } + + /*************************************************************************** + ** Current inventory changes invalidate readiness and removed pins are no + ** longer required; a changed pin during transfer cannot publish readiness. + ***************************************************************************/ + @Test + void contextInventoryChangesAndRemovalAreReconciled() throws Exception + { + WarmPublicationSource.Pin first = contextPublication("first context"); + WarmPublicationSource.Pin second = contextPublication("second context"); + AtomicReference> ids = new AtomicReference<>(List.of(first)); + runtime.withWarmPublicationSource((tables, limit) -> ids.get()); + start(); + await(() -> health() == HealthStatus.UP); + ids.set(List.of(second)); + assertEquals(HealthStatus.DOWN, health()); + warm.callback = () -> ids.set(List.of(first)); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + warm.callback = null; + ids.set(List.of()); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.UP, health()); + } + + /*************************************************************************** + ** Malformed/overflow inventories fail before any additional cold transfer. + ***************************************************************************/ + @Test + void malformedContextInventoryCannotBecomePartialReadiness() throws Exception + { + WarmPublicationSource.Pin publication = contextPublication("malformed context list"); + AtomicReference> ids = new AtomicReference<>(List.of(publication)); + runtime.withWarmPublicationSource((tables, limit) -> ids.get()); + start(); + await(() -> health() == HealthStatus.UP); + int before = warm.preparations.get(); + assertThrows(IllegalArgumentException.class, () -> new WarmPublicationSource.Pin(0, 1, "0".repeat(64))); + assertThrows(IllegalArgumentException.class, () -> new WarmPublicationSource.Pin(1, -1, "0".repeat(64))); + assertThrows(IllegalArgumentException.class, () -> new WarmPublicationSource.Pin(1, 1, "invalid")); + for(List invalid : List.of(List.of(publication, publication), java.util.Arrays.asList((WarmPublicationSource.Pin) null), + List.of(new WarmPublicationSource.Pin(publication.publicationId(), 999999, publication.bundleSha256())), + List.of(new WarmPublicationSource.Pin(publication.publicationId(), publication.artifactId(), "0".repeat(64))), + List.of(new WarmPublicationSource.Pin(999999, publication.artifactId(), publication.bundleSha256())))) + { + ids.set(invalid); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + } + ids.set(null); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(before, warm.preparations.get()); + } + + /*************************************************************************** + ** A hook-only publication cannot qualify as a context; missing signed code + ** and lost project visibility remain fail-closed on each current read. + ***************************************************************************/ + @Test + void contextRequiresSuccessfulExactTestAndCurrentProjectVisibility() throws Exception + { + WarmPublicationSource.Pin publication = warmPin(publishNew(newProject(), "hook only")); + runtime.withWarmPublicationSource((tables, limit) -> List.of(publication)); + start(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, warm.preparations.get()); + Integer buildId = rows(ScriptingTables.PUBLICATION).getFirst().getValueInteger("build_id"); + warm.hookJson = "{\"data\":{}}"; + assertTrue(service.testContext(everything(), buildId, "{}", List.of()).succeeded()); + await(() -> + { + WarmRuntimeProbe.reconcile(lifecycle); + return (health() == HealthStatus.UP); + }); + int before = warm.preparations.get(); + instance.getTable("script").withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS)); + warm.ready.clear(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(before, warm.preparations.get()); + } + + /*************************************************************************** + ** Context pins do not bypass HMAC validation when cached code is lost. + ***************************************************************************/ + @Test + void contextTamperedCodeRefusesBeforeTransfer() throws Exception + { + WarmPublicationSource.Pin publication = contextPublication("context tamper"); + runtime.withWarmPublicationSource((tables, limit) -> List.of(publication)); + start(); + await(() -> health() == HealthStatus.UP); + QRecord artifact = rows(ScriptingTables.ARTIFACT).getFirst(); + MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.ARTIFACT) + .withRecords(List.of(new QRecord().withValue("id", artifact.getValueInteger("id")).withValue("bundle", "tampered"))), false); + warm.ready.clear(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(1, warm.preparations.get()); + } + + /*************************************************************************** + ** Both inventories share a finite count, including when they share code. + ***************************************************************************/ + @Test + void contextAndHookShareCombinedCountBudget() throws Exception + { + WarmPublicationSource.Pin publication = contextPublication("shared count"); + bindHook(publication.publicationId()); + runtime.withWarmReconciliation(policy(1, 1024 * 1024)); + AtomicInteger offered = new AtomicInteger(-1); + runtime.withWarmPublicationSource((tables, limit) -> + { + offered.set(limit); + return (List.of(publication)); + }); + start(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, offered.get()); + assertEquals(0, warm.preparations.get()); + } + + /*************************************************************************** + ** Shared artifact code is counted/transferred once across hooks and contexts. + ***************************************************************************/ + @Test + void contextAndHookDeduplicateCodeWithinByteBudget() throws Exception + { + WarmPublicationSource.Pin publication = contextPublication("shared code"); + bindHook(publication.publicationId()); + long bytes = rows(ScriptingTables.ARTIFACT).getFirst().getValueString("bundle").getBytes(java.nio.charset.StandardCharsets.UTF_8).length; + runtime.withWarmReconciliation(policy(2, bytes)); + runtime.withWarmPublicationSource((tables, limit) -> List.of(publication)); + start(); + await(() -> health() == HealthStatus.UP); + assertEquals(1, warm.preparations.get()); + } + + /*************************************************************************** + ** Inventory callbacks cannot widen authority or replace their own source. + ***************************************************************************/ + @Test + void contextInventoryAuthorityMutationAndReplacementAreRefused() throws Exception + { + WarmPublicationSource.Pin publication = contextPublication("callback mutation"); + AtomicInteger mutation = new AtomicInteger(); + runtime.withWarmPublicationSource((tables, limit) -> + { + if(mutation.get() == 1) + { + QContext.getQSession().withSecurityKeyValue("broader", true); + } + else if(mutation.get() == 2) + { + runtime.withWarmPublicationSource((scope, bound) -> List.of()); + } + return (List.of(publication)); + }); + start(); + await(() -> health() == HealthStatus.UP); + var caller = QContext.getQSession(); + mutation.set(1); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertSame(caller, QContext.getQSession()); + assertFalse(caller.hasSecurityKeyValue("broader", true)); + mutation.set(2); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(1, warm.preparations.get()); + } + + /*************************************************************************** + ** A missing publication artifact is never substituted by another build. + ***************************************************************************/ + @Test + void contextMissingArtifactCannotBePrepared() throws Exception + { + WarmPublicationSource.Pin publication = contextPublication("missing artifact"); + MemoryRecordStore.getInstance().update(new UpdateInput().withTableName(ScriptingTables.PUBLICATION) + .withRecords(List.of(new QRecord().withValue("id", publication.publicationId()).withValue("artifact_id", 999999))), false); + runtime.withWarmPublicationSource((tables, limit) -> List.of(publication)); + start(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, warm.preparations.get()); + } + + /*************************************************************************** + ** Distinct hook and context bundles consume one aggregate byte budget before + ** any transfer, even when each individual bundle would fit independently. + ***************************************************************************/ + @Test + void contextAndHookCannotExceedCombinedByteBudget() throws Exception + { + activeBinding(); + WarmPublicationSource.Pin publication = contextPublication("separate context code"); + long bytes = rows(ScriptingTables.ARTIFACT).stream() + .mapToLong(row -> row.getValueString("bundle").getBytes(java.nio.charset.StandardCharsets.UTF_8).length).sum(); + runtime.withWarmReconciliation(policy(2, bytes - 1)); + runtime.withWarmPublicationSource((tables, limit) -> List.of(publication)); + start(); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(0, warm.preparations.get()); + } + + /*************************************************************************** + ** Supplier outages are uncertainty, not an empty inventory; final aggregate + ** callbacks cannot change the required context pins and still report UP. + ***************************************************************************/ + @Test + void contextSupplierFailureAndFinalPinMutationRefuseReadiness() throws Exception + { + WarmPublicationSource.Pin publication = contextPublication("final inventory check"); + AtomicReference> ids = new AtomicReference<>(List.of(publication)); + AtomicInteger fail = new AtomicInteger(); + runtime.withWarmPublicationSource((tables, limit) -> + { + if(fail.get() != 0) + { + throw new QException("Fixture metadata unavailable"); + } + return (ids.get()); + }); + start(); + await(() -> health() == HealthStatus.UP); + fail.set(1); + assertEquals(HealthStatus.DOWN, health()); + fail.set(0); + assertEquals(HealthStatus.UP, health()); + warm.readinessCallback = () -> ids.set(List.of()); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(1, warm.preparations.get()); + } + + /*************************************************************************** + ** Retirement is external cold work: changed pins or widened authority must + ** be refused before subsequent code transfer or readiness publication. + ***************************************************************************/ + @Test + void retirementCallbackCannotChangeAuthorityOrInventory() throws Exception + { + WarmPublicationSource.Pin first = contextPublication("retire first"); + WarmPublicationSource.Pin second = contextPublication("retire second"); + AtomicReference> required = new AtomicReference<>(List.of(first)); + runtime.withWarmPublicationSource((tables, limit) -> required.get()); + start(); + await(() -> health() == HealthStatus.UP); + warm.retirementCallback = () -> required.set(List.of(second)); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(1, warm.preparations.get()); + warm.retirementCallback = () -> QContext.getQSession().withSecurityKeyValue("changed", true); + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.DOWN, health()); + assertEquals(1, warm.preparations.get()); + warm.retirementCallback = null; + WarmRuntimeProbe.reconcile(lifecycle); + assertEquals(HealthStatus.UP, health()); + assertEquals(2, warm.preparations.get()); + } + + /*************************************************************************** + ** Test-only native mutation without executing script text. + ***************************************************************************/ + @FunctionalInterface + private interface Callback + { + /*********************************************************************** + ** Apply a deterministic host callback. + ***********************************************************************/ + void run() throws Exception; + } + + /*************************************************************************** + ** The skip window never exceeds a quarter of readiness age and always leaves + ** one interval plus pass budget before acknowledgment age can expire. + ***************************************************************************/ + @Test + void heartbeatSkipWindowIsBoundedByReadinessPolicy() + { + java.util.function.BiFunction window = (maxAge, pass) -> new WarmReconciliationConfig("p", Set.of("t"), 1, 1, + Duration.ofSeconds(1), Duration.ofSeconds(pass), Duration.ofSeconds(maxAge)).heartbeatSkipWindow(); + assertEquals(Duration.ofSeconds(4), window.apply(35, 30)); + assertEquals(Duration.ofSeconds(150), window.apply(600, 30)); + assertEquals(Duration.ZERO, window.apply(31, 30)); + } + + /*************************************************************************** + ** A protocol-only warm cache with no worker evaluation or networking. + ***************************************************************************/ + private static class MaintenanceWorker extends FakeWorker implements WarmWorkerMaintenance + { + private final Set ready = ConcurrentHashMap.newKeySet(); + private final AtomicInteger preparations = new AtomicInteger(); + private final AtomicInteger heartbeats = new AtomicInteger(); + private volatile CountDownLatch entered = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + private volatile Callback callback; + private volatile Callback readinessCallback; + private volatile Callback retirementCallback; + private volatile boolean block; + private volatile boolean closed; + private volatile boolean requirePrepared; + private volatile Artifact last; + private volatile String lastActor; + private volatile Object lastVariants; + + /*********************************************************************** + ** Optional cache-miss refusal verifies that actual service execution has + ** no hidden prepare/retry path. No guest code is evaluated by this fixture. + ***********************************************************************/ + @Override + public String execute(String request, BrokerChannel broker) throws ScriptingException + { + if(requirePrepared && !ready.contains(new JSONObject(request).getString("artifactSha256"))) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Fixture code is cold"); + } + return (super.execute(request, broker)); + } + + /*********************************************************************** + ** User authoring cold seam; tests clear this capacity before native start. + ***********************************************************************/ + @Override + public void prepareVerified(Artifact artifact) throws ScriptingException + { + prepareVerified(artifact, Duration.ofSeconds(5)); + } + + /*********************************************************************** + ** Only immutable code and an explicit finite deadline enter preparation. + ***********************************************************************/ + @Override + public void prepareVerified(Artifact artifact, Duration budget) throws ScriptingException + { + last = artifact; + lastActor = QContext.getQSession().getUser().getIdReference(); + lastVariants = QContext.getQSession().getBackendVariants(); + preparations.incrementAndGet(); + entered.countDown(); + if(block) + { + try + { + if(!release.await(budget.toMillis(), TimeUnit.MILLISECONDS)) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Fixture deadline"); + } + } + catch(InterruptedException failure) + { + Thread.currentThread().interrupt(); + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Fixture interrupted"); + } + } + if(callback != null) + { + try + { + callback.run(); + } + catch(Exception failure) + { + throw new AssertionError(failure); + } + } + ready.add(artifact.sha256()); + } + + /*********************************************************************** + ** Heartbeat is code-free and never increments execution calls. + ***********************************************************************/ + @Override + public void heartbeat(Duration budget) + { + heartbeats.incrementAndGet(); + } + + /*********************************************************************** + ** Fixed protocol-only retirement; tests can mutate host policy at its edge. + ***********************************************************************/ + @Override + public void retireUnused(List required, Duration budget) + { + if(retirementCallback != null) + { + try + { + retirementCallback.run(); + } + catch(Exception failure) + { + throw new AssertionError(failure); + } + } + ready.retainAll(required.stream().map(CodeIdentity::sha256).toList()); + } + + /*********************************************************************** + ** Fixture current cache status; authoritative transport tests are separate. + ***********************************************************************/ + @Override + public boolean isPrepared(String hash, String entry, String toolchain, Duration maxAge) + { + return (!closed && ENTRY.equals(entry) && TOOLCHAIN.equals(toolchain) && ready.contains(hash)); + } + + /*********************************************************************** + ** One fixture snapshot of the required set; optional hostile host callback. + ***********************************************************************/ + @Override + public boolean arePrepared(List identities, Duration maxAge) + { + if(readinessCallback != null) + { + try + { + readinessCallback.run(); + } + catch(Exception failure) + { + throw new AssertionError(failure); + } + } + Set snapshot = Set.copyOf(ready); + return (!closed && identities.stream().allMatch(identity -> ENTRY.equals(identity.entryPoint()) + && TOOLCHAIN.equals(identity.toolchainId()) && snapshot.contains(identity.sha256()))); + } + + /*********************************************************************** + ** Instance-owned shutdown only. + ***********************************************************************/ + @Override + public void close() + { + closed = true; + ready.clear(); + } + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/WorkerProtocolTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/WorkerProtocolTest.java new file mode 100644 index 0000000..b97dd06 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/WorkerProtocolTest.java @@ -0,0 +1,210 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.util.Set; +import com.kingsrook.qbits.scripting.hooks.HookResult; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecuteRequest; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecuteResponse; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecutionLimits; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.Outcome; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import static com.kingsrook.qbits.scripting.TestAssertions.assertKind; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The worker protocol is bounded in size, depth, shape and vocabulary, and has + ** no field that could carry a credential. + *******************************************************************************/ +class WorkerProtocolTest +{ + private static final String V = WorkerProtocol.VERSION; + private static final String OK_RESULT = "{\"protocol\":\"" + V + "\",\"type\":\"executeResult\",\"requestId\":\"x-1\",\"outcome\":\"OK\",\"result\":{\"action\":\"ALLOW\"}}"; + + + + /*************************************************************************** + ** A broker request with the given operation. + ***************************************************************************/ + private static String broker(String operation, String fieldsJson, String criteriaJson) + { + return ("{\"protocol\":\"" + V + "\",\"type\":\"broker\",\"requestId\":\"r1\",\"operation\":\"" + operation + "\",\"token\":\"t\",\"table\":\"purchase_order\",\"fields\":" + + fieldsJson + ",\"criteria\":" + criteriaJson + ",\"limit\":5}"); + } + + + + /*************************************************************************** + ** A well-formed answer decodes. + ***************************************************************************/ + @Test + void wellFormedExecuteResultDecodes() throws Exception + { + ExecuteResponse response = WorkerProtocol.decodeExecuteResponse(OK_RESULT); + assertEquals("x-1", response.requestId()); + assertEquals(Outcome.OK, response.outcome()); + assertEquals(HookResult.Action.ALLOW, response.hook().action()); + } + + + + /*************************************************************************** + ** Size is checked before parsing, so a huge body is refused as too large. + ***************************************************************************/ + @Test + void oversizedMessageIsRefusedBeforeParsing() throws Exception + { + String huge = "x".repeat(WorkerProtocol.MAX_RESULT_MESSAGE_BYTES + 1); + assertKind(ScriptingException.Kind.LIMIT, () -> WorkerProtocol.decodeExecuteResponse(huge)); + assertKind(ScriptingException.Kind.LIMIT, () -> WorkerProtocol.decodeBrokerRequest("y".repeat(WorkerProtocol.MAX_BROKER_MESSAGE_BYTES + 1))); + assertKind(ScriptingException.Kind.LIMIT, () -> WorkerProtocol.decodeExecuteResponse(null)); + } + + + + /*************************************************************************** + ** Pathological nesting is refused without recursing. + ***************************************************************************/ + @Test + void deeplyNestedMessageIsRefused() throws Exception + { + String nested = "{\"a\":".repeat(50) + "1" + "}".repeat(50); + assertKind(ScriptingException.Kind.LIMIT, () -> WorkerProtocol.decodeExecuteResponse(nested)); + } + + + + /*************************************************************************** + ** Wrong version, wrong type, garbage and unknown keys all fail closed with a + ** generic message that does not echo the input. + ***************************************************************************/ + @Test + void wrongVersionTypeGarbageAndUnknownKeysAreRefused() throws Exception + { + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeExecuteResponse(OK_RESULT.replace(V, "qbit-scripting-worker/2"))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeExecuteResponse(OK_RESULT.replace("executeResult", "compileResult"))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeExecuteResponse(OK_RESULT.replace("\"outcome\"", "\"credentials\":\"s3cr3t\",\"outcome\""))); + ScriptingException garbage = assertThrows(ScriptingException.class, () -> WorkerProtocol.decodeExecuteResponse("{not json s3cr3t")); + assertEquals(ScriptingException.Kind.INVALID, garbage.getKind()); + assertTrue(!garbage.getMessage().contains("s3cr3t")); + } + + + + /*************************************************************************** + ** A hook decision must be one of the known shapes with scalar patch values. + ***************************************************************************/ + @Test + void malformedHookDecisionsAreRefused() throws Exception + { + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeExecuteResponse(OK_RESULT.replace("ALLOW", "PATCH"))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeExecuteResponse(OK_RESULT.replace("{\"action\":\"ALLOW\"}", "{\"action\":\"ALLOW\",\"patch\":{\"a\":1}}"))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeExecuteResponse(OK_RESULT.replace("{\"action\":\"ALLOW\"}", "{\"action\":\"PATCH\",\"patch\":{\"a;b\":1}}"))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeExecuteResponse(OK_RESULT.replace("{\"action\":\"ALLOW\"}", "{\"action\":\"PATCH\",\"patch\":{\"a\":[1]}}"))); + } + + + + /*************************************************************************** + ** A successful compile answer needs a bounded bundle and its hash; a failure + ** may not smuggle a bundle. + ***************************************************************************/ + @Test + void compileAnswersAreShapeChecked() throws Exception + { + String head = "{\"protocol\":\"" + V + "\",\"type\":\"compileResult\",\"requestId\":\"b-1\",\"strict\":true,\"toolchainId\":\"t1\","; + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeCompileResponse(head + "\"success\":true}")); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeCompileResponse(head + "\"success\":true,\"bundle\":\"x\",\"bundleSha256\":\"nothex\"}")); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeCompileResponse(head + "\"success\":false,\"bundle\":\"x\"}")); + assertEquals(0, WorkerProtocol.decodeCompileResponse(head + "\"success\":false}").diagnostics().size()); + } + + + + /*************************************************************************** + ** The execute request has exactly the contract's keys: no credential, secret + ** or session field exists, only an opaque capability token. + ***************************************************************************/ + @Test + void executeRequestHasOnlyContractKeys() throws Exception + { + String json = WorkerProtocol.encodeExecute(new ExecuteRequest("x-1", "a-1", "ab".repeat(32), "bundle", "src/index.ts", "PRE_INSERT", "{\"proposed\":{\"a\":1}}", "qct_opaque", ExecutionLimits.hookDefaults())); + + assertEquals(Set.of("protocol", "type", "requestId", "artifactId", "artifactSha256", "bundle", "entryPoint", "event", "payload", "capabilityToken", "limits"), new JSONObject(json).keySet()); + assertEquals(Set.of("wallMillis", "cpuMillis", "memoryBytes", "outputBytes", "brokerCalls"), new JSONObject(json).getJSONObject("limits").keySet()); + } + + + + /*************************************************************************** + ** Oversized or non-object input is refused on the way out. + ***************************************************************************/ + @Test + void outboundInputIsBounded() throws Exception + { + ExecutionLimits limits = ExecutionLimits.hookDefaults(); + String bigPayload = "{\"a\":\"" + "x".repeat(WorkerProtocol.MAX_PAYLOAD_BYTES) + "\"}"; + assertKind(ScriptingException.Kind.LIMIT, () -> WorkerProtocol.encodeExecute(new ExecuteRequest("x-1", "a-1", "h", "b", "e", "PRE_INSERT", bigPayload, "t", limits))); + assertKind(ScriptingException.Kind.LIMIT, () -> WorkerProtocol.encodeExecute(new ExecuteRequest("x-1", "a-1", "h", "b".repeat(WorkerProtocol.MAX_BUNDLE_CHARS + 1), "e", "PRE_INSERT", "{}", "t", limits))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.encodeExecute(new ExecuteRequest("x-1", "a-1", "h", "b", "e", "PRE_INSERT", "[1,2]", "t", limits))); + assertKind(ScriptingException.Kind.LIMIT, () -> WorkerProtocol.checkPayload("[".repeat(40) + "]".repeat(40))); + } + + + + /*************************************************************************** + ** Limits above the absolute maxima cannot be constructed. + ***************************************************************************/ + @Test + void executionLimitsAreClamped() throws Exception + { + assertThrows(IllegalArgumentException.class, () -> new ExecutionLimits(ExecutionLimits.MAX_WALL_MILLIS + 1, 1, 1, 1, 1)); + assertThrows(IllegalArgumentException.class, () -> new ExecutionLimits(1, 1, ExecutionLimits.MAX_MEMORY_BYTES + 1, 1, 1)); + assertThrows(IllegalArgumentException.class, () -> new ExecutionLimits(1, 1, 1, 1, ExecutionLimits.MAX_BROKER_CALLS + 1)); + assertThrows(IllegalArgumentException.class, () -> new ExecutionLimits(0, 1, 1, 1, 1)); + } + + + + /*************************************************************************** + ** Worker read requests: only a bounded query with known operators and plain + ** names is representable. + ***************************************************************************/ + @Test + void brokerRequestsAreRestrictedToBoundedQueries() throws Exception + { + assertEquals(1, WorkerProtocol.decodeBrokerRequest(broker("query", "[\"po_number\"]", "[{\"field\":\"status\",\"operator\":\"EQUALS\",\"values\":[\"OPEN\"]}]")).criteria().size()); + + assertKind(ScriptingException.Kind.DENIED, () -> WorkerProtocol.decodeBrokerRequest(broker("delete", "[\"po_number\"]", "[]"))); + assertKind(ScriptingException.Kind.DENIED, () -> WorkerProtocol.decodeBrokerRequest(broker("query", "[\"po_number\"]", "[{\"field\":\"status\",\"operator\":\"LIKE\",\"values\":[\"%\"]}]"))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeBrokerRequest(broker("query", "[\"po_number; drop\"]", "[]"))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeBrokerRequest(broker("query", "[]", "[]"))); + assertKind(ScriptingException.Kind.INVALID, () -> WorkerProtocol.decodeBrokerRequest(broker("query", "[\"a\"]", "[{\"field\":\"status\",\"operator\":\"EQUALS\",\"values\":[{\"x\":1}]}]"))); + String many = "[" + "{\"field\":\"status\",\"operator\":\"EQUALS\",\"values\":[1]},".repeat(11) + "{\"field\":\"status\",\"operator\":\"EQUALS\",\"values\":[1]}]"; + assertKind(ScriptingException.Kind.LIMIT, () -> WorkerProtocol.decodeBrokerRequest(broker("query", "[\"a\"]", many))); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/hooks/DeleteHookSnapshotsTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/hooks/DeleteHookSnapshotsTest.java new file mode 100644 index 0000000..38e8b47 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/hooks/DeleteHookSnapshotsTest.java @@ -0,0 +1,57 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.hooks; + + +import java.io.Serializable; +import java.util.Date; +import java.util.List; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.worker.WorkerProtocol; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Byte quotas include hidden primary keys, with tiny deterministic fixtures. + *******************************************************************************/ +class DeleteHookSnapshotsTest +{ + /*************************************************************************** + ** Scalar keys count toward quota even when absent from the projected JSON. + ***************************************************************************/ + @Test + void keysAreBoundedScalarsAndIncludedInAggregateBudget() throws Exception + { + assertEquals(5, DeleteHookSnapshots.addBytes(0, "a", "{}")); + for(Serializable key : List.of("a", 1, 2L, true, 1.5)) + { + assertTrue(DeleteHookSnapshots.addBytes(0, key, "{}") > 2); + } + for(Serializable key : List.of(new byte[] { 1 }, new Date(0), Double.NaN, Float.POSITIVE_INFINITY, "x".repeat(1025))) + { + assertThrows(ScriptingException.class, () -> DeleteHookSnapshots.addBytes(0, key, "{}")); + } + assertThrows(ScriptingException.class, () -> DeleteHookSnapshots.addBytes(WorkerProtocol.MAX_PAYLOAD_BYTES - 3, "a", "{}")); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/service/WarmRuntimeProbe.java b/base/src/test/java/com/kingsrook/qbits/scripting/service/WarmRuntimeProbe.java new file mode 100644 index 0000000..f06ea15 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/service/WarmRuntimeProbe.java @@ -0,0 +1,41 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.service; + + +/******************************************************************************* + ** Test-only access to a single coalesced pass; no new public production control. + *******************************************************************************/ +public final class WarmRuntimeProbe +{ + /*************************************************************************** + ** No state or instances. + ***************************************************************************/ + private WarmRuntimeProbe() {} + + /*************************************************************************** + ** Use the same implementation as the native scheduled future. + ***************************************************************************/ + public static void reconcile(WarmRuntimeService service) + { + service.reconcile(); + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/worker/PersistentWorkerProviderTest.java b/base/src/test/java/com/kingsrook/qbits/scripting/worker/PersistentWorkerProviderTest.java new file mode 100644 index 0000000..57b1a3b --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/worker/PersistentWorkerProviderTest.java @@ -0,0 +1,1102 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + +import javax.net.ssl.SSLContext; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.worker.PersistentWorkerProvider.Configuration; +import com.kingsrook.qbits.scripting.worker.PersistentWorkerProvider.Endpoint; +import com.kingsrook.qbits.scripting.worker.PersistentWorkerProvider.PreparedArtifact; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecuteRequest; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecutionLimits; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/******************************************************************************* + ** Actual mutual-TLS exchanges with a fixed trusted supervisor fixture. No user + ** code, isolation qualification, runtime selection or latency claim is involved. + *******************************************************************************/ +class PersistentWorkerProviderTest +{ + private static SSLContext identity; + private static final PreparedArtifact ARTIFACT = artifact("trusted-fixture-code"); + @TempDir + static Path directory; + + /*************************************************************************** + ** One ephemeral identity shared only by the trusted loopback test peers. + ***************************************************************************/ + @BeforeAll + static void identity() throws Exception + { + identity = SupervisorTlsFixture.identity(directory); + } + + /*************************************************************************** + ** Two executions share a prepared connection but fresh tokens and request IDs. + ***************************************************************************/ + @Test + void sequentialInvocationsReuseOnlyPreparedCode() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 2, 2, 4096, Duration.ofSeconds(5))) + { + assertThrows(ScriptingException.class, () -> provider.execute(request("first", "token-one", ARTIFACT), message -> "{}")); + assertEquals(0, server.accepted.get(), "Hot cache miss must not connect"); + provider.prepare(ARTIFACT, 1); + provider.prepare(ARTIFACT, 1); + Thread caller = Thread.currentThread(); + AtomicInteger calls = new AtomicInteger(); + for(String id : List.of("first", "second")) + { + String token = "token-" + id; + String result = provider.execute(request(id, token, ARTIFACT), message -> + { + assertEquals(caller, Thread.currentThread()); + assertEquals(token, new JSONObject(message).getString("token")); + calls.incrementAndGet(); + return (WorkerProtocol.encodeBrokerResponse(List.of(), false)); + }); + assertEquals(id, WorkerProtocol.decodeExecuteResponse(result).requestId()); + } + assertEquals(2, calls.get()); + assertEquals(1, server.accepted.get()); + assertEquals(1, server.requests.stream().filter(frame -> "PREPARE".equals(frame.opt("type"))).count()); + assertEquals(2, server.requests.stream().filter(frame -> "INVOKE".equals(frame.opt("type"))).count()); + assertEquals("fixture-evidence", provider.qualificationId()); + assertEquals(Duration.ofSeconds(5), provider.executionTransportBudget()); + server.healthy(); + } + } + + /*************************************************************************** + ** The authoring SPI uses operator copies/profile and sends no invocation data. + ***************************************************************************/ + @Test + void authoringPreparationUsesBoundedOperatorCopies() throws Exception + { + try(var server = new SupervisorTlsFixture(identity)) + { + Endpoint endpoint = new Endpoint("localhost", server.server.getLocalPort()); + Configuration config = new Configuration(identity, endpoint, endpoint, "fixture", "profile", "fixture-toolchain", + "fixture-evidence", "fixture-evidence", Duration.ofSeconds(5), 2, 1, 2, 4096); + assertThrows(IllegalArgumentException.class, () -> new PersistentWorkerProvider(config, 0)); + assertThrows(IllegalArgumentException.class, () -> new PersistentWorkerProvider(config, 3)); + try(var provider = new PersistentWorkerProvider(config, 2)) + { + WarmWorkerPreparation.Artifact wrong = new WarmWorkerPreparation.Artifact(ARTIFACT.sha256(), ARTIFACT.entryPoint(), ARTIFACT.bundle(), "wrong"); + assertThrows(ScriptingException.class, () -> provider.prepareVerified(wrong)); + assertEquals(0, server.accepted.get()); + provider.prepareVerified(new WarmWorkerPreparation.Artifact(ARTIFACT.sha256(), ARTIFACT.entryPoint(), ARTIFACT.bundle(), "fixture-toolchain")); + assertEquals(2, server.accepted.get()); + var preparations = server.requests.stream().filter(frame -> "PREPARE".equals(frame.opt("type"))).toList(); + assertEquals(2, preparations.size()); + for(JSONObject frame : preparations) + { + assertFalse(frame.toString().contains("capabilityToken")); + assertFalse(frame.has("payload")); + assertEquals(ARTIFACT.entryPoint(), frame.getString("entryPoint")); + assertEquals(ARTIFACT.sha256(), frame.getString("sha256")); + } + assertTrue(server.requests.stream().filter(frame -> "HELLO".equals(frame.opt("type"))).allMatch(frame -> "profile".equals(frame.opt("profile")))); + server.healthy(); + } + } + } + + /*************************************************************************** + ** A busy slot causes immediate admission refusal without queue or eviction. + ***************************************************************************/ + @Test + void activeSlotAndColdCacheBoundsArePreserved() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5)); + var callers = Executors.newSingleThreadExecutor()) + { + provider.prepare(ARTIFACT, 1); + server.fault = "BLOCK"; + var active = callers.submit(() -> provider.execute(request("active", "token-a", ARTIFACT), message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + String malformed = "{\"protocol\":\"wrong\",\"type\":\"execute\"}"; + assertEquals(ScriptingException.Kind.UNAVAILABLE, + assertThrows(ScriptingException.class, () -> provider.execute(malformed, message -> "{}")).getKind(), "Admission must precede parsing"); + assertThrows(ScriptingException.class, () -> provider.execute(request("other", "token-b", ARTIFACT), message -> "{}")); + assertThrows(ScriptingException.class, () -> provider.prepare(artifact("different"), 1)); + assertEquals(1, server.accepted.get()); + server.continueInvocation.countDown(); + assertEquals("active", WorkerProtocol.decodeExecuteResponse(active.get(3, TimeUnit.SECONDS)).requestId()); + assertEquals(ScriptingException.Kind.INVALID, + assertThrows(ScriptingException.class, () -> provider.execute(malformed, message -> "{}")).getKind(), "Returned permit now admits validation"); + assertEquals("after-invalid", WorkerProtocol.decodeExecuteResponse(provider.execute(request("after-invalid", "token", ARTIFACT), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))).requestId(), "Validation failure must release its permit"); + assertThrows(ScriptingException.class, () -> provider.prepare(artifact("different"), 1), "Full entry cache cannot silently evict"); + assertEquals(1, server.accepted.get()); + } + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 1, Duration.ofSeconds(5))) + { + assertThrows(ScriptingException.class, () -> provider.prepare(ARTIFACT, 1)); + assertEquals(0, server.accepted.get()); + } + } + + /*************************************************************************** + ** Full obsolete caches recover only on an explicit cold maintenance pass; + ** hot misses neither retire, reconnect nor prepare their replacement. + ***************************************************************************/ + @Test + void coldRetirementReclaimsFullHistoricalCapacity() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5))) + { + var next = artifact("next-required-code"); + provider.prepare(ARTIFACT, 1); + assertThrows(ScriptingException.class, () -> provider.prepare(next, 1)); + provider.retireUnused(List.of(code(next)), Duration.ofSeconds(2)); + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofMinutes(1))); + assertThrows(ScriptingException.class, () -> provider.execute(request("cold", "token", next), message -> "{}")); + assertEquals(1, server.accepted.get()); + provider.prepare(next, 1); + assertEquals(2, server.accepted.get()); + assertTrue(provider.arePrepared(List.of(code(next)), Duration.ofMinutes(1))); + assertEquals("new", WorkerProtocol.decodeExecuteResponse(provider.execute(request("new", "token", next), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))).requestId()); + server.healthy(); + } + } + + /*************************************************************************** + ** Retiring a mixed epoch invalidates its required entries too; the lifecycle + ** must reprepare all of them before it can report aggregate readiness. + ***************************************************************************/ + @Test + void mixedEpochRequiresReprepareAndUnchangedInventoryDoesNotChurn() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + provider.prepare(artifact("historical-code"), 1); + provider.retireUnused(List.of(code(ARTIFACT)), Duration.ofSeconds(2)); + assertFalse(provider.arePrepared(List.of(code(ARTIFACT)), Duration.ofMinutes(1))); + provider.prepare(ARTIFACT, 1); + assertTrue(provider.arePrepared(List.of(code(ARTIFACT)), Duration.ofMinutes(1))); + int connections = server.accepted.get(); + provider.retireUnused(List.of(code(ARTIFACT)), Duration.ofSeconds(2)); + assertTrue(provider.arePrepared(List.of(code(ARTIFACT)), Duration.ofMinutes(1))); + assertEquals(connections, server.accepted.get()); + provider.retireUnused(List.of(), Duration.ofSeconds(2)); + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofMinutes(1))); + server.healthy(); + } + } + + /*************************************************************************** + ** A concurrent valid invocation owns its session until normal retirement; + ** later cold reclamation can replace that now-idle obsolete epoch. + ***************************************************************************/ + @Test + void coldRetirementNeverStealsBusyInvocation() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5)); + var callers = Executors.newSingleThreadExecutor()) + { + var next = artifact("replacement-after-busy"); + provider.prepare(ARTIFACT, 1); + server.fault = "BLOCK"; + var active = callers.submit(() -> provider.execute(request("busy", "token", ARTIFACT), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + provider.retireUnused(List.of(code(next)), Duration.ofSeconds(2)); + assertThrows(ScriptingException.class, () -> provider.prepare(next, 1)); + assertEquals(1, server.accepted.get()); + server.continueInvocation.countDown(); + assertEquals("busy", WorkerProtocol.decodeExecuteResponse(active.get(3, TimeUnit.SECONDS)).requestId()); + server.fault = "NONE"; + provider.retireUnused(List.of(code(next)), Duration.ofSeconds(2)); + provider.prepare(next, 1); + assertTrue(provider.arePrepared(List.of(code(next)), Duration.ofMinutes(1))); + assertEquals(2, server.accepted.get()); + server.healthy(); + } + } + + /*************************************************************************** + ** An invalid maintenance request cannot discard valid existing capacity. + ***************************************************************************/ + @Test + void invalidRetirementPolicyDoesNotRemoveCode() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + assertThrows(ScriptingException.class, () -> provider.retireUnused(List.of(code(ARTIFACT)), Duration.ZERO)); + assertThrows(ScriptingException.class, () -> provider.retireUnused( + List.of(new WarmWorkerMaintenance.CodeIdentity(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "wrong")), Duration.ofSeconds(1))); + assertTrue(provider.arePrepared(List.of(code(ARTIFACT)), Duration.ofMinutes(1))); + assertEquals(1, server.accepted.get()); + } + } + + /*************************************************************************** + ** A maintenance pass between cold preparation and use cannot retire the + ** authoring code. Closing the bounded lease releases capacity exactly once. + ***************************************************************************/ + @Test + void authoringLeaseProtectsPreparedCodeUntilUseFinishes() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5))) + { + var code = new WarmWorkerPreparation.Artifact(ARTIFACT.sha256(), ARTIFACT.entryPoint(), ARTIFACT.bundle(), "fixture-toolchain"); + var lease = provider.prepareForUse(code); + try + { + provider.retireUnused(List.of(), Duration.ofSeconds(2)); + assertTrue(provider.arePrepared(List.of(code(ARTIFACT)), Duration.ofMinutes(1))); + assertThrows(ScriptingException.class, () -> provider.prepareForUse(code)); + assertEquals("leased", WorkerProtocol.decodeExecuteResponse(provider.execute(request("leased", "token", ARTIFACT), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))).requestId()); + assertEquals(1, server.accepted.get()); + } + finally + { + lease.close(); + } + lease.close(); + provider.retireUnused(List.of(), Duration.ofSeconds(2)); + assertFalse(provider.arePrepared(List.of(code(ARTIFACT)), Duration.ofMinutes(1))); + try(var nextLease = provider.prepareForUse(code)) + { + assertThrows(ScriptingException.class, () -> provider.prepareForUse(code)); + assertTrue(provider.arePrepared(List.of(code(ARTIFACT)), Duration.ofMinutes(1))); + } + assertEquals(2, server.accepted.get()); + server.healthy(); + } + } + + /*************************************************************************** + ** Failed cold admission releases its use permit and protection without a + ** hot retry; an explicit later maintenance/preparation can recover. + ***************************************************************************/ + @Test + void failedAuthoringPreparationDoesNotLeakUseLease() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + var next = artifact("next-after-failed-preparation"); + var code = new WarmWorkerPreparation.Artifact(next.sha256(), next.entryPoint(), next.bundle(), "fixture-toolchain"); + assertThrows(ScriptingException.class, () -> provider.prepareForUse(code)); + provider.retireUnused(List.of(code(next)), Duration.ofSeconds(2)); + try(var lease = provider.prepareForUse(code)) + { + assertTrue(provider.arePrepared(List.of(code(next)), Duration.ofMinutes(1))); + } + provider.retireUnused(List.of(), Duration.ofSeconds(2)); + assertFalse(provider.arePrepared(List.of(code(next)), Duration.ofMinutes(1))); + server.healthy(); + } + } + + /*************************************************************************** + + ** Code identity excludes bundle bytes and invocation credentials. + + ***************************************************************************/ + private static WarmWorkerMaintenance.CodeIdentity code(PreparedArtifact artifact) + { + return (new WarmWorkerMaintenance.CodeIdentity(artifact.sha256(), artifact.entryPoint(), "fixture-toolchain")); + } + + /*************************************************************************** + ** Identity, framing and retirement faults discard readiness without replay. + ***************************************************************************/ + @Test + void invalidTranscriptsAreNeverReusedOrRetried() throws Exception + { + for(String fault : List.of("TOKEN", "BROKER_ID", "SEQUENCE", "RESULT", "RETIRE", "LATE_BROKER", "LENGTH", "UTF8")) + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + server.fault = fault; + AtomicInteger calls = new AtomicInteger(); + assertThrows(ScriptingException.class, () -> provider.execute(request("bad", "token", ARTIFACT), message -> + { + calls.incrementAndGet(); + return (WorkerProtocol.encodeBrokerResponse(List.of(), false)); + }), fault); + assertThrows(ScriptingException.class, () -> provider.execute(request("again", "token", ARTIFACT), message -> "{}")); + assertEquals(1, server.accepted.get(), fault + " must not reconnect automatically"); + assertEquals(List.of("RESULT", "RETIRE", "LATE_BROKER").contains(fault) ? 1 : 0, calls.get()); + server.healthy(); + } + } + } + + /*************************************************************************** + ** Handshake/ready identity cannot be replaced by matching only a digest. + ***************************************************************************/ + @Test + void coldIdentityAndEntryPointAreExact() throws Exception + { + for(String fault : List.of("HELLO", "PROFILE", "NAMESPACE", "TOOLCHAIN", "READY")) + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(5))) + { + server.fault = fault; + assertThrows(ScriptingException.class, () -> provider.prepare(ARTIFACT, 1)); + assertThrows(ScriptingException.class, () -> provider.execute(request("miss", "token", ARTIFACT), message -> "{}")); + assertEquals(1, server.accepted.get()); + } + } + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + PreparedArtifact differentEntry = new PreparedArtifact(ARTIFACT.sha256(), "src/other.ts", ARTIFACT.bundle()); + assertThrows(ScriptingException.class, () -> provider.execute(request("miss", "token", differentEntry), message -> "{}")); + provider.prepare(differentEntry, 1); + assertEquals(2, server.requests.stream().filter(frame -> "PREPARE".equals(frame.opt("type"))).count()); + assertEquals(1, server.accepted.get()); + } + } + + /*************************************************************************** + ** Read completion and broker return remain subject to whole-call deadline. + ***************************************************************************/ + @Test + void resultWithoutRetirementAndLateBrokerCannotSucceed() throws Exception + { + for(boolean lateBroker : List.of(false, true)) + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(1))) + { + provider.prepare(ARTIFACT, 1); + server.fault = lateBroker ? "NONE" : "NO_RETIRE"; + AtomicInteger callbacks = new AtomicInteger(); + assertThrows(ScriptingException.class, () -> provider.execute(request("deadline", "token", ARTIFACT), message -> + { + callbacks.incrementAndGet(); + if(lateBroker) + { + try + { + new java.util.concurrent.CountDownLatch(1).await(1300, TimeUnit.MILLISECONDS); + } + catch(InterruptedException failure) + { + Thread.currentThread().interrupt(); + } + } + return (WorkerProtocol.encodeBrokerResponse(List.of(), false)); + })); + assertEquals(1, callbacks.get()); + assertThrows(ScriptingException.class, () -> provider.execute(request("lost", "token", ARTIFACT), message -> "{}")); + assertEquals(1, server.accepted.get()); + } + } + } + + /*************************************************************************** + ** A dequeued timer from an ended operation cannot kill the next operation. + ***************************************************************************/ + @Test + void staleDeadlineCallbackCannotCloseReusedSession() throws Exception + { + List callbacks = new CopyOnWriteArrayList<>(); + try(var scheduler = new ScheduledThreadPoolExecutor(1) + { + /******************************************************************** + ** Capture the actual production callback, scheduling it far in future. + ********************************************************************/ + @Override + public ScheduledFuture schedule(Runnable command, long delay, TimeUnit unit) + { + callbacks.add(command); + return (super.schedule(command, 1, TimeUnit.DAYS)); + } + }) + { + scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + Configuration config = new Configuration(identity, new Endpoint("localhost", 1), null, "fixture", "profile", "fixture-toolchain", + "fixture-evidence", "fixture-evidence", Duration.ofSeconds(5), 1, 1, 1, 4096); + try(var connection = new SupervisorConnection(config)) + { + connection.begin(scheduler); + connection.end(); + connection.begin(scheduler); + callbacks.getFirst().run(); + connection.check(); + callbacks.getLast().run(); + assertThrows(ScriptingException.class, connection::check); + connection.end(); + } + } + } + + /*************************************************************************** + ** Bounded cold admission cannot create a second preparation or compilation. + ***************************************************************************/ + @Test + void coldAdmissionRefusesRatherThanQueues() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 2, 2, 4096, Duration.ofSeconds(5)); + var callers = Executors.newSingleThreadExecutor()) + { + server.fault = "PREPARE_BLOCK"; + var active = callers.submit(() -> + { + provider.prepare(ARTIFACT, 1); + return (true); + }); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + assertThrows(ScriptingException.class, () -> provider.prepare(ARTIFACT, 2)); + assertThrows(ScriptingException.class, () -> provider.compile("{}")); + assertEquals(1, server.accepted.get()); + server.continueInvocation.countDown(); + assertTrue(active.get(3, TimeUnit.SECONDS)); + } + } + + /*************************************************************************** + ** Caller interruption aborts the exchange, preserves its flag and loses slot. + ***************************************************************************/ + @Test + void interruptedCallerCannotReturnCapacityOrSuccess() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + server.fault = "BLOCK"; + AtomicInteger refused = new AtomicInteger(); + Thread caller = Thread.ofPlatform().start(() -> + { + try + { + provider.execute(request("interrupt", "token", ARTIFACT), message -> "{}"); + } + catch(ScriptingException expected) + { + if(Thread.currentThread().isInterrupted()) + { + refused.incrementAndGet(); + } + } + }); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + caller.interrupt(); + caller.join(1500); + assertFalse(caller.isAlive()); + assertEquals(1, refused.get()); + assertThrows(ScriptingException.class, () -> provider.execute(request("after", "token", ARTIFACT), message -> "{}")); + } + } + + /*************************************************************************** + ** Compilation uses a separate cold connection, leaving prepared slots ready. + ***************************************************************************/ + @Test + void compileAndExplicitCopiesDoNotConsumeHotSlots() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 2, 2, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 2); + String compile = new JSONObject().put("protocol", WorkerProtocol.VERSION).put("type", "compile").put("requestId", "compile") + .put("toolchainId", "fixture-toolchain").put("strict", true).put("entryPoint", "src/index.ts") + .put("sourceSha256", "1".repeat(64)).put("files", List.of()).toString(); + assertFalse(WorkerProtocol.decodeCompileResponse(provider.compile(compile)).success()); + assertEquals(3, server.accepted.get()); + assertEquals("warm", WorkerProtocol.decodeExecuteResponse(provider.execute(request("warm", "token", ARTIFACT), message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))).requestId()); + assertEquals(3, server.accepted.get()); + } + } + + /*************************************************************************** + ** Trust and host verification remain mandatory, including operator contexts. + ***************************************************************************/ + @Test + void tlsTrustAndHostnameAreEnforced() throws Exception + { + try(var server = new SupervisorTlsFixture(identity, false); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(2))) + { + assertThrows(ScriptingException.class, () -> provider.prepare(ARTIFACT, 1)); + assertTrue(server.requests.isEmpty(), "No namespace/code may be sent without negotiated client identity"); + } + try(var server = new SupervisorTlsFixture(identity)) + { + Endpoint address = new Endpoint("localhost", server.server.getLocalPort()); + Configuration config = new Configuration(SSLContext.getDefault(), address, null, "fixture", "profile", "fixture-toolchain", + "fixture-evidence", "fixture-evidence", Duration.ofSeconds(2), 1, 1, 2, 4096); + try(var provider = new PersistentWorkerProvider(config)) + { + assertThrows(ScriptingException.class, () -> provider.prepare(ARTIFACT, 1)); + } + Configuration wrongHost = new Configuration(identity, new Endpoint("127.0.0.1", server.server.getLocalPort()), null, + "fixture", "profile", "fixture-toolchain", "fixture-evidence", "fixture-evidence", Duration.ofSeconds(2), 1, 1, 2, 4096); + try(var provider = new PersistentWorkerProvider(wrongHost)) + { + assertThrows(ScriptingException.class, () -> provider.prepare(ARTIFACT, 1)); + } + } + } + + /*************************************************************************** + ** Closed schemas, integer limits and configuration refuse before any network. + ***************************************************************************/ + @Test + void localContractValidationIsBoundedAndClosed() throws Exception + { + assertThrows(IllegalArgumentException.class, () -> new Endpoint("https://localhost/path", 443)); + assertThrows(IllegalArgumentException.class, () -> new PreparedArtifact("0".repeat(64), "src/index.ts", "wrong")); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.parse("{".repeat(17))); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.parse("invalid")); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.parse("{\"x\":1} trailing")); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.parse("{\"x\":1,\"x\":2}")); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.parse("{x:1}")); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.parse("{'x':1}")); + assertEquals("[\\\"]", SupervisorProtocol.parse("{\"x\":\"[\\\\\\\"]\"}").getString("x")); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.identity(new JSONObject().put("id", 2), "id")); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.object(new JSONObject().put("id", "{}"), "id")); + assertThrows(ScriptingException.class, () -> SupervisorProtocol.keys(new JSONObject().put("x", true), "y")); + assertFalse(ARTIFACT.toString().contains(ARTIFACT.bundle())); + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(5))) + { + JSONObject extra = new JSONObject(request("bad", "token", ARTIFACT)).put("url", "private"); + assertThrows(ScriptingException.class, () -> provider.execute(extra.toString(), message -> "{}")); + JSONObject coercion = new JSONObject(request("bad", "token", ARTIFACT)); + coercion.getJSONObject("limits").put("brokerCalls", -4294967296L); + assertThrows(ScriptingException.class, () -> provider.execute(coercion.toString(), message -> "{}")); + provider.close(); + assertThrows(ScriptingException.class, () -> provider.prepare(ARTIFACT, 1)); + assertThrows(ScriptingException.class, () -> provider.execute(request("closed", "token", ARTIFACT), message -> "{}")); + assertEquals(0, server.accepted.get()); + } + } + + /*************************************************************************** + ** Standard immutable worker input, including existing host-side bundle check. + ***************************************************************************/ + private static String request(String id, String token, PreparedArtifact artifact) throws ScriptingException + { + return (WorkerProtocol.encodeExecute(new ExecuteRequest(id, "artifact", artifact.sha256(), artifact.bundle(), artifact.entryPoint(), + "PRE_INSERT", "{\"proposed\":{}}", token, ExecutionLimits.hookDefaults()))); + } + + /*************************************************************************** + ** Small synthetic immutable code; no engine consumes it in these tests. + ***************************************************************************/ + private static PreparedArtifact artifact(String bundle) + { + return (new PreparedArtifact(BuildCoordinator.sha256Hex(bundle), "src/index.ts", bundle)); + } + + /*************************************************************************** + ** The fixture evidence label is test-only, never deployment qualification. + ***************************************************************************/ + private static PersistentWorkerProvider provider(SupervisorTlsFixture server, int slots, int entries, long bytes, Duration timeout) + { + Endpoint endpoint = new Endpoint("localhost", server.server.getLocalPort()); + return (new PersistentWorkerProvider(new Configuration(identity, endpoint, endpoint, "fixture", "profile", "fixture-toolchain", + "fixture-evidence", "fixture-evidence", timeout, slots, 1, entries, bytes))); + } + + + + /*************************************************************************** + ** Idle heartbeat refreshes the same cache epoch with no artifact or token. + ***************************************************************************/ + @Test + void maintenanceKeepsExactReadyEpochWithoutPreparingAgain() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 2, 2, 4096, Duration.ofSeconds(5))) + { + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + provider.prepare(ARTIFACT, 1); + provider.heartbeat(Duration.ofSeconds(2)); + assertTrue(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "wrong", Duration.ofSeconds(5))); + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofNanos(1))); + assertEquals(1, server.accepted.get()); + assertEquals(1, server.requests.stream().filter(frame -> "PREPARE".equals(frame.optString("type"))).count()); + assertEquals(1, server.requests.stream().filter(frame -> "PING".equals(frame.optString("type"))).count()); + assertEquals("after-ping", WorkerProtocol.decodeExecuteResponse(provider.execute(request("after-ping", "new-token", ARTIFACT), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))).requestId()); + server.healthy(); + } + } + + /*************************************************************************** + ** A slot confirmed by a retired PREPARE or INVOKE needs no PING inside the + ** native window; an expired window, zero window or unconfirmed state pings. + ***************************************************************************/ + @Test + void confirmedSlotSkipsWireHeartbeatOnlyInsideWindow() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 2, 2, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + long pings = server.requests.stream().filter(frame -> "PING".equals(frame.optString("type"))).count(); + provider.heartbeat(Duration.ofSeconds(2), Duration.ofSeconds(30)); + assertEquals(pings, server.requests.stream().filter(frame -> "PING".equals(frame.optString("type"))).count()); + assertTrue(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + + provider.heartbeat(Duration.ofSeconds(2), Duration.ZERO); + assertEquals(pings + 1, server.requests.stream().filter(frame -> "PING".equals(frame.optString("type"))).count()); + + assertEquals("skip-after-invoke", WorkerProtocol.decodeExecuteResponse(provider.execute(request("skip-after-invoke", "token", ARTIFACT), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))).requestId()); + provider.heartbeat(Duration.ofSeconds(2), Duration.ofSeconds(30)); + assertEquals(pings + 1, server.requests.stream().filter(frame -> "PING".equals(frame.optString("type"))).count()); + + Thread.sleep(40); + provider.heartbeat(Duration.ofSeconds(2), Duration.ofMillis(5)); + assertEquals(pings + 2, server.requests.stream().filter(frame -> "PING".equals(frame.optString("type"))).count()); + server.healthy(); + } + } + + /*************************************************************************** + ** The window is bounded and a legacy single-argument heartbeat always pings. + ***************************************************************************/ + @Test + void heartbeatWindowIsBoundedAndLegacyHeartbeatAlwaysPings() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 2, 2, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + assertThrows(ScriptingException.class, () -> provider.heartbeat(Duration.ofSeconds(2), Duration.ofSeconds(-1))); + assertThrows(ScriptingException.class, () -> provider.heartbeat(Duration.ofSeconds(2), Duration.ofMinutes(11))); + assertThrows(ScriptingException.class, () -> provider.heartbeat(Duration.ofSeconds(2), null)); + provider.heartbeat(Duration.ofSeconds(2)); + assertEquals(1, server.requests.stream().filter(frame -> "PING".equals(frame.optString("type"))).count()); + server.healthy(); + } + } + + /*************************************************************************** + ** Forged/extended heartbeat replies evict only the failed authenticated epoch. + ***************************************************************************/ + @Test + void invalidHeartbeatRequiresColdReconnect() throws Exception + { + for(String fault : List.of("PONG", "PONG_EXTRA")) + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + server.fault = fault; + assertThrows(ScriptingException.class, () -> provider.heartbeat(Duration.ofSeconds(2))); + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + assertThrows(ScriptingException.class, () -> provider.execute(request("cold", "token", ARTIFACT), message -> "{}")); + assertEquals(1, server.accepted.get()); + server.fault = ""; + provider.prepareVerified(new WarmWorkerPreparation.Artifact(ARTIFACT.sha256(), ARTIFACT.entryPoint(), ARTIFACT.bundle(), "fixture-toolchain"), Duration.ofSeconds(2)); + assertEquals(2, server.accepted.get()); + assertTrue(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + server.healthy(); + } + } + } + + /*************************************************************************** + ** A busy valid invocation retains its slot while cold heartbeats skip it. + ***************************************************************************/ + @Test + void heartbeatDoesNotStealBusyInvocation() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5)); + var callers = Executors.newSingleThreadExecutor()) + { + provider.prepare(ARTIFACT, 1); + server.fault = "BLOCK"; + var invocation = callers.submit(() -> provider.execute(request("busy-heartbeat", "token", ARTIFACT), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + provider.heartbeat(Duration.ofSeconds(1)); + assertEquals(0, server.requests.stream().filter(frame -> "PING".equals(frame.optString("type"))).count()); + server.continueInvocation.countDown(); + assertEquals("busy-heartbeat", WorkerProtocol.decodeExecuteResponse(invocation.get(3, TimeUnit.SECONDS)).requestId()); + assertTrue(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + server.healthy(); + } + } + + /*************************************************************************** + ** A matching warm invocation waits for heartbeat retirement without issuing + ** a second prepare, connection or invocation, and keeps its broker caller. + ***************************************************************************/ + @Test + void executionHandsOffMatchingHeartbeatLease() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5)); + var callers = Executors.newFixedThreadPool(2)) + { + provider.prepare(ARTIFACT, 1); + server.fault = "PING_BLOCK"; + var heartbeat = callers.submit(() -> + { + provider.heartbeat(Duration.ofSeconds(4)); + return (true); + }); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + assertTrue(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + var coldMiss = callers.submit(() -> assertThrows(ScriptingException.class, + () -> provider.execute(request("not-prepared", "token", artifact("missing-code")), message -> "{}"))); + assertEquals(ScriptingException.Kind.UNAVAILABLE, coldMiss.get(1, TimeUnit.SECONDS).getKind()); + CountDownLatch entered = new CountDownLatch(1); + var invocation = callers.submit(() -> + { + Thread caller = Thread.currentThread(); + entered.countDown(); + return (provider.execute(request("heartbeat-handoff", "token", ARTIFACT), message -> + { + assertEquals(caller, Thread.currentThread()); + return (WorkerProtocol.encodeBrokerResponse(List.of(), false)); + })); + }); + try + { + assertTrue(entered.await(1, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> invocation.get(150, TimeUnit.MILLISECONDS)); + assertEquals(0, server.requests.stream().filter(frame -> "INVOKE".equals(frame.optString("type"))).count()); + } + finally + { + server.continueInvocation.countDown(); + } + assertTrue(heartbeat.get(2, TimeUnit.SECONDS)); + assertEquals("heartbeat-handoff", WorkerProtocol.decodeExecuteResponse(invocation.get(2, TimeUnit.SECONDS)).requestId()); + assertEquals(1, server.accepted.get()); + assertEquals(1, server.requests.stream().filter(frame -> "PREPARE".equals(frame.optString("type"))).count()); + assertEquals(1, server.requests.stream().filter(frame -> "INVOKE".equals(frame.optString("type"))).count()); + server.healthy(); + } + } + + /*************************************************************************** + ** Failed or expired heartbeat retirement wakes a matching waiter without + ** sending an invocation, reconnecting or treating a stale epoch as ready. + ***************************************************************************/ + @Test + void failedHeartbeatRefusesWaitingExecutionBeforeInvoke() throws Exception + { + for(boolean timeout : List.of(false, true)) + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5)); + var callers = Executors.newFixedThreadPool(2)) + { + provider.prepare(ARTIFACT, 1); + server.fault = "PING_BLOCK"; + var heartbeat = callers.submit(() -> assertThrows(ScriptingException.class, + () -> provider.heartbeat(timeout ? Duration.ofMillis(700) : Duration.ofSeconds(4)))); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + CountDownLatch entered = new CountDownLatch(1); + var invocation = callers.submit(() -> + { + entered.countDown(); + return (assertThrows(ScriptingException.class, () -> provider.execute(request("failed-ping", "token", ARTIFACT), message -> "{}"))); + }); + try + { + assertTrue(entered.await(1, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> invocation.get(150, TimeUnit.MILLISECONDS)); + if(!timeout) + { + server.fault = "PONG"; + server.continueInvocation.countDown(); + } + assertEquals(timeout ? ScriptingException.Kind.UNAVAILABLE : ScriptingException.Kind.INTEGRITY, + heartbeat.get(2, TimeUnit.SECONDS).getKind()); + assertEquals(ScriptingException.Kind.UNAVAILABLE, invocation.get(1, TimeUnit.SECONDS).getKind()); + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + assertEquals(0, server.requests.stream().filter(frame -> "INVOKE".equals(frame.optString("type"))).count()); + assertEquals(1, server.accepted.get()); + } + finally + { + server.continueInvocation.countDown(); + } + } + } + } + + /*************************************************************************** + ** Interruption and provider closure wake a maintenance waiter promptly; + ** cancellation returns its permit without corrupting a valid heartbeat. + ***************************************************************************/ + @Test + void maintenanceWaiterCancelsOrClosesWithoutInvocation() throws Exception + { + for(boolean close : List.of(false, true)) + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5)); + var callers = Executors.newFixedThreadPool(2)) + { + provider.prepare(ARTIFACT, 1); + server.fault = "PING_BLOCK"; + var heartbeat = callers.submit(() -> + { + try + { + provider.heartbeat(Duration.ofSeconds(4)); + return (true); + } + catch(ScriptingException failure) + { + return (false); + } + }); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + AtomicReference caller = new AtomicReference<>(); + CountDownLatch entered = new CountDownLatch(1); + var invocation = callers.submit(() -> + { + caller.set(Thread.currentThread()); + entered.countDown(); + assertThrows(ScriptingException.class, () -> provider.execute(request("cancel-ping", "token", ARTIFACT), message -> "{}")); + return (Thread.currentThread().isInterrupted()); + }); + try + { + assertTrue(entered.await(1, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> invocation.get(150, TimeUnit.MILLISECONDS)); + if(close) + { + provider.close(); + } + else + { + caller.get().interrupt(); + } + assertEquals(!close, invocation.get(1, TimeUnit.SECONDS)); + assertEquals(0, server.requests.stream().filter(frame -> "INVOKE".equals(frame.optString("type"))).count()); + if(!close) + { + assertEquals(ScriptingException.Kind.INVALID, + assertThrows(ScriptingException.class, () -> provider.execute("{}", message -> "{}")).getKind(), "Cancellation released the execution permit"); + } + } + finally + { + server.continueInvocation.countDown(); + } + assertEquals(!close, heartbeat.get(2, TimeUnit.SECONDS)); + if(!close) + { + assertEquals("after-cancel", WorkerProtocol.decodeExecuteResponse(provider.execute(request("after-cancel", "token", ARTIFACT), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))).requestId()); + } + server.healthy(); + } + } + } + + /*************************************************************************** + ** The handoff consumes the original wire deadline; a valid result without + ** retirement cannot gain a second full deadline after the heartbeat ends. + ***************************************************************************/ + @Test + void maintenanceHandoffSharesOriginalRetirementDeadline() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(3)); + var callers = Executors.newFixedThreadPool(2)) + { + provider.prepare(ARTIFACT, 1); + server.fault = "PING_BLOCK"; + var heartbeat = callers.submit(() -> + { + provider.heartbeat(Duration.ofSeconds(3)); + return (true); + }); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + CountDownLatch entered = new CountDownLatch(1); + AtomicInteger brokers = new AtomicInteger(); + var invocation = callers.submit(() -> + { + entered.countDown(); + return (provider.execute(request("remaining-deadline", "token", ARTIFACT), message -> + { + brokers.incrementAndGet(); + return (WorkerProtocol.encodeBrokerResponse(List.of(), false)); + })); + }); + try + { + assertTrue(entered.await(1, TimeUnit.SECONDS)); + assertThrows(TimeoutException.class, () -> invocation.get(1400, TimeUnit.MILLISECONDS)); + server.fault = "NO_RETIRE"; + server.continueInvocation.countDown(); + assertTrue(heartbeat.get(1, TimeUnit.SECONDS)); + ExecutionException failure = assertThrows(ExecutionException.class, () -> invocation.get(2400, TimeUnit.MILLISECONDS)); + assertTrue(failure.getCause() instanceof ScriptingException); + assertEquals(1, brokers.get()); + assertEquals(1, server.requests.stream().filter(frame -> "INVOKE".equals(frame.optString("type"))).count()); + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + assertEquals(1, server.accepted.get()); + } + finally + { + server.continueInvocation.countDown(); + provider.close(); + } + server.healthy(); + } + } + + /*************************************************************************** + ** Cold preparation of another entry is not heartbeat maintenance and cannot + ** turn an already prepared invocation into a queued cold operation. + ***************************************************************************/ + @Test + void executionDoesNotWaitForColdPreparation() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 2, 4096, Duration.ofSeconds(5)); + var callers = Executors.newFixedThreadPool(2)) + { + provider.prepare(ARTIFACT, 1); + server.fault = "PREPARE_BLOCK"; + var preparation = callers.submit(() -> + { + provider.prepare(artifact("another-code-entry"), 1); + return (true); + }); + assertTrue(server.invoked.await(2, TimeUnit.SECONDS)); + try + { + var invocation = callers.submit(() -> assertThrows(ScriptingException.class, + () -> provider.execute(request("not-maintenance", "token", ARTIFACT), message -> "{}"))); + assertEquals(ScriptingException.Kind.UNAVAILABLE, invocation.get(1, TimeUnit.SECONDS).getKind()); + assertEquals(0, server.requests.stream().filter(frame -> "INVOKE".equals(frame.optString("type"))).count()); + } + finally + { + server.continueInvocation.countDown(); + } + assertTrue(preparation.get(2, TimeUnit.SECONDS)); + server.healthy(); + } + } + + /*************************************************************************** + ** Whole cold maintenance deadlines retire blocked slots without fallback. + ***************************************************************************/ + @Test + void heartbeatBudgetClosesBlockedTransport() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 1, 1, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + server.fault = "PING_BLOCK"; + long start = System.nanoTime(); + try + { + assertThrows(ScriptingException.class, () -> provider.heartbeat(Duration.ofMillis(150))); + assertTrue(Duration.ofNanos(System.nanoTime() - start).compareTo(Duration.ofSeconds(2)) < 0); + assertFalse(provider.isPrepared(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain", Duration.ofSeconds(5))); + } + finally + { + server.continueInvocation.countDown(); + } + } + } + + /*************************************************************************** + ** A non-reading TLS peer cannot retain the single watchdog after cancellation. + ***************************************************************************/ + @Test + void blockedPeerCloseReleasesDeadlineExecutor() throws Exception + { + try(var server = new SupervisorTlsFixture(identity); var scheduler = new ScheduledThreadPoolExecutor(1)) + { + scheduler.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + Configuration config = new Configuration(identity, new Endpoint("localhost", server.server.getLocalPort()), null, + "fixture", "profile", "fixture-toolchain", "fixture-evidence", "fixture-evidence", Duration.ofSeconds(5), 1, 1, 1, 4096); + try(var connection = new SupervisorConnection(config)) + { + connection.begin(scheduler); + connection.connect(config.executionEndpoint()); + connection.end(); + server.fault = "PING_BLOCK"; + connection.begin(scheduler, Duration.ofMillis(150).toNanos()); + connection.next(); + connection.write(connection.envelope("PING", "blocked-heartbeat")); + assertTrue(server.invoked.await(1, TimeUnit.SECONDS)); + assertTrue(scheduler.schedule(() -> true, 200, TimeUnit.MILLISECONDS).get(1, TimeUnit.SECONDS)); + assertFalse(connection.isOpen()); + assertThrows(ScriptingException.class, connection::check); + } + finally + { + server.continueInvocation.countDown(); + } + } + } + + + + /*************************************************************************** + ** Aggregate readiness requires every exact artifact in the same live cache, + ** including after one connection retires while another artifact stays warm. + ***************************************************************************/ + @Test + void aggregateReadinessRefusesPartialEpochAfterRetirement() throws Exception + { + PreparedArtifact other = artifact("second-trusted-fixture"); + var first = new WarmWorkerMaintenance.CodeIdentity(ARTIFACT.sha256(), ARTIFACT.entryPoint(), "fixture-toolchain"); + var second = new WarmWorkerMaintenance.CodeIdentity(other.sha256(), other.entryPoint(), "fixture-toolchain"); + try(var server = new SupervisorTlsFixture(identity); var provider = provider(server, 2, 1, 4096, Duration.ofSeconds(5))) + { + provider.prepare(ARTIFACT, 1); + assertFalse(provider.arePrepared(List.of(first, second), Duration.ofSeconds(5))); + provider.prepare(other, 1); + assertTrue(provider.arePrepared(List.of(first, second), Duration.ofSeconds(5))); + assertFalse(provider.arePrepared(java.util.Collections.nCopies(1001, first), Duration.ofSeconds(5))); + server.fault = "RESULT"; + assertThrows(ScriptingException.class, () -> provider.execute(request("retire-first", "token", ARTIFACT), + message -> WorkerProtocol.encodeBrokerResponse(List.of(), false))); + assertTrue(provider.arePrepared(List.of(second), Duration.ofSeconds(5))); + assertFalse(provider.arePrepared(List.of(first, second), Duration.ofSeconds(5))); + server.fault = ""; + provider.prepare(ARTIFACT, 1); + assertTrue(provider.arePrepared(List.of(first, second), Duration.ofSeconds(5))); + server.healthy(); + } + } +} diff --git a/base/src/test/java/com/kingsrook/qbits/scripting/worker/SupervisorTlsFixture.java b/base/src/test/java/com/kingsrook/qbits/scripting/worker/SupervisorTlsFixture.java new file mode 100644 index 0000000..8028993 --- /dev/null +++ b/base/src/test/java/com/kingsrook/qbits/scripting/worker/SupervisorTlsFixture.java @@ -0,0 +1,396 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.worker; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.TrustManagerFactory; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.json.JSONArray; +import org.json.JSONObject; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/******************************************************************************* + ** Fixed trusted TLS protocol peer. Never compiles or executes author code. + *******************************************************************************/ +final class SupervisorTlsFixture implements AutoCloseable +{ + final SSLContext context; + final SSLServerSocket server; + final AtomicInteger accepted = new AtomicInteger(); + final List requests = new CopyOnWriteArrayList<>(); + final CountDownLatch invoked = new CountDownLatch(1); + final CountDownLatch continueInvocation = new CountDownLatch(1); + volatile String fault = "NONE"; + private final List sockets = new CopyOnWriteArrayList<>(); + private final List failures = new CopyOnWriteArrayList<>(); + private final List peers = new CopyOnWriteArrayList<>(); + private final Thread listener; + + /*************************************************************************** + ** Bind only an owned ephemeral loopback port; client certificates required. + ***************************************************************************/ + SupervisorTlsFixture(SSLContext context) throws IOException + { + this(context, true); + } + + + + /*************************************************************************** + ** As above, with the client-certificate requirement fixed BEFORE the listener starts. + ** Flipping it later is a race: the listener's first accept() captures the socket + ** configuration, so a connection arriving after a late setNeedClientAuth(false) can + ** still be handshaken under the old setting (and then present its client certificate). + ***************************************************************************/ + SupervisorTlsFixture(SSLContext context, boolean needClientAuth) throws IOException + { + this.context = context; + server = (SSLServerSocket) context.getServerSocketFactory().createServerSocket(0, 10, java.net.InetAddress.getLoopbackAddress()); + server.setNeedClientAuth(needClientAuth); + server.setEnabledProtocols(new String[] {"TLSv1.3"}); + listener = Thread.ofPlatform().daemon().name("trusted-supervisor-fixture").start(this::accept); + } + + /*************************************************************************** + ** Generate an ephemeral test identity using the current JDK, never a trust-all + ** manager or production credential. Password exists only in child environment. + ***************************************************************************/ + static SSLContext identity(Path directory) throws Exception + { + String password = UUID.randomUUID().toString(); + Path store = directory.resolve("fixture.p12"); + String executable = Path.of(System.getProperty("java.home"), "bin", System.getProperty("os.name").startsWith("Windows") ? "keytool.exe" : "keytool").toString(); + ProcessBuilder builder = new ProcessBuilder(executable, "-genkeypair", "-alias", "fixture", "-keyalg", "EC", "-groupname", "secp256r1", + "-dname", "CN=localhost", "-ext", "SAN=dns:localhost", "-validity", "2", "-storetype", "PKCS12", + "-keystore", store.toString(), "-storepass:env", "QQQ_TEST_TLS_PASSWORD", "-keypass:env", "QQQ_TEST_TLS_PASSWORD", "-noprompt"); + builder.environment().put("QQQ_TEST_TLS_PASSWORD", password); + builder.redirectOutput(ProcessBuilder.Redirect.DISCARD); + builder.redirectError(ProcessBuilder.Redirect.DISCARD); + Process process = builder.start(); + try + { + assertTrue(process.waitFor(15, TimeUnit.SECONDS)); + assertEquals(0, process.exitValue()); + } + finally + { + if(process.isAlive()) + { + process.destroyForcibly(); + } + } + KeyStore keyStore = KeyStore.getInstance("PKCS12"); + try(var input = Files.newInputStream(store)) + { + keyStore.load(input, password.toCharArray()); + } + KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keys.init(keyStore, password.toCharArray()); + TrustManagerFactory trust = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trust.init(keyStore); + SSLContext context = SSLContext.getInstance("TLSv1.3"); + context.init(keys.getKeyManagers(), trust.getTrustManagers(), null); + return (context); + } + + /*************************************************************************** + ** Accept independent cold and warm connections without a single-test bottleneck. + ***************************************************************************/ + private void accept() + { + while(!server.isClosed()) + { + try + { + SSLSocket socket = (SSLSocket) server.accept(); + sockets.add(socket); + accepted.incrementAndGet(); + peers.add(Thread.ofPlatform().daemon().name("trusted-supervisor-peer").start(() -> serve(socket))); + } + catch(IOException failure) + { + if(!server.isClosed()) + { + failures.add(failure); + } + } + } + } + + /*************************************************************************** + ** Fixed protocol replies and explicit faults, never submitted code evaluation. + ***************************************************************************/ + private void serve(SSLSocket socket) + { + try(socket) + { + socket.setSoTimeout(5000); + socket.startHandshake(); + var input = new DataInputStream(socket.getInputStream()); + var output = new DataOutputStream(socket.getOutputStream()); + JSONObject hello = read(input); + requests.add(hello); + String session = UUID.randomUUID().toString(); + JSONObject accepted = new JSONObject(hello.toString()).put("type", "ACCEPT").put("session", session); + if(List.of("HELLO", "PROFILE", "NAMESPACE", "TOOLCHAIN").contains(fault)) + { + String key = switch(fault) + { + case "PROFILE" -> "profile"; + case "NAMESPACE" -> "namespace"; + case "TOOLCHAIN" -> "toolchain"; + default -> "evidence"; + }; + accepted.put(key, "wrong"); + } + write(output, accepted); + while(true) + { + JSONObject request = read(input); + requests.add(request); + assertEquals(session, request.getString("session")); + String type = request.getString("type"); + if("PING".equals(type)) + { + assertEquals(java.util.Set.of("protocol", "type", "session", "sequence", "requestId"), request.keySet()); + if("PING_BLOCK".equals(fault)) + { + invoked.countDown(); + assertTrue(continueInvocation.await(5, TimeUnit.SECONDS)); + } + JSONObject pong = frame(request, "PONG"); + if("PONG".equals(fault)) + { + pong.put("sequence", 0); + } + if("PONG_EXTRA".equals(fault)) + { + pong.put("inventory", new JSONArray()); + } + write(output, pong); + retire(output, request); + } + else if("PREPARE".equals(type)) + { + if("PREPARE_BLOCK".equals(fault)) + { + invoked.countDown(); + assertTrue(continueInvocation.await(5, TimeUnit.SECONDS)); + } + assertFalse(request.has("payload")); + assertFalse(request.has("capabilityToken")); + JSONObject ready = frame(request, "READY").put("sha256", request.getString("sha256")) + .put("entryPoint", request.getString("entryPoint")).put("readyId", "ready-" + request.getLong("sequence")); + if("READY".equals(fault)) + { + ready.put("sha256", "0".repeat(64)); + } + write(output, ready); + retire(output, request); + } + else if("COMPILE".equals(type)) + { + JSONObject inner = request.getJSONObject("request"); + JSONObject result = new JSONObject().put("protocol", WorkerProtocol.VERSION).put("type", "compileResult") + .put("requestId", request.getString("requestId")).put("success", false).put("strict", true) + .put("toolchainId", inner.getString("toolchainId")).put("diagnostics", new JSONArray()); + write(output, frame(request, "COMPILE_BROKER".equals(fault) ? "BROKER" : "RESULT").put("message", result)); + retire(output, request); + } + else if("INVOKE".equals(type)) + { + invoke(input, output, request); + } + else + { + throw new AssertionError("Unexpected request type"); + } + } + } + catch(IOException ignored) + { + // Negative cases deliberately close TLS on malformed/uncertain frames. + } + catch(Throwable failure) + { + failures.add(failure); + } + } + + /*************************************************************************** + ** One broker request then result and trusted retirement, or an injected fault. + ***************************************************************************/ + private void invoke(DataInputStream input, DataOutputStream output, JSONObject request) throws Exception + { + JSONObject inner = request.getJSONObject("request"); + assertFalse(inner.has("bundle")); + invoked.countDown(); + if("BLOCK".equals(fault)) + { + assertTrue(continueInvocation.await(5, TimeUnit.SECONDS)); + } + if("LENGTH".equals(fault)) + { + output.writeInt(SupervisorProtocol.MAX_FRAME + 1); + output.flush(); + return; + } + if("UTF8".equals(fault)) + { + output.writeInt(2); + output.write(new byte[] {(byte) 0xc0, (byte) 0xaf}); + output.flush(); + return; + } + JSONObject broker = new JSONObject().put("protocol", WorkerProtocol.VERSION).put("type", "broker") + .put("requestId", request.getString("requestId")).put("operation", "query").put("token", inner.getString("capabilityToken")) + .put("table", "purchase_order").put("fields", new JSONArray().put("id")).put("criteria", new JSONArray()).put("limit", 1); + if("TOKEN".equals(fault)) + { + broker.put("token", "wrong"); + } + if("BROKER_ID".equals(fault)) + { + broker.put("requestId", "wrong"); + } + JSONObject brokerFrame = frame(request, "BROKER").put("message", broker); + if("SEQUENCE".equals(fault)) + { + brokerFrame.put("sequence", 0); + } + write(output, brokerFrame); + JSONObject answer = read(input); + requests.add(answer); + assertEquals("BROKER_REPLY", answer.getString("type")); + JSONObject result = new JSONObject().put("protocol", WorkerProtocol.VERSION).put("type", "executeResult") + .put("requestId", request.getString("requestId")).put("outcome", "OK").put("result", new JSONObject().put("action", "ALLOW")); + if("RESULT".equals(fault)) + { + result.put("requestId", "wrong"); + } + write(output, frame(request, "RESULT").put("message", result)); + if("RETIRE".equals(fault)) + { + write(output, frame(request, "RESULT").put("message", new JSONObject().put("retired", true))); + } + else if("LATE_BROKER".equals(fault)) + { + write(output, frame(request, "BROKER").put("message", broker)); + } + else if("NO_RETIRE".equals(fault)) + { + // A valid result does not imply completion; wait for client deadline. + } + else + { + retire(output, request); + } + } + + /*************************************************************************** + ** Copy only fixed transport correlation fields, not request contents. + ***************************************************************************/ + private static JSONObject frame(JSONObject request, String type) + { + return (new JSONObject().put("protocol", SupervisorProtocol.VERSION).put("type", type).put("session", request.getString("session")) + .put("sequence", request.getLong("sequence")).put("requestId", request.getString("requestId"))); + } + + /*************************************************************************** + ** Fixture retirement is protocol evidence only, never real isolation proof. + ***************************************************************************/ + private static void retire(DataOutputStream output, JSONObject request) throws IOException + { + write(output, frame(request, "RETIRED").put("retired", true)); + } + + /*************************************************************************** + ** Independent wire writer, separate from production connection framing. + ***************************************************************************/ + private static void write(DataOutputStream output, JSONObject frame) throws IOException + { + byte[] bytes = frame.toString().getBytes(StandardCharsets.UTF_8); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + } + + /*************************************************************************** + ** The trusted fixture still bounds client frames before allocating buffers. + ***************************************************************************/ + private static JSONObject read(DataInputStream input) throws IOException + { + int length = input.readInt(); + if(length < 2 || length > SupervisorProtocol.MAX_FRAME) + { + throw new IOException("Invalid test frame"); + } + byte[] bytes = new byte[length]; + input.readFully(bytes); + return (new JSONObject(new String(bytes, StandardCharsets.UTF_8))); + } + + /*************************************************************************** + ** Never treat a server assertion failure as a successful client refusal test. + ***************************************************************************/ + void healthy() + { + assertTrue(failures.isEmpty(), () -> "Trusted fixture failed: " + failures.stream().map(value -> value.getClass().getSimpleName()).toList()); + } + + /*************************************************************************** + ** Release only owned connections/port and unblock the fixed peer fixture. + ***************************************************************************/ + @Override + public void close() throws Exception + { + continueInvocation.countDown(); + server.close(); + listener.join(1000); + for(SSLSocket socket : sockets) + { + socket.close(); + } + for(Thread peer : peers) + { + peer.join(1000); + assertFalse(peer.isAlive()); + } + healthy(); + } +} diff --git a/base/src/test/resources/FixtureCoordinator.java b/base/src/test/resources/FixtureCoordinator.java new file mode 100644 index 0000000..bd0d49c --- /dev/null +++ b/base/src/test/resources/FixtureCoordinator.java @@ -0,0 +1,90 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; + +/******************************************************************************* + ** Trusted protocol-only child process fixture. It never compiles or executes + ** user code. Java source-file launch makes transport tests independent of Node. + *******************************************************************************/ +class FixtureCoordinator +{ + /*************************************************************************** + ** Fixed fixture modes selected by requestId, never executable input. + ***************************************************************************/ + public static void main(String[] args) throws Exception + { + BufferedReader input = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); + String request = input.readLine(); + String mode = request.replaceAll(".*\\\"requestId\\\"\\s*:\\s*\\\"([^\\\"]+)\\\".*", "$1"); + String prefix = "{\"protocol\":\"qbit-scripting-worker/1\","; + if("hang".equals(mode)) + { + Thread.sleep(30000); + } + if("partial".equals(mode)) + { + System.out.print(prefix); + return; + } + if("huge".equals(mode)) + { + System.out.println("x".repeat(4 * 1024 * 1024 + 1)); + return; + } + if("diagnostics".equals(mode)) + { + System.err.print("private-fixture-diagnostic".repeat(10000)); + return; + } + boolean execute = request.contains("\"execute\""); + if(execute) + { + String token = "forged".equals(mode) ? "wrong" : "fixture-token"; + System.out.println(prefix + "\"type\":\"broker\",\"requestId\":\"read-1\",\"operation\":\"query\",\"token\":\"" + token + "\",\"table\":\"orders\",\"fields\":[\"id\"],\"limit\":1}"); + System.out.flush(); + if(input.readLine() == null) + { + return; + } + } + if(System.getenv("JAVA_HOME") != null || System.getenv("USERPROFILE") != null || System.getenv("HOME") != null + || System.getenv("QQQ_WORKER_OWNER_ID") == null) + { + System.exit(9); + } + String id = "wrong-id".equals(mode) ? "other" : mode; + String type = execute ? "executeResult" : "compileResult"; + String result = prefix + "\"type\":\"" + type + "\",\"requestId\":\"" + id + "\"}"; + System.out.println(result); + if("duplicate".equals(mode)) + { + System.out.println(result); + } + if("nonzero".equals(mode)) + { + System.err.println("private-fixture-diagnostic"); + System.exit(2); + } + } +} diff --git a/base/src/test/resources/log4j2-test.xml b/base/src/test/resources/log4j2-test.xml new file mode 100644 index 0000000..9ed980c --- /dev/null +++ b/base/src/test/resources/log4j2-test.xml @@ -0,0 +1 @@ + diff --git a/checkstyle/config.xml b/checkstyle/config.xml new file mode 100644 index 0000000..b1b80ec --- /dev/null +++ b/checkstyle/config.xml @@ -0,0 +1,215 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/checkstyle/license.txt b/checkstyle/license.txt new file mode 100644 index 0000000..92332b5 --- /dev/null +++ b/checkstyle/license.txt @@ -0,0 +1,19 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/deploy/.gitignore b/deploy/.gitignore new file mode 100644 index 0000000..7a60b85 --- /dev/null +++ b/deploy/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/deploy/NOTICE-GAPS.md b/deploy/NOTICE-GAPS.md new file mode 100644 index 0000000..286b7e0 --- /dev/null +++ b/deploy/NOTICE-GAPS.md @@ -0,0 +1,16 @@ +# Unqualified candidate: redistribution evidence incomplete + +The included LICENSE/NOTICE describe our standalone supervisor. The included +release-license-review.md and license-inventory.json are provenance inventories, +not substitutes for third-party license texts or legal approval. + +Release blockers still include complete target-linked Rust/WASI/QuickJS/Javy +notices, JNA's selected license plus native libffi notice, JSON's exact public +domain declaration, Extism's BSD notice and its separately compiled static +transitives, and final base/JRE/OS inventory. Preserve base-provided notices. +The unchanged JNA JAR retains its bundled platform resources; only its verified +linux-x86-64 JNI payload is selected externally, with runtime unpack disabled. + +No certificate, private key, password, QQQ dependency, author script or arbitrary +guest Wasm belongs in this image. The fixed Wasm SHA is 4B247288D6CDFF6783F8087A2A9039935F50AF9099451CC841A3639C9625A18C. +No production or hostile-runtime qualification is granted by this package. diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..c4bd68a --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,153 @@ +# Execution supervisor qualification packaging + +This subtree packages a candidate only. `productionQualified=false` and +`redistributionCleared=false` remain explicit in the image, snapshot and workload. +No build, image execution, cluster call or deployment was performed by these tools. +This is the execution lane only: no strict TypeScript compiler, Node, renderer, +QQQ application or printing service is added. Existing peer resources are untouched. + +## Local snapshot and build handoff + +Use existing CPython 3.13+; scripts use only its standard library. No environment +install or dependency resolver is needed. `snapshot_execution.py` accepts a JSON +object with exactly these keys: `supervisor`, `jna`, `json`, `extism`, `wasm`, +`license`, `notice`, `licenseReview`, `cargoInventory`. Each value has exactly +`path` (absolute existing local file) and lowercase `sha256`. Supply actual values; +there is deliberately no descriptor containing guessed artifact locations/hashes. + +The supervisor hash must come from the accepted standalone build, not an app QBit +JAR. LICENSE/NOTICE are the supervisor's own files; licenseReview/cargoInventory +are the bounded release inventory under `../wasm-runtime/docs`. Extism1.41.0, +JNA5.18.1, JSON20251224 and the fixed Wasm hash (`WASM_SHA`, kept equal to `../wasm-runtime/PINNED-WASM.json` by `test_deploy.PinnedWasmTest`; rebuild with `../wasm-runtime/Build-Wasm.ps1`) are also enforced in the script. +The JNA Linux x86-64 JNI bytes are extracted from that exact JAR and separately +hashed. No guest code, certificates, secrets, test fixtures, SDK, compiler or +operator configuration is copied. Supervisor class inventory rejects foreign +Java package classes; native headers must be ELF64 little-endian x86-64. + +Run from this directory (paths below are operator-created, not bundled defaults): + +```text +python snapshot_execution.py ABSOLUTE_LOCAL_INPUTS_JSON ABSOLUTE_NEW_USER_CACHE_DIRECTORY +python snapshot_execution.py --verify ABSOLUTE_EXISTING_CACHE_CONTEXT +python -m unittest -v test_deploy +``` + +The output directory must be new and resolve beneath the current user's `.cache`. +All hashes are checked before copying and again after copying. Failure never +overwrites an existing snapshot; preserve a partial failed context for diagnosis. +`metadata/candidate.json` inventories copied inputs without leaking host paths. +The closed Docker context contains only Containerfile/payload/metadata. All input +files remain unchanged; no recursive cleanup or global package/cache writes occur. +The verification mode refuses any unlisted file, symlink or changed hash. Run it +again immediately before the separately authorized build; keep the context +operator-owned and frozen between verification and build. It does not authenticate +an attacker-rewritten manifest: retain its independently recorded digest outside +the context as part of the root artifact handoff. + +Root/operator-only build recipe, **not executed here**, after checking the base +image is already cached and the snapshot is reviewed: + +```text +docker build --network=none --pull=false --platform=linux/amd64 --file ABSOLUTE_CACHE_CONTEXT/Containerfile --tag LOCAL_CANDIDATE_TAG ABSOLUTE_CACHE_CONTEXT +``` + +BuildKit itself must already exist; this file does not fetch an external syntax +frontend. Base is the exact reviewed Temurin21.0.13+9 amd64 manifest +`eclipse-temurin@sha256:db724162e4ff5f71fa0a2815026314d9e5e0e9a139f49be1d4717e6930e93341`. +The sole RUN removes explicitly named shell/download-client files **inside that +image layer**, never on the host. It installs nothing. Root must inspect final +layers for unexpected shell aliases/utilities and dependency/linker availability; +this pruning is not a containment guarantee or a complete minimal-OS inventory. +The exec-form entrypoint bypasses the base certificate shell script. TLS stores +are supplied separately. JNA uses its explicit read-only native directory with +unpack disabled. Root-owned0555 payload directories/files are non-writable by +UID/GID65532; Kubernetes also makes the whole root filesystem read-only. + +Final-image checks remain required: exact source/artifact/OCI hashes, actual +Java/native versions and linkage, no duplicate provider/dependency, real nonroot +read-only launch, fixed trusted ABI corpus, mTLS identity, resource/retirement +behavior, final OS/native/Wasm SBOM and complete notices. See NOTICE-GAPS.md; +packaging that gap report does not discharge any notice obligation. + +## Qualification manifest renderer + +Copy `qualification.disabled.json` to a private local descriptor, explicitly fill +every field, and enable only for the agreed qualification exercise. Rendering +refuses disabled/missing/unknown/duplicate keys, floating image tags, default +RuntimeClass, absent client selectors, secrets as values, and invalid bounds. + +```text +python render_qualification.py PRIVATE_OPERATOR_DESCRIPTOR NEW_LOCAL_MANIFEST_JSON +``` + +This creates a Kubernetes `List` and never runs kubectl. The namespace must begin +`qqq-qual-` and be distinct from the selected QQQ client namespace. It must be a +new, dedicated qualification namespace: a Namespace object is not proof that an +existing namespace is safe to adopt. Before any separately authorized deployment, +review resource ownership and validate against the target Kubernetes schema and +admission policies. The script does not create or change RuntimeClass, node PID +settings, client resources, credentials, roles or service accounts. + +`fixtures/` contains a complete **synthetic, never-deploy** input and rendered +List using the reserved `registry.invalid` domain. The positive tests compare +the whole artifact and separately check object references, network selectors, +volume/Secret paths, security and actual supervisor configuration keys. This is +local structural validation, not a substitute for target-cluster schema and +admission validation. Wire profile/toolchain/evidence/client namespace strings +are limited to the actual supervisor's128-character `[A-Za-z0-9._:-]` grammar; +Kubernetes selector keys/values use their separate label and DNS-prefix grammar. + +Required operator inputs include immutable execution image@sha256; actual +sandbox RuntimeClass and selected sandbox nodes; explicit Pod Security version; +existing Secret names; exact client certificate DER SHA256→protocol namespace +map; QQQ namespace plus nonempty pod labels; profile/toolchain/evidence IDs; +finite CPU/memory/tmp/guest/cache/admission limits; and actual node PID policy +evidence. Equal evidence strings are a wire precondition, **not qualification**. + +The server Secret must contain `server.p12`, the trust Secret `clients.p12`, and +the password Secret `keyStorePassword`/`trustStorePassword`. Only references enter +the manifest. Restrict these Secrets to the qualification namespace and ensure +the server certificate matches the DNS name used by the client. The fixed image +contains neither credentials nor mutable trust. Clients retain current QQQ +authority and broker tokens; readiness supplies no execution permission. + +Restricted Pod security includes nonroot65532, dropped capabilities, no privilege +escalation, RuntimeDefault seccomp, read-only root and bounded memory-backed/tmp. +CPU/memory requests equal limits; the memory floor is a conservative refusal +check, not demonstrated capacity. Extism common-runtime compilation, per-slot +native heap, JVM, frames and caches still need measurements. No service-account +token is mounted. No host namespaces, host volumes, PVC, ingress exposure or +service-account/RBAC resources are generated. + +The namespace default-deny policy denies ingress and egress. A second rule permits +only TCP18443 from **both** the explicit QQQ namespace and pod labels in one peer. +No DNS/HTTP/other outbound rule is added; established response traffic uses the +incoming connection. An enforcing CNI and control of other additive policies are +prerequisites. Host/kubelet traffic and CNI semantics require actual verification. +See the primary [NetworkPolicy specification](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +and [Restricted Pod Security controls](https://kubernetes.io/docs/concepts/security/pod-security-standards/). + +Kubernetes has no per-PodSpec PID-limit field. `pidLimit` and `pidPolicyEvidence` +produce explicit expectation annotations only; the operator must configure and +verify the selected kubelet/runtime `podPidsLimit` (or stronger sandbox equivalent) +and node process reservation before qualification. Node labels are scheduling +intent, not enforcement. See [official PID-limit guidance](https://kubernetes.io/docs/concepts/policy/pid-limiting/). + +## Readiness and termination limits + +The server binds its port after fixed-runtime slot compilation and CPU-accounting +checks. TCP startup/readiness probes establish **listener availability only**; +they cannot prove authenticated admission capacity, intact prepared publications, +usable compiler, current grants or safe native cleanup. They may transiently +consume a handshake thread; measure this with configured connection limits. +The existing QQQ native warm health check, including fresh authenticated heartbeat +and complete current required pins, must gate QQQ application traffic. This +manifest does not install or bypass that application health check. + +There is deliberately no TCP liveness probe presented as a native-free watchdog. +Hung cleanup and occupied/retired engine slots can leave a listening process. +Independent hard-termination/liveness policy is an **open qualification gate**. +The30s Pod termination grace permits the current15s bounded server drain and then +orchestrator termination; actual CRI/sandbox enforcement must be tested. Do not +claim that a signal, Java timer, NetworkPolicy or these manifests alone qualify +hostile code or production Kubernetes operation. diff --git a/deploy/execution.Containerfile b/deploy/execution.Containerfile new file mode 100644 index 0000000..ece50e6 --- /dev/null +++ b/deploy/execution.Containerfile @@ -0,0 +1,16 @@ +# Qualification candidate only. Build context is snapshot_execution.py output. +# Exact linux/amd64 Temurin 21.0.13+9 JRE; no package installation or downloads. +FROM eclipse-temurin@sha256:db724162e4ff5f71fa0a2815026314d9e5e0e9a139f49be1d4717e6930e93341 +LABEL com.kingsrook.productionQualified="false" com.kingsrook.redistributionCleared="false" +USER 0:0 +# Merged-/usr shell executables and download clients are not runtime entrypoints. +# Actual final-image inventory must confirm their absence; this is not a sandbox. +RUN ["/usr/bin/rm", "-f", "/usr/bin/sh", "/usr/bin/bash", "/usr/bin/dash", "/usr/bin/rbash", "/usr/bin/ash", "/usr/bin/zsh", "/usr/bin/ksh", "/usr/bin/csh", "/usr/bin/tcsh", "/usr/bin/curl", "/usr/bin/wget", "/__cacert_entrypoint.sh"] +COPY --chown=0:0 --chmod=0555 payload/ /opt/qqq-supervisor/ +COPY --chown=0:0 --chmod=0555 metadata/ /usr/share/doc/qqq-supervisor/ +WORKDIR /opt/qqq-supervisor +ENV HOME=/tmp +USER 65532:65532 +EXPOSE 18443 +STOPSIGNAL SIGTERM +ENTRYPOINT ["/opt/java/openjdk/bin/java", "-XX:MaxRAMPercentage=25", "-XX:+ExitOnOutOfMemoryError", "-Djava.io.tmpdir=/tmp", "-Djna.boot.library.path=/opt/qqq-supervisor/native", "-Djna.nounpack=true", "-Djna.nosys=true", "-cp", "/opt/qqq-supervisor/lib/*", "com.kingsrook.scripting.supervisor.SupervisorMain", "/run/qqq-supervisor/config/config.json"] diff --git a/deploy/fixtures/README.md b/deploy/fixtures/README.md new file mode 100644 index 0000000..2e5c1d6 --- /dev/null +++ b/deploy/fixtures/README.md @@ -0,0 +1,19 @@ +# Synthetic static fixture — never deploy + +These files contain no credentials, real certificate fingerprints, accepted +evidence, actual sandbox configuration or deployable image. The reserved +`registry.invalid` image reference and repeated test digests are deliberate. +`enabled=true` exists only to exercise the positive rendering branch. The normal +operator template remains `../qualification.disabled.json`, disabled and empty. + +`synthetic-input.json` and the full six-resource `synthetic-manifest.json` are +compared by tests. Separate assertions check namespace isolation, resource +identity uniqueness, Deployment/Service/NetworkPolicy selection, ports, mounts, +Secret key references, file modes, exact SupervisorMain config keys, security, +finite resources, and the absence of a false native-cleanup liveness claim. +The local CLI test renders to a temporary file, refuses an overwrite, and refuses +the disabled template without output. It invokes no container or cluster tool. + +These checks are not target Kubernetes OpenAPI/admission validation, actual CNI +enforcement, RuntimeClass qualification, resource measurements, or production +readiness. No fixture has been applied to a cluster. diff --git a/deploy/fixtures/synthetic-input.json b/deploy/fixtures/synthetic-input.json new file mode 100644 index 0000000..5c8f66e --- /dev/null +++ b/deploy/fixtures/synthetic-input.json @@ -0,0 +1,38 @@ +{ + "enabled": true, + "namespace": "qqq-qual-unit", + "image": "registry.invalid/qqq/synthetic-execution@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "sandboxRuntimeClass": "unit-sandbox", + "podSecurityVersion": "v1.34", + "tlsSecret": "unit-server", + "trustSecret": "unit-clients", + "passwordSecret": "unit-passwords", + "certificateNamespaces": { + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": "unit-client" + }, + "clientNamespace": "unit-client", + "clientPodLabels": { + "app": "unit-client" + }, + "nodeSelector": { + "qualification-pool": "unit" + }, + "profile": "unit-profile", + "toolchain": "unit-toolchain", + "evidence": "unit-evidence", + "requiredEvidence": "unit-evidence", + "cpuMillis": 1000, + "memoryMiB": 1024, + "pidLimit": 256, + "pidPolicyEvidence": "unit-node-policy", + "tmpMiB": 16, + "startupSeconds": 120, + "slots": 1, + "connections": 2, + "coldOperations": 1, + "entries": 8, + "codeBytes": 8388608, + "guestMemoryMiB": 64, + "idleMillis": 30000, + "fuel": 100000000 +} diff --git a/deploy/fixtures/synthetic-manifest.json b/deploy/fixtures/synthetic-manifest.json new file mode 100644 index 0000000..bc153a5 --- /dev/null +++ b/deploy/fixtures/synthetic-manifest.json @@ -0,0 +1,295 @@ +{ + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": "qqq-qual-unit", + "labels": { + "pod-security.kubernetes.io/enforce": "restricted", + "pod-security.kubernetes.io/enforce-version": "v1.34", + "pod-security.kubernetes.io/audit": "restricted", + "pod-security.kubernetes.io/warn": "restricted" + } + } + }, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "execution-supervisor", + "namespace": "qqq-qual-unit" + }, + "immutable": true, + "data": { + "config.json": "{\"bind\":\"0.0.0.0\",\"port\":18443,\"keyStore\":\"/run/qqq-supervisor/tls/server.p12\",\"trustStore\":\"/run/qqq-supervisor/trust/clients.p12\",\"certificateNamespaces\":{\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\":\"unit-client\"},\"profile\":\"unit-profile\",\"toolchain\":\"unit-toolchain\",\"evidence\":\"unit-evidence\",\"requiredEvidence\":\"unit-evidence\",\"runtimePath\":\"/opt/qqq-supervisor/runtime/qqq_javy_runtime.wasm\",\"runtimeSha256\":\"4b247288d6cdff6783f8087a2a9039935f50af9099451cc841a3639c9625a18c\",\"nativePath\":\"/opt/qqq-supervisor/native/libextism.so\",\"nativeSha256\":\"44eb7ed56c6b5610b97b9aa62a436e6ff7e3175730e487c42bb67105185d7e01\",\"slots\":1,\"connections\":2,\"coldOperations\":1,\"entries\":8,\"codeBytes\":8388608,\"memoryBytes\":67108864,\"idleMillis\":30000,\"fuel\":100000000}" + } + }, + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": "execution-supervisor", + "namespace": "qqq-qual-unit" + }, + "spec": { + "replicas": 1, + "strategy": { + "type": "Recreate" + }, + "selector": { + "matchLabels": { + "app.kubernetes.io/name": "execution-supervisor" + } + }, + "template": { + "metadata": { + "labels": { + "app.kubernetes.io/name": "execution-supervisor" + }, + "annotations": { + "qqq.kingsrook.com/productionQualified": "false", + "qqq.kingsrook.com/redistributionCleared": "false", + "qqq.kingsrook.com/expected-pod-pids-limit": "256", + "qqq.kingsrook.com/pids-policy-evidence": "unit-node-policy", + "qqq.kingsrook.com/readiness-scope": "cold-listener-only-not-required-bindings" + } + }, + "spec": { + "automountServiceAccountToken": false, + "enableServiceLinks": false, + "runtimeClassName": "unit-sandbox", + "nodeSelector": { + "qualification-pool": "unit" + }, + "terminationGracePeriodSeconds": 30, + "securityContext": { + "runAsNonRoot": true, + "runAsUser": 65532, + "runAsGroup": 65532, + "fsGroup": 65532, + "seccompProfile": { + "type": "RuntimeDefault" + } + }, + "containers": [ + { + "name": "execution-supervisor", + "image": "registry.invalid/qqq/synthetic-execution@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "imagePullPolicy": "IfNotPresent", + "ports": [ + { + "name": "mtls", + "containerPort": 18443 + } + ], + "securityContext": { + "allowPrivilegeEscalation": false, + "readOnlyRootFilesystem": true, + "capabilities": { + "drop": [ + "ALL" + ] + } + }, + "resources": { + "requests": { + "cpu": "1000m", + "memory": "1024Mi", + "ephemeral-storage": "64Mi" + }, + "limits": { + "cpu": "1000m", + "memory": "1024Mi", + "ephemeral-storage": "128Mi" + } + }, + "startupProbe": { + "tcpSocket": { + "port": "mtls" + }, + "timeoutSeconds": 1, + "periodSeconds": 5, + "failureThreshold": 24 + }, + "readinessProbe": { + "tcpSocket": { + "port": "mtls" + }, + "timeoutSeconds": 1, + "periodSeconds": 5, + "failureThreshold": 1 + }, + "env": [ + { + "name": "QQQ_SUPERVISOR_KEYSTORE_PASSWORD", + "valueFrom": { + "secretKeyRef": { + "name": "unit-passwords", + "key": "keyStorePassword" + } + } + }, + { + "name": "QQQ_SUPERVISOR_TRUSTSTORE_PASSWORD", + "valueFrom": { + "secretKeyRef": { + "name": "unit-passwords", + "key": "trustStorePassword" + } + } + } + ], + "volumeMounts": [ + { + "name": "config", + "mountPath": "/run/qqq-supervisor/config", + "readOnly": true + }, + { + "name": "tls", + "mountPath": "/run/qqq-supervisor/tls", + "readOnly": true + }, + { + "name": "trust", + "mountPath": "/run/qqq-supervisor/trust", + "readOnly": true + }, + { + "name": "tmp", + "mountPath": "/tmp" + } + ] + } + ], + "volumes": [ + { + "name": "config", + "configMap": { + "name": "execution-supervisor", + "defaultMode": 292 + } + }, + { + "name": "tls", + "secret": { + "secretName": "unit-server", + "defaultMode": 288, + "items": [ + { + "key": "server.p12", + "path": "server.p12" + } + ] + } + }, + { + "name": "trust", + "secret": { + "secretName": "unit-clients", + "defaultMode": 288, + "items": [ + { + "key": "clients.p12", + "path": "clients.p12" + } + ] + } + }, + { + "name": "tmp", + "emptyDir": { + "medium": "Memory", + "sizeLimit": "16Mi" + } + } + ] + } + } + } + }, + { + "apiVersion": "v1", + "kind": "Service", + "metadata": { + "name": "execution-supervisor", + "namespace": "qqq-qual-unit" + }, + "spec": { + "type": "ClusterIP", + "selector": { + "app.kubernetes.io/name": "execution-supervisor" + }, + "ports": [ + { + "name": "mtls", + "port": 18443, + "targetPort": "mtls" + } + ] + } + }, + { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "default-deny", + "namespace": "qqq-qual-unit" + }, + "spec": { + "podSelector": {}, + "policyTypes": [ + "Ingress", + "Egress" + ], + "ingress": [], + "egress": [] + } + }, + { + "apiVersion": "networking.k8s.io/v1", + "kind": "NetworkPolicy", + "metadata": { + "name": "execution-supervisor", + "namespace": "qqq-qual-unit" + }, + "spec": { + "podSelector": { + "matchLabels": { + "app.kubernetes.io/name": "execution-supervisor" + } + }, + "policyTypes": [ + "Ingress" + ], + "ingress": [ + { + "from": [ + { + "namespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "unit-client" + } + }, + "podSelector": { + "matchLabels": { + "app": "unit-client" + } + } + } + ], + "ports": [ + { + "protocol": "TCP", + "port": 18443 + } + ] + } + ] + } + } + ] +} diff --git a/deploy/qualification.disabled.json b/deploy/qualification.disabled.json new file mode 100644 index 0000000..0f78d5a --- /dev/null +++ b/deploy/qualification.disabled.json @@ -0,0 +1,32 @@ +{ + "enabled": false, + "namespace": null, + "image": null, + "sandboxRuntimeClass": null, + "podSecurityVersion": null, + "tlsSecret": null, + "trustSecret": null, + "passwordSecret": null, + "certificateNamespaces": {}, + "clientNamespace": null, + "clientPodLabels": {}, + "nodeSelector": {}, + "profile": null, + "toolchain": null, + "evidence": null, + "requiredEvidence": null, + "cpuMillis": null, + "memoryMiB": null, + "pidLimit": null, + "pidPolicyEvidence": null, + "tmpMiB": null, + "startupSeconds": null, + "slots": null, + "connections": null, + "coldOperations": null, + "entries": null, + "codeBytes": null, + "guestMemoryMiB": null, + "idleMillis": null, + "fuel": null +} diff --git a/deploy/render_qualification.py b/deploy/render_qualification.py new file mode 100644 index 0000000..fa014c7 --- /dev/null +++ b/deploy/render_qualification.py @@ -0,0 +1,193 @@ +# Copyright (C) 2026 Kingsrook, LLC. +# SPDX-License-Identifier: Apache-2.0 +"""Render inert qualification JSON into a bounded Kubernetes List; never apply it.""" +import argparse +import json +import logging +from pathlib import Path +import re +from typing import Any + +from snapshot_execution import EXTISM_SHA, WASM_SHA + +NAME = r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?" +IDENTITY = r"[A-Za-z0-9._:-]{1,128}" +LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9_.-]{0,61}[A-Za-z0-9])?" +SHA = r"[a-f0-9]{64}" +REQUIRED = { + "enabled", "namespace", "image", "sandboxRuntimeClass", "podSecurityVersion", + "tlsSecret", "trustSecret", "passwordSecret", "certificateNamespaces", + "clientNamespace", "clientPodLabels", "nodeSelector", "profile", "toolchain", + "evidence", "requiredEvidence", "cpuMillis", "memoryMiB", "pidLimit", + "pidPolicyEvidence", "tmpMiB", "startupSeconds", "slots", "connections", + "coldOperations", "entries", "codeBytes", "guestMemoryMiB", "idleMillis", "fuel", +} + + +def require(condition: bool, message: str) -> None: + """Refuse malformed deployment intent without a fallback.""" + if not condition: + raise ValueError(message) + + +def text(value: Any, pattern: str) -> bool: + """Validate exact string types and the whole bounded value.""" + return isinstance(value, str) and re.fullmatch(pattern, value) is not None + + +def number(config: dict[str, Any], name: str, minimum: int, maximum: int) -> int: + """Reject booleans, coercion and values outside the candidate limits.""" + value = config[name] + require(type(value) is int and minimum <= value <= maximum, "Invalid bound: " + name) + return value + + +def labels(value: Any) -> dict[str, str]: + """Require explicit, nonempty selectors for client and node isolation.""" + require(isinstance(value, dict) and 1 <= len(value) <= 8, "Explicit nonempty selectors required") + for key, item in value.items(): + require(isinstance(key, str) and key.count("/") <= 1, "Invalid selector key") + parts = key.split("/") + if len(parts) == 2: + require(len(parts[0]) <= 253 and all(text(part, NAME) for part in parts[0].split(".")), + "Invalid selector DNS prefix") + require(text(parts[-1], LABEL) and text(item, LABEL), "Invalid selector") + return dict(value) + + +def render(config: dict[str, Any]) -> dict[str, Any]: + """Build namespace-scoped qualification objects from explicit operator intent.""" + require(isinstance(config, dict) and set(config) == REQUIRED, "Exact qualification fields required") + require(config["enabled"] is True, "Qualification is disabled") + for key in ("namespace", "sandboxRuntimeClass", "tlsSecret", "trustSecret", "passwordSecret", "clientNamespace"): + require(text(config[key], NAME), "Invalid name: " + key) + require(config["namespace"].startswith("qqq-qual-") and config["namespace"] != config["clientNamespace"], + "Use a separate explicit qqq-qual-* namespace") + require(config["sandboxRuntimeClass"] not in {"runc", "default"}, "An explicit sandbox RuntimeClass is required") + require(text(config["podSecurityVersion"], r"v1\.[0-9]{2}"), "Explicit Pod Security version required") + require(text(config["image"], r"[a-z0-9][a-z0-9./:_-]{1,240}@sha256:" + SHA), "Immutable image@sha256 required") + for key in ("profile", "toolchain", "evidence", "requiredEvidence", "pidPolicyEvidence"): + require(text(config[key], IDENTITY), "Explicit identity required: " + key) + require(not any(word in config[key].lower() for word in ("replace", "placeholder", "unqualified")), "Placeholder identity refused") + require(config["evidence"] == config["requiredEvidence"], "Evidence identity mismatch") + mappings = config["certificateNamespaces"] + require(isinstance(mappings, dict) and 1 <= len(mappings) <= 64, "Exact client certificate mappings required") + require(all(text(key, SHA) and text(value, IDENTITY) for key, value in mappings.items()), "Invalid certificate mapping") + client_labels = labels(config["clientPodLabels"]) + node_labels = labels(config["nodeSelector"]) + cpu = number(config, "cpuMillis", 100, 8000) + memory = number(config, "memoryMiB", 512, 16384) + pids = number(config, "pidLimit", 64, 2048) + temporary = number(config, "tmpMiB", 1, 256) + startup = number(config, "startupSeconds", 30, 600) + slots = number(config, "slots", 1, 64) + connections = number(config, "connections", slots, 128) + cold = number(config, "coldOperations", 1, 8) + entries = number(config, "entries", 1, 256) + code = number(config, "codeBytes", 1, 64 * 1024 * 1024) + guest = number(config, "guestMemoryMiB", 64, 256) + require(guest % 64 == 0, "Guest profile must use whole 64MiB steps") + require(memory >= slots * guest + 256 + temporary, "Insufficient minimum candidate memory headroom") + idle = number(config, "idleMillis", 100, 60000) + fuel = number(config, "fuel", 1, 1000000000) + namespace = config["namespace"] + name = "execution-supervisor" + selector = {"app.kubernetes.io/name": name} + annotations = { + "qqq.kingsrook.com/productionQualified": "false", + "qqq.kingsrook.com/redistributionCleared": "false", + "qqq.kingsrook.com/expected-pod-pids-limit": str(pids), + "qqq.kingsrook.com/pids-policy-evidence": config["pidPolicyEvidence"], + "qqq.kingsrook.com/readiness-scope": "cold-listener-only-not-required-bindings", + } + settings = { + "bind": "0.0.0.0", "port": 18443, + "keyStore": "/run/qqq-supervisor/tls/server.p12", "trustStore": "/run/qqq-supervisor/trust/clients.p12", + "certificateNamespaces": mappings, "profile": config["profile"], "toolchain": config["toolchain"], + "evidence": config["evidence"], "requiredEvidence": config["requiredEvidence"], + "runtimePath": "/opt/qqq-supervisor/runtime/qqq_javy_runtime.wasm", "runtimeSha256": WASM_SHA, + "nativePath": "/opt/qqq-supervisor/native/libextism.so", "nativeSha256": EXTISM_SHA, + "slots": slots, "connections": connections, "coldOperations": cold, "entries": entries, + "codeBytes": code, "memoryBytes": guest * 1024 * 1024, "idleMillis": idle, "fuel": fuel, + } + + def resource(kind: str, spec: dict[str, Any], api: str = "v1", extra: dict[str, Any] | None = None) -> dict[str, Any]: + """Give each object an explicit qualification namespace.""" + result = {"apiVersion": api, "kind": kind, "metadata": {"name": name, "namespace": namespace}, **spec} + if extra: + result["metadata"].update(extra) + return result + + probe = {"tcpSocket": {"port": "mtls"}, "timeoutSeconds": 1, "periodSeconds": 5} + container = { + "name": name, "image": config["image"], "imagePullPolicy": "IfNotPresent", + "ports": [{"name": "mtls", "containerPort": 18443}], + "securityContext": {"allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, + "capabilities": {"drop": ["ALL"]}}, + "resources": {"requests": {"cpu": str(cpu) + "m", "memory": str(memory) + "Mi", "ephemeral-storage": "64Mi"}, + "limits": {"cpu": str(cpu) + "m", "memory": str(memory) + "Mi", "ephemeral-storage": "128Mi"}}, + "startupProbe": {**probe, "failureThreshold": (startup + 4) // 5}, + "readinessProbe": {**probe, "failureThreshold": 1}, + "env": [{"name": "QQQ_SUPERVISOR_" + key + "_PASSWORD", + "valueFrom": {"secretKeyRef": {"name": config["passwordSecret"], "key": field}}} + for key, field in (("KEYSTORE", "keyStorePassword"), ("TRUSTSTORE", "trustStorePassword"))], + "volumeMounts": [{"name": "config", "mountPath": "/run/qqq-supervisor/config", "readOnly": True}, + {"name": "tls", "mountPath": "/run/qqq-supervisor/tls", "readOnly": True}, + {"name": "trust", "mountPath": "/run/qqq-supervisor/trust", "readOnly": True}, + {"name": "tmp", "mountPath": "/tmp"}], + } + pod = {"automountServiceAccountToken": False, "enableServiceLinks": False, + "runtimeClassName": config["sandboxRuntimeClass"], "nodeSelector": node_labels, + "terminationGracePeriodSeconds": 30, + "securityContext": {"runAsNonRoot": True, "runAsUser": 65532, "runAsGroup": 65532, + "fsGroup": 65532, "seccompProfile": {"type": "RuntimeDefault"}}, + "containers": [container], + "volumes": [{"name": "config", "configMap": {"name": name, "defaultMode": 292}}, + {"name": "tls", "secret": {"secretName": config["tlsSecret"], "defaultMode": 288, + "items": [{"key": "server.p12", "path": "server.p12"}]}}, + {"name": "trust", "secret": {"secretName": config["trustSecret"], "defaultMode": 288, + "items": [{"key": "clients.p12", "path": "clients.p12"}]}}, + {"name": "tmp", "emptyDir": {"medium": "Memory", "sizeLimit": str(temporary) + "Mi"}}]} + items = [ + {"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": namespace, "labels": { + "pod-security.kubernetes.io/enforce": "restricted", "pod-security.kubernetes.io/enforce-version": config["podSecurityVersion"], + "pod-security.kubernetes.io/audit": "restricted", "pod-security.kubernetes.io/warn": "restricted"}}}, + resource("ConfigMap", {"immutable": True, "data": {"config.json": json.dumps(settings, separators=(",", ":"))}}), + resource("Deployment", {"spec": {"replicas": 1, "strategy": {"type": "Recreate"}, "selector": {"matchLabels": selector}, + "template": {"metadata": {"labels": selector, "annotations": annotations}, "spec": pod}}}, "apps/v1"), + resource("Service", {"spec": {"type": "ClusterIP", "selector": selector, + "ports": [{"name": "mtls", "port": 18443, "targetPort": "mtls"}]}}), + resource("NetworkPolicy", {"spec": {"podSelector": {}, "policyTypes": ["Ingress", "Egress"], "ingress": [], "egress": []}}, + "networking.k8s.io/v1", {"name": "default-deny"}), + resource("NetworkPolicy", {"spec": {"podSelector": {"matchLabels": selector}, "policyTypes": ["Ingress"], + "ingress": [{"from": [{"namespaceSelector": {"matchLabels": {"kubernetes.io/metadata.name": config["clientNamespace"]}}, + "podSelector": {"matchLabels": client_labels}}], + "ports": [{"protocol": "TCP", "port": 18443}]}]}}, "networking.k8s.io/v1"), + ] + return {"apiVersion": "v1", "kind": "List", "items": items} + + +def no_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Refuse ambiguous policy documents.""" + result = {} + for key, value in pairs: + require(key not in result, "Duplicate JSON key") + result[key] = value + return result + + +def main() -> None: + """Write a new local artifact; never contact Kubernetes.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + parser.add_argument("output", type=Path) + args = parser.parse_args() + require(args.input.stat().st_size <= 32768, "Input too large") + manifest = render(json.loads(args.input.read_text(encoding="utf-8"), object_pairs_hook=no_duplicates)) + with args.output.open("x", encoding="utf-8") as stream: + stream.write(json.dumps(manifest, indent=2) + "\n") + logging.warning("Qualification manifest rendered; not deployed or qualified") + + +if __name__ == "__main__": + main() diff --git a/deploy/snapshot_execution.py b/deploy/snapshot_execution.py new file mode 100644 index 0000000..68befab --- /dev/null +++ b/deploy/snapshot_execution.py @@ -0,0 +1,181 @@ +# Copyright (C) 2026 Kingsrook, LLC. +# SPDX-License-Identifier: Apache-2.0 +"""Copy a closed, hash-verified local candidate context. Never build or download.""" +import argparse +import hashlib +import json +import logging +from pathlib import Path +import shutil +from typing import Any +import zipfile + +ROOT = Path(__file__).resolve().parent +WASM_SHA = "4b247288d6cdff6783f8087a2a9039935f50af9099451cc841a3639c9625a18c" +EXTISM_SHA = "44eb7ed56c6b5610b97b9aa62a436e6ff7e3175730e487c42bb67105185d7e01" +JNA_SHA = "260c4b1e22b1db9e110ee441c4f13ce115f841fa48c41d78750986214b395557" +JNA_JNI_SHA = "ca07953d595210082339753d9e818a1fdb40509a17a41914d9a2cb0d2df6b6af" +JSON_SHA = "2a3438fbf9293174ea82bdfa35e83d8b965cfe6bddc034a1ac5d60107174c209" +FILES = { + "supervisor": ("payload/lib/supervisor.jar", None), + "jna": ("payload/lib/jna-5.18.1.jar", JNA_SHA), + "json": ("payload/lib/json-20251224.jar", JSON_SHA), + "extism": ("payload/native/libextism.so", EXTISM_SHA), + "wasm": ("payload/runtime/qqq_javy_runtime.wasm", WASM_SHA), + "license": ("metadata/LICENSE", None), + "notice": ("metadata/NOTICE", None), + "licenseReview": ("metadata/release-license-review.md", None), + "cargoInventory": ("metadata/license-inventory.json", None), +} + + +def digest(path: Path) -> str: + """Hash local bytes without loading entire native binaries into memory.""" + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def snapshot(inputs: dict[str, Any], destination: Path) -> dict[str, Any]: + """Verify the closed input set, then copy into a new cache directory.""" + if set(inputs) != set(FILES): + raise ValueError("Exact named local inputs required") + # Validate everything before creating the previously nonexistent context. + verified = {} + for name, (relative, pinned) in FILES.items(): + item = inputs[name] + if not isinstance(item, dict) or set(item) != {"path", "sha256"}: + raise ValueError("Each input needs only path and SHA256") + path = Path(item["path"]) + expected = item["sha256"] + if (not path.is_absolute() or not path.is_file() or path.is_symlink() + or path.stat().st_size > 256 * 1024 * 1024 + or not isinstance(expected, str) or len(expected) != 64 + or any(c not in "0123456789abcdef" for c in expected) + or (pinned is not None and expected != pinned) + or digest(path) != expected): + raise ValueError("Local input verification failed: " + name) + verified[name] = (path, relative, expected) + destination = Path(destination) + cache = (Path.home() / ".cache").resolve() + if (not destination.is_absolute() or destination.exists() + or not destination.resolve().is_relative_to(cache)): + raise ValueError("A new absolute output directory inside the current user's cache is required") + with zipfile.ZipFile(verified["supervisor"][0]) as jar: + names = jar.namelist() + if ("com/kingsrook/scripting/supervisor/SupervisorMain.class" not in names + or any(n.endswith(".class") and not n.startswith("com/kingsrook/scripting/supervisor/") for n in names)): + raise ValueError("Standalone supervisor JAR required; QQQ/QBit classes refused") + with zipfile.ZipFile(verified["jna"][0]) as jar: + jni = jar.read("com/sun/jna/linux-x86-64/libjnidispatch.so") + if (jni[:6] != b"\x7fELF\x02\x01" or jni[18:20] != b"\x3e\x00" + or len(jni) > 4 * 1024 * 1024 or hashlib.sha256(jni).hexdigest() != JNA_JNI_SHA): + raise ValueError("Pinned Linux JNI payload invalid") + with verified["extism"][0].open("rb") as stream: + header = stream.read(20) + if header[:6] != b"\x7fELF\x02\x01" or header[18:20] != b"\x3e\x00": + raise ValueError("Linux x86-64 Extism required") + with verified["wasm"][0].open("rb") as stream: + if stream.read(8) != b"\0asm\x01\0\0\0": + raise ValueError("Fixed Wasm header invalid") + destination.mkdir(parents=True) + records = [] + for name, (source, relative, expected) in verified.items(): + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + if digest(target) != expected: + raise ValueError("Snapshot changed during copy: " + name) + records.append({"path": relative, "sha256": expected, "bytes": target.stat().st_size}) + jni_path = destination / "payload/native/libjnidispatch.so" + jni_path.write_bytes(jni) + records.append({"path": jni_path.relative_to(destination).as_posix(), + "sha256": digest(jni_path), "bytes": len(jni)}) + for filename in ("execution.Containerfile", "NOTICE-GAPS.md"): + relative = "Containerfile" if filename.endswith("Containerfile") else "metadata/NOTICE-GAPS.md" + shutil.copyfile(ROOT / filename, destination / relative) + records.append({"path": relative, "sha256": digest(destination / relative)}) + (destination / ".dockerignore").write_text("*\n!Containerfile\n!payload/\n!payload/**\n!metadata/\n!metadata/**\n", encoding="utf-8") + records.append({"path": ".dockerignore", "sha256": digest(destination / ".dockerignore")}) + manifest = {"productionQualified": False, "redistributionCleared": False, + "architecture": "linux/amd64", "files": records, + "supervisorClasses": sorted(n for n in names if n.endswith(".class")), + "jnaNativeEntry": "com/sun/jna/linux-x86-64/libjnidispatch.so", + "nativeSha256": EXTISM_SHA, "runtimeSha256": WASM_SHA, + "supervisorSha256": verified["supervisor"][2], "jnaJniSha256": digest(jni_path)} + (destination / "metadata/candidate.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + verify_snapshot(destination) + return manifest + + +def verify_snapshot(directory: Path) -> None: + """Refuse unexpected context files or changed bytes immediately before a build.""" + directory = directory.resolve(strict=True) + manifest_path = directory / "metadata/candidate.json" + if manifest_path.stat().st_size > 128 * 1024: + raise ValueError("Candidate manifest too large") + manifest = json.loads(manifest_path.read_text(encoding="utf-8"), object_pairs_hook=no_duplicates) + expected = {relative for relative, _ in FILES.values()} | { + "payload/native/libjnidispatch.so", "Containerfile", "metadata/NOTICE-GAPS.md", ".dockerignore", + } + rows = manifest["files"] + if (manifest["productionQualified"] is not False or manifest["redistributionCleared"] is not False + or manifest["architecture"] != "linux/amd64" or manifest["runtimeSha256"] != WASM_SHA + or manifest["nativeSha256"] != EXTISM_SHA or not isinstance(rows, list) + or len(rows) != len(expected) or {row["path"] for row in rows} != expected): + raise ValueError("Invalid candidate inventory") + actual = set() + for path in directory.rglob("*"): + if path.is_symlink() or not path.resolve().is_relative_to(directory): + raise ValueError("Links are not permitted in the build context") + if path.is_file(): + actual.add(path.relative_to(directory).as_posix()) + if actual != expected | {"metadata/candidate.json"}: + raise ValueError("Unexpected or missing build-context file") + hashes = {row["path"]: row["sha256"] for row in rows} + for relative in expected: + if digest(directory / relative) != hashes[relative]: + raise ValueError("Changed build-context bytes") + for relative, pinned in FILES.values(): + if pinned is not None and hashes[relative] != pinned: + raise ValueError("Pinned input identity changed") + if (hashes["payload/lib/supervisor.jar"] != manifest["supervisorSha256"] + or hashes["payload/native/libjnidispatch.so"] != manifest["jnaJniSha256"] + or manifest["jnaJniSha256"] != JNA_JNI_SHA + or hashes["Containerfile"] != digest(ROOT / "execution.Containerfile") + or hashes["metadata/NOTICE-GAPS.md"] != digest(ROOT / "NOTICE-GAPS.md")): + raise ValueError("Candidate linkage mismatch") + + +def no_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Reject ambiguous input descriptors rather than last-key-wins parsing.""" + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate input key") + result[key] = value + return result + + +def main() -> None: + """Create local packaging evidence without a build or network operation.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("inputs", type=Path, nargs="?") + parser.add_argument("output", type=Path, nargs="?") + parser.add_argument("--verify", type=Path) + args = parser.parse_args() + if args.verify is not None: + if args.inputs is not None or args.output is not None: + raise ValueError("Verification accepts only the existing context directory") + verify_snapshot(args.verify) + logging.warning("Closed candidate context verified; no build performed") + return + if args.inputs is None or args.output is None: + raise ValueError("Local input descriptor and new cache output are required") + if args.inputs.stat().st_size > 32 * 1024: + raise ValueError("Input descriptor too large") + snapshot(json.loads(args.inputs.read_text(encoding="utf-8"), object_pairs_hook=no_duplicates), args.output) + logging.warning("Local candidate snapshot verified; unqualified; no build performed") + + +if __name__ == "__main__": + main() diff --git a/deploy/test_deploy.py b/deploy/test_deploy.py new file mode 100644 index 0000000..d2fbd55 --- /dev/null +++ b/deploy/test_deploy.py @@ -0,0 +1,296 @@ +# Copyright (C) 2026 Kingsrook, LLC. +# SPDX-License-Identifier: Apache-2.0 +"""Static rendering and fail-closed snapshot tests; no containers or clusters.""" +import copy +import hashlib +import json +from pathlib import Path +import re +import subprocess +import sys +import tempfile +from typing import Any +import unittest + +import render_qualification as renderer +import snapshot_execution as snapshot + + +def fixture() -> dict[str, Any]: + """Supply synthetic identifiers for rendering tests only.""" + # Synthetic non-secret identifiers; fixtures are never applied. + return { + "enabled": True, "namespace": "qqq-qual-unit", "image": "invalid.example/unit@sha256:" + "a" * 64, + "sandboxRuntimeClass": "unit-sandbox", "podSecurityVersion": "v1.34", + "tlsSecret": "unit-server", "trustSecret": "unit-clients", "passwordSecret": "unit-passwords", + "certificateNamespaces": {"b" * 64: "unit-client"}, "clientNamespace": "unit-client", + "clientPodLabels": {"app": "unit-client"}, "nodeSelector": {"qualification-pool": "unit"}, + "profile": "unit-profile", "toolchain": "unit-toolchain", "evidence": "unit-evidence", + "requiredEvidence": "unit-evidence", "cpuMillis": 1000, "memoryMiB": 1024, "pidLimit": 256, + "pidPolicyEvidence": "unit-node-policy", "tmpMiB": 16, "startupSeconds": 120, + "slots": 1, "connections": 2, "coldOperations": 1, "entries": 8, "codeBytes": 8388608, + "guestMemoryMiB": 64, "idleMillis": 30000, "fuel": 100000000, + } + + +class DeploymentTest(unittest.TestCase): + """Verify generated boundaries without a deployment or native runtime.""" + + def test_disabled_input_and_every_missing_field_refuse(self) -> None: + """An incomplete descriptor never yields runnable manifests.""" + disabled = json.loads((Path(__file__).parent / "qualification.disabled.json").read_text()) + with self.assertRaises(ValueError): + renderer.render(disabled) + for field in renderer.REQUIRED: + value = fixture() + value.pop(field) + with self.subTest(field=field), self.assertRaises(ValueError): + renderer.render(value) + + def test_fixed_security_network_and_exact_supervisor_config(self) -> None: + """Keep secrets referenced and network authority narrowly selected.""" + manifest = renderer.render(fixture()) + by_kind = {entry["kind"]: entry for entry in manifest["items"] if entry["kind"] != "NetworkPolicy"} + pod = by_kind["Deployment"]["spec"]["template"]["spec"] + container = pod["containers"][0] + self.assertFalse(pod["automountServiceAccountToken"]) + self.assertFalse(pod["enableServiceLinks"]) + self.assertEqual(65532, pod["securityContext"]["runAsUser"]) + self.assertEqual({"type": "RuntimeDefault"}, pod["securityContext"]["seccompProfile"]) + self.assertEqual(["ALL"], container["securityContext"]["capabilities"]["drop"]) + self.assertTrue(container["securityContext"]["readOnlyRootFilesystem"]) + self.assertFalse(container["securityContext"]["allowPrivilegeEscalation"]) + self.assertNotIn("livenessProbe", container) + self.assertNotIn("pidsLimit", pod) + self.assertTrue(all("valueFrom" in item and "value" not in item for item in container["env"])) + settings = json.loads(by_kind["ConfigMap"]["data"]["config.json"]) + self.assertEqual(snapshot.WASM_SHA, settings["runtimeSha256"]) + self.assertEqual(snapshot.EXTISM_SHA, settings["nativeSha256"]) + self.assertEqual("0.0.0.0", settings["bind"]) + self.assertFalse(any("password" in key.lower() for key in settings)) + policies = [item["spec"] for item in manifest["items"] if item["kind"] == "NetworkPolicy"] + self.assertEqual([], policies[0]["egress"]) + self.assertEqual([], policies[0]["ingress"]) + peer = policies[1]["ingress"][0]["from"] + self.assertEqual(1, len(peer)) + self.assertEqual({"namespaceSelector", "podSelector"}, set(peer[0])) + self.assertEqual([{"protocol": "TCP", "port": 18443}], policies[1]["ingress"][0]["ports"]) + self.assertNotIn("Secret", by_kind) + self.assertNotIn("RuntimeClass", by_kind) + + def test_malformed_policy_cannot_fall_back(self) -> None: + """Reject permissive substitutes and mixed JSON types.""" + changes = {"image": "example/unit:latest", "namespace": "production", "sandboxRuntimeClass": "runc", + "certificateNamespaces": {"*": "unit"}, "clientPodLabels": {}, "nodeSelector": {}, + "pidLimit": 0, "cpuMillis": True, "memoryMiB": 32, "slots": 65, + "requiredEvidence": "different", "enabled": "true", "profile": "REPLACE_ME"} + for field, value in changes.items(): + config = fixture() + config[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + renderer.render(config) + config = fixture() + config["secretValue"] = "not-accepted" + with self.assertRaises(ValueError): + renderer.render(config) + + def test_input_is_not_mutated_and_duplicate_json_refused(self) -> None: + """Preserve caller intent and reject ambiguous repeated keys.""" + config = fixture() + before = copy.deepcopy(config) + renderer.render(config) + self.assertEqual(before, config) + for hook in (renderer.no_duplicates, snapshot.no_duplicates): + with self.assertRaises(ValueError): + json.loads('{"enabled":true,"enabled":false}', object_pairs_hook=hook) + + def test_wire_identity_and_kubernetes_label_grammars_refuse_invalid_values(self) -> None: + """Refuse values accepted previously but rejected by the real consumers.""" + for value in ("wrong/profile", "x" * 129): + for key in ("profile", "toolchain", "evidence", "requiredEvidence"): + config = fixture() + config[key] = value + if key in {"evidence", "requiredEvidence"}: + config["evidence"] = config["requiredEvidence"] = value + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + renderer.render(config) + config = fixture() + config["certificateNamespaces"] = {"b" * 64: value} + with self.assertRaises(ValueError): + renderer.render(config) + for selectors in ({"app-": "client"}, {"app": "client."}, {"bad..domain/app": "client"}, + {"-domain/app": "client"}, {"a" * 64 + "/app": "client"}): + for key in ("clientPodLabels", "nodeSelector"): + config = fixture() + config[key] = selectors + with self.subTest(key=key, selectors=selectors), self.assertRaises(ValueError): + renderer.render(config) + config = fixture() + config["profile"] = "x" * 128 + config["clientPodLabels"] = {"qualified.example/app_name": "client.v1"} + renderer.render(config) + + def test_complete_positive_fixture_cross_object_contracts(self) -> None: + """Review the entire rendered fixture and its actual mount/selector linkage.""" + directory = Path(__file__).parent / "fixtures" + source = json.loads((directory / "synthetic-input.json").read_text(encoding="utf-8")) + expected = json.loads((directory / "synthetic-manifest.json").read_text(encoding="utf-8")) + actual = renderer.render(source) + self.assertEqual(expected, actual) + self.assertEqual(6, len(actual["items"])) + identities = [(item["kind"], item["metadata"].get("namespace"), item["metadata"]["name"]) for item in actual["items"]] + self.assertEqual(len(identities), len(set(identities))) + by_kind = {item["kind"]: item for item in actual["items"] if item["kind"] != "NetworkPolicy"} + namespace = by_kind["Namespace"] + self.assertNotIn("namespace", namespace["metadata"]) + self.assertEqual("restricted", namespace["metadata"]["labels"]["pod-security.kubernetes.io/enforce"]) + for item in actual["items"]: + if item["kind"] != "Namespace": + self.assertEqual(namespace["metadata"]["name"], item["metadata"]["namespace"]) + deployment = by_kind["Deployment"]["spec"] + template = deployment["template"] + pod = template["spec"] + container = pod["containers"][0] + self.assertEqual(deployment["selector"]["matchLabels"], template["metadata"]["labels"]) + self.assertEqual(template["metadata"]["labels"], by_kind["Service"]["spec"]["selector"]) + self.assertEqual("false", template["metadata"]["annotations"]["qqq.kingsrook.com/productionQualified"]) + self.assertEqual("false", template["metadata"]["annotations"]["qqq.kingsrook.com/redistributionCleared"]) + self.assertEqual(source["cpuMillis"], int(container["resources"]["limits"]["cpu"][:-1])) + self.assertEqual(container["resources"]["requests"]["memory"], container["resources"]["limits"]["memory"]) + self.assertEqual("ClusterIP", by_kind["Service"]["spec"]["type"]) + self.assertNotIn("externalIPs", by_kind["Service"]["spec"]) + self.assertEqual("mtls", by_kind["Service"]["spec"]["ports"][0]["targetPort"]) + self.assertEqual(18443, container["ports"][0]["containerPort"]) + config_map = by_kind["ConfigMap"] + settings = json.loads(config_map["data"]["config.json"]) + self.assertTrue(config_map["immutable"]) + volumes = {volume["name"]: volume for volume in pod["volumes"]} + mounts = {mount["name"]: mount for mount in container["volumeMounts"]} + self.assertEqual(set(volumes), set(mounts)) + self.assertEqual(config_map["metadata"]["name"], volumes["config"]["configMap"]["name"]) + for name, setting, secret_field in (("tls", "keyStore", "tlsSecret"), ("trust", "trustStore", "trustSecret")): + secret = volumes[name]["secret"] + self.assertEqual(source[secret_field], secret["secretName"]) + self.assertEqual(0o440, secret["defaultMode"]) + self.assertTrue(mounts[name]["readOnly"]) + self.assertEqual(mounts[name]["mountPath"] + "/" + secret["items"][0]["path"], settings[setting]) + self.assertEqual(0o444, volumes["config"]["configMap"]["defaultMode"]) + self.assertEqual({"medium": "Memory", "sizeLimit": str(source["tmpMiB"]) + "Mi"}, volumes["tmp"]["emptyDir"]) + for forbidden in ("hostPID", "hostIPC", "hostNetwork", "hostUsers", "initContainers", "ephemeralContainers"): + self.assertNotIn(forbidden, pod) + self.assertFalse(any("hostPath" in value or "persistentVolumeClaim" in value for value in volumes.values())) + self.assertEqual(30, pod["terminationGracePeriodSeconds"]) + self.assertNotIn("livenessProbe", container) + + def test_actual_cli_renders_fixture_once_and_refuses_disabled_without_output(self) -> None: + """Exercise file parsing and exclusive output creation without cluster tools.""" + root = Path(__file__).parent + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "manifest.json" + command = [sys.executable, str(root / "render_qualification.py"), str(root / "fixtures/synthetic-input.json"), str(output)] + first = subprocess.run(command, capture_output=True, timeout=10, check=False) + self.assertEqual(0, first.returncode) + self.assertEqual(json.loads((root / "fixtures/synthetic-manifest.json").read_text(encoding="utf-8")), + json.loads(output.read_text(encoding="utf-8"))) + before = output.read_bytes() + second = subprocess.run(command, capture_output=True, timeout=10, check=False) + self.assertNotEqual(0, second.returncode) + self.assertEqual(before, output.read_bytes()) + disabled_output = Path(temporary) / "disabled.json" + refused = subprocess.run([sys.executable, str(root / "render_qualification.py"), str(root / "qualification.disabled.json"), + str(disabled_output)], capture_output=True, timeout=10, check=False) + self.assertNotEqual(0, refused.returncode) + self.assertFalse(disabled_output.exists()) + + def test_rendered_config_matches_actual_closed_java_entrypoint(self) -> None: + """Catch drift between packaging and the real SupervisorMain schema.""" + java = (Path(__file__).parent.parent / "supervisor/src/main/java/com/kingsrook/scripting/supervisor/SupervisorMain.java").read_text(encoding="utf-8") + matched = re.search(r"Protocol\.keys\(config, (.*?)\);", java, re.DOTALL) + self.assertIsNotNone(matched) + assert matched is not None + accepted = set(re.findall(r'"([A-Za-z0-9]+)"', matched.group(1))) + config_map = next(item for item in renderer.render(fixture())["items"] if item["kind"] == "ConfigMap") + self.assertEqual(accepted, set(json.loads(config_map["data"]["config.json"]))) + + def test_context_verifier_refuses_unlisted_file(self) -> None: + """Even a harmless extra context file cannot silently enter the image.""" + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + rows = [] + files = {relative for relative, _ in snapshot.FILES.values()} | { + "payload/native/libjnidispatch.so", "Containerfile", "metadata/NOTICE-GAPS.md", ".dockerignore", + } + for relative in files: + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(b"bounded test bytes") + rows.append({"path": relative, "sha256": snapshot.digest(target)}) + manifest = {"productionQualified": False, "redistributionCleared": False, "architecture": "linux/amd64", + "runtimeSha256": snapshot.WASM_SHA, "nativeSha256": snapshot.EXTISM_SHA, "files": rows} + (root / "metadata/candidate.json").write_text(json.dumps(manifest), encoding="utf-8") + (root / "unlisted.txt").write_text("must refuse", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "Unexpected or missing build-context file"): + snapshot.verify_snapshot(root) + + def test_snapshot_missing_unknown_and_hash_mismatch_refuse_before_output(self) -> None: + """Do not create a context after any input-integrity failure.""" + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "output" + for descriptor in ({}, {"certificate": {}}): + with self.assertRaises(ValueError): + snapshot.snapshot(descriptor, output) + source = Path(temporary) / "source" + source.write_bytes(b"fixed harmless fixture") + descriptor = {key: {"path": str(source), "sha256": "0" * 64} for key in snapshot.FILES} + with self.assertRaises(ValueError): + snapshot.snapshot(descriptor, output) + self.assertFalse(output.exists()) + + +class PinnedWasmTest(unittest.TestCase): + """The fixed Wasm pin must always correspond to the recorded source inputs.""" + + RUNTIME = Path(__file__).resolve().parent.parent / "wasm-runtime" + + def pin(self) -> dict[str, Any]: + """Load the recorded pin.""" + return json.loads((self.RUNTIME / "PINNED-WASM.json").read_text(encoding="utf-8")) + + def test_deploy_constant_and_inventories_equal_the_recorded_pin(self) -> None: + """Every place that names the fixed Wasm hash must agree with PINNED-WASM.json.""" + pin = self.pin() + self.assertEqual(pin["sha256"], snapshot.WASM_SHA) + inventory = json.loads((self.RUNTIME / "docs" / "license-inventory.json").read_text(encoding="utf-8")) + imports = json.loads((self.RUNTIME / "docs" / "wasm-imports.json").read_text(encoding="utf-8")) + self.assertEqual(pin["sha256"], inventory["artifactSha256"]) + self.assertEqual(pin["sha256"], imports["sha256"]) + self.assertEqual(pin["bytes"], imports["bytes"]) + review = (self.RUNTIME / "docs" / "release-license-review.md").read_text(encoding="utf-8") + self.assertIn(pin["sha256"], review) + + def test_pin_matches_the_current_source_inputs_and_lockfile(self) -> None: + """A source or Cargo.lock edit without a reviewed re-pin must fail here, not silently drift.""" + pin = self.pin() + combined = "" + for name in pin["sourceInputs"]: + digest = hashlib.sha256((self.RUNTIME / name).read_bytes()).hexdigest() + combined += name + "\n" + digest + "\n" + self.assertEqual(pin["sourceInputsSha256"], hashlib.sha256(combined.encode("utf-8")).hexdigest()) + self.assertEqual(pin["cargoLockSha256"], hashlib.sha256((self.RUNTIME / "Cargo.lock").read_bytes()).hexdigest()) + + def test_pin_names_locked_toolchain_and_previous_artifact(self) -> None: + """The recipe inputs are explicit: no unpinned tool, and the superseded hash stays recorded.""" + pin = self.pin() + self.assertRegex(pin["sha256"], r"^[0-9a-f]{64}$") + self.assertNotEqual(pin["sha256"], pin["previous"]["sha256"]) + self.assertEqual("1.97.1", pin["toolchain"]["rust"]) + for key in ("archiveSha256", "clangExeSha256", "libclangDllSha256", "wasiLibcSha256"): + self.assertRegex(pin["toolchain"]["wasiSdk"][key], r"^[0-9a-f]{64}$") + self.assertFalse(pin["qualification"].count("productionQualified=true")) + script = (self.RUNTIME / "Build-Wasm.ps1").read_text(encoding="utf-8") + for fragment in ("--offline", "--locked", "OutputDir must not already exist", "does not match the pin"): + self.assertIn(fragment, script) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/AUTHORING-PREPARATION.md b/docs/AUTHORING-PREPARATION.md new file mode 100644 index 0000000..6205ff3 --- /dev/null +++ b/docs/AUTHORING-PREPARATION.md @@ -0,0 +1,17 @@ +# Authoring preparation + +An optional `WarmWorkerPreparation` provider receives only coordinator-verified immutable bundle bytes, their SHA-256, the original `.ts` entry point and toolchain identity. It receives no actor, record data, capability token or invocation. The SPI is not a qualification or readiness guarantee. + +Explicit test/context-test, publish, bind and enabling an existing binding may prepare code before dispatch or mutation. Draft saving, compilation, ordinary execution and record hooks never prepare code. Preparation refuses ambient transactions and recursive worker/preparation callbacks. The service rechecks the original session/instance/provider, current actor permissions, native project/target access, signature and exact build/artifact/publication association after external preparation and again at the dispatch/write boundary. Preparation failure leaves existing publication activation and bindings intact. Partial remote warming may safely remain, since it grants no authority. + +`PersistentWorkerProvider(Configuration)` retains source compatibility and prepares one copy. The additive `(Configuration, int preparationCopies)` constructor accepts only 1 through the configured execution connection count. The SPI checks the operator toolchain and delegates to the existing bounded cold admission/deadline path; the operator profile is pinned by the existing authenticated handshake. Warm execute has no new fallback or cold work. Non-cache providers continue using the existing lifecycle without implementing this SPI. + +No new table, registry, queue or runtime is introduced. Native startup/reconnect reconciliation is now implemented separately in [WARM-LIFECYCLE.md](WARM-LIFECYCLE.md); deployed cache capacity/eviction and supervisor qualification remain gates; publication does not promise perpetual residency or a functioning compiler. A miss still refuses rather than silently compiling inside a record write. + +Verified on 2026-10-31: focused 48 tests passed with normal loopback access (11 new authoring cases, 12 TLS transport cases, 25 existing lifecycle/hook/context cases), followed by the full 173-test main suite with zero failures/errors/skips, Checkstyle and configured coverage gates. Main JAR SHA-256: `B1F62E12FA4BFC5884AEF0E20DCD9217EEAEBA86F8D992BEF619CFF490039778`. No install or deployment occurred. + +The evidence preserves five separate runs: initial import-order Checkstyle failure; restricted-loopback focused run (48 tests, 2 failures and 7 errors); unchanged single TLS case under normal loopback access (1 pass); normal-loopback focused suite (48 pass); full main verify (173 pass). The environment control did not change TLS settings or assertions. Logs and their exact hashes distinguish these scopes; a passing retry is not represented as a product TLS fix. + +Preparation rechecks existing native credential/client-channel write ceilings before transfer and afterward. Enabling a binding also pins its capabilities JSON. A callback that installs an unexpected ambient transaction is refused and the original absent ambient reference is restored without committing, rolling back or closing that collaborator-owned transaction. If preparation itself throws, that primary failure is preserved. + +The source/JAR and filtered-log evidence is sealed separately in `docs/evidence/authoring-preparation-20261031-1050`. Full raw logs remain local at the hashed paths; they are not copied into the proof because framework diagnostics can include synthetic session identifiers. Prior 161-test/JFR bundles remain unchanged. Native service/mutual-TLS tests do not qualify runtime isolation, Kubernetes readiness or a durable database deployment; separate fixed trusted Wasm trials are not production qualification. diff --git a/docs/DURABLE-AFTER-HOOKS.md b/docs/DURABLE-AFTER-HOOKS.md new file mode 100644 index 0000000..e4b81cd --- /dev/null +++ b/docs/DURABLE-AFTER-HOOKS.md @@ -0,0 +1,239 @@ +# Durable AFTER record hooks + +Status: full base verification passed 2026-10-31 against the root-verified core/RDBMS +candidate in a dedicated local repository: 145 tests, no failures/errors/skips, +zero Checkstyle violations, unchanged instruction/branch coverage checks passed. +The 13 native file-H2 AFTER tests use the real transactional Jobs owner/runtime +and a trusted protocol worker. The corrected optional provider gate also passed all 44 tests: 13 file-H2, +14 actual AFTER/MySQL and 17 baseline MySQL. This is not a hostile-worker or +deployment qualification. +No new queue, script registry, database store or generation dependency was added. + +## Event semantics + +Add distinct AFTER_INSERT, AFTER_UPDATE and AFTER_DELETE phases. Existing PRE and +POST behavior is unchanged: POST observes the backend action before its caller +may commit, and a warning cannot undo that action. AFTER is dispatched from a +Jobs intent enlisted in the exact source transaction and becomes deliverable only +after commit. Enlistment failure refuses the required mutation; delivery failure +cannot roll back its committed source. Preview never enlists or executes AFTER. + +INSERT/UPDATE retain only a bounded locator and immutable script pin. The worker +receives a newly authorized, field-projected record at delivery time, explicitly +not the historical mutation snapshot. A deleted/missing/inaccessible current row +refuses execution. DELETE is an event-metadata notification, not an old-record +snapshot: no deleted values or inferred missing-row readability. Current native +table/action/project/run authority is still mandatory. Jobs parameters contain +no source record field snapshot, passwords, BLOBs, hidden fields or credentials. + +## Minimal components + +- Extend HookEvent and ALLOW-only result validation. AFTER cannot patch or veto; + the broker remains bounded read-only. SDK event metadata explains delivery + semantics; authoring discovery/test/binding uses the existing APIs. +- An opt-in host metadata helper installs a required transactional record listener + for selected AFTER phases, preserving existing Java customizers and refusing + conflicting required listeners rather than silently replacing them. +- The listener resolves the active immutable binding, publication/build/artifact + pin, checks current caller authority and bounds the action/locator count. A + core-owned UUID identifies this exact active native event invocation; per-record request + keys include that identity, binding ID and ordinal. Repeated callback enlistment + therefore reuses the same owner key without a consumer registry. Identical later writes + are different events. Re-delivery of the persisted job preserves its event ID. +- A single fixed Java backend process (stable transactional process revision) + consumes only a core-verified currently leased enlisted intent. Direct process + invocation and an ordinary launched Jobs process with forged parameters fail. + The consumer must not duplicate core grant/lease verification or trust params + as credentials. +- Delivery checks current restored actor, native table/action and project access, + enabled exact binding, immutable publication/build/artifact hashes, field + policy and qualified worker. INSERT/UPDATE use genuine enforced native Get + with display/heavy enrichment disabled. DELETE rechecks table-level policy + using a genuine read input and does not fetch or expose deleted values. +- Existing append-only script execution audit records attempts/outcomes with + event correlation and generic host-owned refusal codes. Worker text and source + values stay out of shared history. Core Jobs owns admission, restart, attempts, + lease and completion; no exactly-once script execution claim. A crash between + external execution and completion can repeat an observational attempt. Terminal + audit failure after worker execution leaves the job failed and the independently + committed STARTED entry visible. An authorized operator retry may run the worker + again with the same event correlation; that ID is not execution deduplication. + +## Compatibility and verification + +MySQL fresh schema includes the nine phases; `003_durable_after_hooks.sql` upgrades +the original unnamed event check after explicit schema inspection. H2 metadata-derived +fixtures have no event check to migrate. Existing rows +and published source pins retain their meaning. Native authoring lists AFTER only +when the host installs its required listener; patch fields remain empty and dry +runs explicitly do not prove commit or enqueue jobs. No worker runtime change. + +Use real native InsertAction/UpdateAction/DeleteAction and qualified JDBC/Jobs +fixtures with a trusted protocol worker: source+intent commit/rollback and caught +admission failures; preview suppression; bounded batch locators; identical later +writes remain distinct; repeated same request dedupes/conflicts correctly; +restart restores current actor and pinned script; revoke project/table/run/binding +before delivery and prove zero worker calls; hidden/credential data never enters +params/context; deleted snapshot never enters worker; direct invocation refused; +ALLOW accepted and PATCH/VETO rejected; audit failure explicit; late source +update produces authorized current-state data. Tests must not substitute a fake +queue for native same-transaction/restart evidence. + +Core dependencies are not installed by this task. Root coordinates final shared +artifact handoff, provider gates and consumer compatibility builds. + +## Host registration and authoring + +```java +ScriptAfterHooks.install(instance, sourceTable, + HookEvent.AFTER_INSERT, HookEvent.AFTER_UPDATE, HookEvent.AFTER_DELETE); +``` + +Configure the existing native Jobs provider on the same supported transaction +resource as the source table, enable transactional admission, and provide the +operator-owned namespace/grant key and identity restoration. The scripting QBit +does not provision or weaken those controls. Bind a tested immutable publication +through normal authorized hook administration. The source editor advertises +AFTER only for installed source phases. Dry runs use a current selected record +for authorization, produce `dryRun: true`, never enlist, and cannot attest commit. +Setting an event string in a direct request never supplies the native dispatch +marker: the fixed delivery entry point always calls the core verifier itself. + +The generated editor SDK and bundled compiler SDK declare `HookEvent` (nine +phases) and `CommittedEvent`, whose `CURRENT` variant has a projected `record` +and whose `DELETED_METADATA` variant has no record. These are type assistance, +not authority. Core lease/grant/session checks run on entry, at the actual worker +boundary after history/artifact host callbacks, and after delivery. + +Trusted SDK verification: worker/after-sdk-tests.log (18 passed), +worker/after-sdk-lint.log; frontend scripting-after-ui-tests.log (15 passed) and +scripting-after-ui-lint.log. Compiler SDK source SHA256 +579ad147e7e0d4f2e025e4513fa8d6ad0436870872b64cb4ce4907ad9eec986a; +worker test source bda2208680b50d81132affb845746ce75d4445aa21b0f594b1cc7ea16f2be5cd. +Root subsequently built and sealed the separate 0447 SDK image candidate; +sealed 0240 image evidence remains unchanged. Runtime qualification remains separate. + +## Historical initial native evidence and compatibility boundary + +`after-hooks-full-verification.log` is the successful full base gate. The earlier +focused 19-test run found two wrong exception-class assertions; actual native +required listeners throw RequiredListenerException. The corrected 20-test gate +passed before the final full run. Tests preserve source rollback and failed +commit assertions. Existing PRE/POST tests explicitly select their six phases; +AFTER has separate host registration. + +The native tests prove source/intent shared commit and rollback, preview refusal, +required-listener admission poisoning even when caught, bounded batches, distinct +identical later mutations, restart with pinned publication and current-state +projection, metadata-only deletion, run/project/source/binding revocation, direct +non-enlisted process refusal, pre-worker scope changes refused after STARTED, and +post-worker scope changes refusing job completion. STARTED audit failure prevents +the worker; outcome audit failure can follow a real execution, and explicit +authorized retry executes again with the same event correlation. + +Verified main JAR SHA256: +`d56cae9d674856b3b8eca68e499896d78f59970b73eee50ef88228d17c207cba`; +tests JAR `2209b33413512d7029c6564ddb7215c0f815732915e341048d5848ff8ef30cd7`. +Exact source manifest: `target/after-hooks-verified-source.json`. The core/RDBMS +prerequisite full gates are 3203/417 tests with 16 existing skips; subsequent +29-test native statement-cleanup gate and exact artifact handoff are parent-owned. + +H2 evidence does not establish a combined AFTER consumer/MySQL execution test. +Core transactional Jobs has its own actual MySQL provider gate; the optional +scripting RDBMS provider compatibility suite remains a distinct gate. Production +identity restoration, operational namespace/key rotation, actual isolated worker +qualification, and deployment recovery policy remain host responsibilities. + +## Historical provider and editor follow-up before the corrected gate + +Test-only provider seams preserve all 13 AFTER scenarios and their assertions; +they pass again on actual file-H2 (`after-hooks-provider-seams-verification.log`). +The production main JAR is byte-identical to the full145 gate. Regenerated test +JAR SHA256 is `ec304ab4e191c4a7e4ffca30fe48bb73ee8511e84a6224471589cad57ed842f7`. +Only the base AFTER fixture changed relative to the original source manifest. + +The optional MySQL subclass uses the existing pinned provider factory, DML-only +application identity, real scripting policy, native Jobs qualification and shipped +scripting migration/triggers. Jobs adds mandatory READ_COMMITTED and bounded DDL +lock wait. Its restart scenario restarts the actual server and refreshes the +inspected dynamic port before recovering the queued intent. There is no weaker +provider/schema substitute. Full42-provider execution is pending. + +The previous optional29 gate had one MySQL communications failure (the generic +qualification refusal intentionally carries no SQL/credentials). A separate +unchanged-deadline retry of that exact method passed with running=true, +oomKilled=false, restarts=0. It does not establish the original failure's cause +or justify calling it transient. Bounded test-only exception class/SQLState and +policy-method diagnostics remain enabled for the combined gate; they never log +SQL, messages, connection properties or credentials. + +The offline editor fallback SDK also now matches the generated/server and bundled +compiler SDK. Actual language-service tests cover completion of all nine hook +phases plus CONTEXT, invalid event diagnostics and deleted-record narrowing. +Combined frontend API/panel/language-service22 tests and scoped ESLint pass in +`scripting-after-sdk-ui-tests.log` and `scripting-after-sdk-ui-lint.log`. No package +or lockfile changes accompany these declarations. + +## Corrected audit schema and final provider gate — 2026-10-31 05:26 + +The first combined42 gate (`rdbms/after-hooks-combined-provider-verification.log`) +failed five AFTER/MySQL assertions and one baseline MySQL setup. The five AFTER +failures exposed a real schema defect: UUID plus ordinal correlation needs at +least 38 characters, while the old physical audit column allowed 36. Producer +metadata and fresh001 now use VARCHAR(64); actual003 widens existing history +before enabling nine-phase bindings. Event identity and retry assertions were +not shortened or weakened. The upgrade test preserves every column of two real +pre-upgrade audit rows and verifies their UPDATE/DELETE guards after migration, +then records STARTED/OK with the exact long event correlation. + +The optional policy now checks the exact current transaction's mapped execution +column: VARCHAR, capacity at least64 and NOT NULL, using a bounded metadata SELECT. +Unmigrated36 and nullable columns refuse before worker execution or new audited +work. There is no qualification cache, second connection or runtime DDL. + +`after-hooks-correlation-full.log`: full145 base tests, zero failures/errors/skips, +Checkstyle and unchanged coverage gates passed. `rdbms/after-hooks-capacity-h2-verification.log` +passed13 file-H2 tests. Root's `rdbms/after-hooks-operation64-provider-verification.log` +passed44 tests at05:26:08:13 file-H2,14 actual AFTER/MySQL and17 baseline MySQL, +zero failures/errors/skips and Checkstyle clean. The actual AFTER scenarios use +native Jobs admission/delivery, DML-only application identity, shipped migrations, +restart, rollback, current-authority refusal and audit-failure retry. + +Exact corrected artifacts (SHA256): + +- Base main: `9d205a77b155cb59360b24f3e5b38d139df24f9548398e62d3ed789c042d0c1d`. +- Base tests: `b6a9db73a761014a322a2551a169fb6eb036b343bd7ee58e342c94d5a0bc11c2`. +- Base POM: `987b2e5456a22b07ce1601c57ea6f50e4d44b21491de53ff6c5048feaa2ff1bd`. +- Optional RDBMS JAR: `2ff374cb881be26b331e9db9ec8fc20a8ea694ee49017b3ba81c5341217cd989`. +- Optional POM: `6be71c8fe8d727f0047c0ce5c2edf50b2e2785d2fa746db6589256770a9c547c`. + +Base source manifest: `target/after-hooks-operation64-source.json`; combined +source/artifact manifest: `target/after-hooks-operation64-provider-source.json`. +Only the dedicated local Maven repository received base artifacts; no shared +installation or deployment occurred. Optional middleware/GitHub compatibility +is recorded separately after its own gates. + +The earlier12954 qualification/transport failure and6946 baseline08S01 failure +remain preserved. A single-method diagnostic retry and this full corrected gate +passed unchanged deadlines, with healthy fixture observations; they do not +establish those original transport failures' cause or justify calling them +transient. Only the separately reproduced audit-width defect is diagnosed and +corrected here. Production storage/recovery, identity restoration and hostile +worker qualification remain deployment gates. + +### Optional consumer compatibility + +After the corrected145/44 snapshot, serial offline wrapper verification in the +same dedicated repository passed middleware7 (including6 actual native HTTP +tests) at05:27:44 and GitHub importer7 at05:28:37, with Checkstyle and no installs. +Logs: `middleware/after-hooks-operation64-compatibility.log` and +`github/after-hooks-operation64-compatibility.log`. GitHub tests use trusted +HTTP fixtures; no private repository was sent externally. + +Middleware JAR SHA256 `4094e2d7b74906fda6dd98ff6e879fa38aec7cd8489ae8681b1c196fb21fe936`; +GitHub JAR SHA256 `63b2ed1a8ecae02fc2d37754ab552a5133525a20087865120b0c4bc7fbc1c300`. +Their exact source, POM and artifact hashes are appended to +`target/after-hooks-operation64-provider-source.json`. No production source changed +during these compatibility gates. These existing HTTP tests establish regression +compatibility; native durable AFTER delivery is exercised by the separate13/14 +H2/MySQL fixtures, not by an invented HTTP commit assertion. diff --git a/docs/KNOWN-ISSUES.md b/docs/KNOWN-ISSUES.md new file mode 100644 index 0000000..2734242 --- /dev/null +++ b/docs/KNOWN-ISSUES.md @@ -0,0 +1,9 @@ +# Known issues and unfinished work + +Updated: 2026-11-04. These are unresolved items, not a plan currently being executed. + +- Public Core 4.1.0-RC.21 lacks isolation, pre-update hook, transactional change, durable intent and source-guard APIs required by this source. +- The Core integration and this repository's new aggregate Maven layout were interrupted during validation. Treat both as unfinished; the private qualification artifacts are not public dependency versions. +- The full `base`, RDBMS, middleware and Git integration build, attached test fixtures, clean-consumer resolution and cross-QBit context-script execution still need verification on the final dependency cohort. +- Runtime images, fully untrusted-script containment, resource limits, recovery, Kubernetes deployment and record-save latency need qualification on the actual configured runtime. Existing qualification flags and fail-closed behavior must not be bypassed. +- Source/Git authoring and UI changes also require the matching canonical Next UI release. The earlier private demo is not proof that the current public artifacts provide the full IDE and lifecycle behavior. diff --git a/docs/NATIVE-AUTHORING.md b/docs/NATIVE-AUTHORING.md new file mode 100644 index 0000000..30b2f17 --- /dev/null +++ b/docs/NATIVE-AUTHORING.md @@ -0,0 +1,55 @@ +# Native scripting authoring + +The producer installs standard QQQ processes on the Script table. A client needs ordinary process discovery/execution, EDIT_FORM, VIEW_FORM, possible-value choices and optional CODE_EDITOR adornments. The enhanced source widget is an additional editing experience; these forms do not depend on Next, a browser-specific route or a separate service. + +| Process | Native flow | Semantic owner action | +| --- | --- | --- | +| `scripting_create_project` | Name → isolated project link | `createProject` | +| `scripting_add_file` | Selected project → relative `.ts` path/source/note → saved revision | Optimistic `saveDraft` | +| `scripting_edit_file` | Selected project → current file choice → exact source editor/note → saved revision | Project-scoped `draft`, optimistic `saveDraft` | +| `scripting_rename_file` | Selected project → current file choice → readonly old path, new path/note → saved revision | Project-scoped `draft`, optimistic `saveDraft` | +| `scripting_remove_file` | Selected project → current file choice → readonly path/note → saved revision | Project-scoped `draft`, optimistic `saveDraft` | +| `scripting_import_commit` | Selected project → repository/exact commit/optional directory → retained provenance and draft revision | Existing configured importer, optimistic `importDraft` | +| `scripting_check_draft` | Selected project → pinned latest revision, entrypoint choice, declarations JSON → diagnostics/build link | `check` | +| `scripting_test_context` | Selected project → pinned latest successful build, context JSON, grants JSON → result/execution link | `testContext` | +| `scripting_publish_build` | Selected project → authorized successful-build choice/note → publication link | `publish` | + +Add/edit reconstruct the immutable base file list and change exactly one file; other files remain byte-for-byte unchanged. Source changed after form preparation is refused, including stale check forms. Existing service limits remain: at most200 files,256KiB per source file and2MiB total. Native code fields are presentation; server path/byte/snapshot validators remain authoritative. + +Rename/remove use the same complete immutable snapshot and expected-revision check. The selected path is loaded by the backend and readonly on the final form. Rename preserves that file's contents and rejects unsafe paths, unchanged names and case-insensitive collisions with other files; a case-only rename of the selected file is valid. Remove omits only that file from the new draft and refuses removal of the last file. Neither operation modifies earlier revisions, builds, publications or bindings, and neither builds or publishes automatically. Imports and configured entrypoints are **not** rewritten: review the new draft and run Check before testing/publishing. These are file operations, not TypeScript symbol refactoring. + +`ScriptingFileActionsProcessTest` covers metadata/selection, immutable history, unchanged imports, path/collision/case rules, stale selection/submit, current permission and credential refusal, and nonempty projects. Execution results belong to the matching verification log; host fixtures do not execute source code. + +Read-only form fields and possible-value choices do not grant authority. Every backend step rechecks current user/project/action access; submitted revisions/builds must belong to that authorized isolated project. Publication uses the exact selected build and the existing requirement for a passing test of that build. A later draft does not change the tested artifact. Build choices are the bounded recent100 owner history; the semantic API remains available for advanced/history workflows. + +Capabilities are an explicit JSON array, for example: + +```json +[{"operation":"READ","table":"purchase_order","fields":["id","status"],"maxRows":10,"maxCalls":2}] +``` + +Default `[]` grants nothing. Unknown properties, non-READ operations, coerced number strings and invalid budgets fail closed. Native forms reuse the existing bounded JSON codec and capability constructors; actual test grants must be within the signed build declaration and the current caller's read authority. Context input is a bounded JSON object. No capabilities, source or record context are inferred from browser state. + +Checking/testing/publishing still requires the configured, qualified external worker and current named permissions. Forms do not enable a fallback runtime. Host tests use a protocol fixture, not JavaScript execution; native HTTP/real-worker acceptance is separately owned and must be recorded after its gate. + +## Project visibility with host record security + +Script projects are the core `script` table. Project access is the caller's own enforced QQQ `Get` of that row, so +native record-security locks configured by the host apply exactly as for any QQQ table: a host that locks the table +(for example by tenant) gets projects private to their lock value. The owner can create, view, check, test, publish +and bind; another tenant holding the same lifecycle permissions is refused for the project, its revisions, builds, +publications and bindings, and ordinary table reads of the child tables return nothing. Lifecycle permissions +(`scripting.draft/check/test/publish/bind/execute`) stay separate from visibility. Without a lock, projects are +shared by anyone holding the permission (the unchanged default). + +The QBit itself reads and writes its project tables as the QQQ system session after authorization. QQQ lets that +session through a lock only when the lock's `QSecurityKeyType` defines an all-access key +(`withAllAccessKeyName`). Producing the QBit therefore refuses a locked `script`, `scriptRevision` or +`scriptRevisionFile` table whose key type defines none, with the table and key type named. (Before this check such a +configuration started, then failed every authoring call for the owner as "not found".) Locks are never bypassed +and no key value is inferred from the actor; configure the key type and stamp the lock field on project creation +with an ordinary table customizer. + +Execution of an already-bound hook is not tied to the end user's project visibility: a binding pins its immutable +publication at bind time, and later saves by users outside the project's lock value still run the bound hook. The +binding's tenant scope is whatever the host declares for the hooked table; Scripting adds no separate namespace. diff --git a/docs/NATIVE-RECORD-HOOKS.md b/docs/NATIVE-RECORD-HOOKS.md new file mode 100644 index 0000000..19c13df --- /dev/null +++ b/docs/NATIVE-RECORD-HOOKS.md @@ -0,0 +1,56 @@ +# Native record hooks + +The host opts a table into phases during instance construction, before QQQ validation: + +```java +ScriptHooks.install(table, HookEvent.PRE_INSERT, HookEvent.PRE_UPDATE, + HookEvent.PRE_DELETE, HookEvent.POST_INSERT, HookEvent.POST_UPDATE, + HookEvent.POST_DELETE); +``` + +Installation preserves existing Java callbacks, runs each once before scripting within its phase, and is idempotent. It stages a new customizer map before replacing metadata. Non-Java callback references are refused. It does not mutate a global configuration singleton. A configured callback must remain trusted host Java; it is never a user-selected code reference. + +Create bindings through ScriptingService.bind with current scripting.bind permission, an authorized publication, declared read grants, and explicit patchable fields. A binding pins its immutable publication. A new publication does not move it. Every execution reloads the active binding and verifies its artifact. Generic binding tables are service-write-only. The optional middleware and source editor expose semantic create/enable/disable actions; public Java service calls enforce the same current target read/write permissions, installed phase and patch-field policy. Binding changes and their attributed BINDING_CHANGE receipts commit atomically. Disabling requires current target read permission and project/BIND authority; a removed phase does not prevent disabling it. + +Binding scope: a binding is administrative deployment authority controlled by `scripting.bind` and applies to ALL records written to its table and event in the configured application namespace. It is not filtered by the project's owner or tenant, and project authoring privacy is not an execution predicate. Execution uses the current record actor's permissions and the binding's declared capabilities. The native binding routes (`POST .../bindings`, `.../bindings/{bindingId}/state`) state this in their OpenAPI descriptions. + +| Phase | Result | Native behavior | +| --- | --- | --- | +| PRE_INSERT | ALLOW, REJECT, or permitted PATCH | Standalone hook runs BEFORE_ALL_VALIDATIONS; QQQ field behaviors, required/PVS, unique, reference and record-security checks follow. | +| PRE_UPDATE | ALLOW, REJECT, or permitted PATCH | Requires reviewed core early-update phase. Stored identities are authorized first; then native field behaviors and all ordinary validations process the patch. | +| PRE_DELETE | ALLOW or REJECT | Uses selected stored values; PATCH is invalid. | +| POST_INSERT / POST_UPDATE / POST_DELETE | ALLOW only | Runs after backend action; the caller transaction may still roll back. PATCH/REJECT is INVALID, creates a visible warning, and does not undo the action. | + +When an existing host callback uses a later insert/update phase, composition preserves it. Patches are unsupported there: binding creation rejects declared patch fields, and runtime rechecks the phase so later metadata changes cannot bypass validation. Veto remains supported. Hosts must explicitly opt trusted callbacks into an early phase if they require script patches. Host validators that must inspect final patched values should use the native subsequent validation mechanisms; existing host callbacks are not secretly rerun. + +Patch values are JSON scalars converted strictly to declared QQQ field types. Primary keys, hidden/noneditable fields, record security lock fields and fields denied by current field policy cannot be patched. Native QQQ owns field constraints and behaviors. Update patches do not run a private substitute validation pipeline. + +The core additive update contract is TableCustomizerInterface.whenToRunPreUpdate(UpdateInput, boolean), returning PreUpdatePhase.BEFORE_FIELD_VALIDATIONS or the default AFTER_ALL_VALIDATIONS. Defaults retain existing host scheduling. Phase selection must be deterministic and free of side effects: UpdateAction selects it once per validation pass; bulk preview routing may consult it beforehand. Early callbacks require stored-record prefetch, receive defensive request/table/record copies of authorized error-free values and copied prior snapshots, and cannot add, remove, replace or retarget identities or associations. Current field and table permission checks precede copy-back. Public previews refresh stored rows rather than trusting caller-supplied old values. Backends without prefetch fail closed for early callbacks. Table and instance callbacks retain their order within each phase. + +## Context and authority + +Worker main receives {event, payload:{proposed, prior?}}. PRE_UPDATE prior is obtained through an enforced native Get in the action transaction, not accepted from a supplied old-record list. POST_INSERT and POST_UPDATE proposed values come from the current authorized stored row; POST_UPDATE deliberately omits unchecked prior values. PRE_DELETE uses an authorized current row; POST_DELETE uses the pre-deletion authorized snapshot described below. Values are bounded JSON with no session, Java object or credentials. Hidden/password/blob fields and fields the actor cannot read are excluded. Read grants remain bounded, read-only and subject to current initiating-user broker policy. The original native action transaction is not committed by scripting. + +Current table write permission, identity and context are rechecked around execution. Stored update/delete/post contexts additionally require genuine read-action permission checks and native record visibility. GetAction uses explicit ENFORCE, preserves the original extensible input source and effective action transaction, and suppresses display URLs/heavy content. The input class seen by host custom permission checkers is GetInput. Post rows failing read-only security locks never reach a worker. Insert contexts derived by a composed host callback require table READ permission; field read filtering remains separate. A read customizer cannot recursively invoke a script while constructing hook context. Nested execution from a worker callback is refused across scripting service instances on the same thread. Capability callbacks remain synchronous and bound to the initiating thread/session. An enabled hook fails closed on pre-action failures or oversized batches. No enabled binding means no script execution. Preview pre hooks may execute because capabilities are read-only; execution history is not evidence of an actual write. + +POST_DELETE installation automatically composes PRE_DELETE capture, even if no pre-delete script is bound. Captures hold bounded JSON and scalar primary keys in QContext, keyed by exact DeleteInput identity, table, effective transaction, original actor/session/instance and thread. Keys are limited to 1024 encoded bytes; payload plus keys is limited to 256KiB across at most eight pending inputs, with the normal per-hook record limit. The post callback consumes the capture once and processes only native successful-deletion IDs. Retries and previews discard any prior same-input capture; preview never captures. All-pre-veto paths discard immediately. Backend failures that never invoke post callbacks may retain bounded entries until context teardown; they cannot match a new input. Serialized/asynchronous contexts cannot carry captures. + +Authorization for a deleted row was established before deletion. Post serialization rechecks current table/credential/channel and field read policy; it never pretends to read a row that no longer exists. A capture/read/budget refusal alone does not veto deletion for a post-only binding: the post callback skips the worker, records a generic refusal and returns a warning. An actual PRE_DELETE binding independently authorizes its context and may veto. + +## Audit and failure semantics + +STARTED and terminal execution audit entries commit independently of the business transaction. They report script execution, not application of its patch or commitment of the record. A successful post observation can coexist with a later rollback; JDBC tests prove this. Audit failures are surfaced, never converted into a success. If durable storage itself is unavailable it cannot manufacture a durable failure receipt. Post failures become visible warnings because the backend action may already have occurred; their script attempts are independently audited where audit storage is available. Context admission refusals append a separate HOOK_REFUSAL / DENIED / CONTEXT_REFUSED history row with actual actor and current binding/publication/build identity, without record values or fabricated STARTED/work execution. Normal project-scoped history visibility applies. Each retry is a new admission attempt. Refusal-audit persistence failure is explicitly reported; it cannot manufacture a receipt or undo the backend action. + +These synchronous callbacks do not implement durable after-commit execution. The separate `ScriptAfterHooks` adapter uses native transactional Jobs; see [DURABLE-AFTER-HOOKS.md](DURABLE-AFTER-HOOKS.md) for committed observation semantics and its verification status. Scripts cannot send messages, write arbitrary records, print, or perform other side effects through the read-only broker. + +## Authoring and dry runs + +The source editor separates publication from activation. Its hook panel discovers only host-installed phases permitted to the current actor, creates an enabled binding against an immutable publication, and explicitly enables/disables retained bindings. Bindings are paged by project across all publications; an old active binding does not disappear when recent source history exceeds its window. Read grants must be a subset of the publication's declared capabilities. Binding names, grants and patch fields are bounded. + +Hook tests require TEST, project/build association, current target read/event-write permissions and a selected record. The service reads that record through the same enforced native Get and field projection as hooks, accepts only bounded scalar proposed changes for pre-insert/update, and rechecks the row after execution before returning a decision. Tests never invoke a native write or apply patches. Insert/delete tests are simulations using the selected current row, not evidence of an insert/deletion; native write validations are not simulated. The qualified external worker and normal read-only capability broker remain mandatory. + +Execution failure summaries and history contain only host-defined outcome enums and generic text. Worker error codes/messages and transport exception details are untrusted and never persisted or returned through these execution diagnostics. Compiler diagnostics concern authored source and use their separate bounded source-authorized response. + +## Qualification + +Tests use a trusted protocol fixture that never executes script source. Native action, authorization, validation, recursion and JDBC rollback tests are local evidence only. Real hostile-worker deployment qualification remains a release gate. Native storage qualification covers H2/MySQL; PostgreSQL is not a supported provider in this QQQ baseline. Existing MySQL installations must widen the scripting_binding event constraint before enabling AFTER; the current bootstrap lists all nine phases. The six synchronous phases retain their existing meaning. diff --git a/docs/PERSISTENT-WORKER-TRANSPORT.md b/docs/PERSISTENT-WORKER-TRANSPORT.md new file mode 100644 index 0000000..f500291 --- /dev/null +++ b/docs/PERSISTENT-WORKER-TRANSPORT.md @@ -0,0 +1,67 @@ +# Persistent supervisor transport + +`worker.PersistentWorkerProvider` is an optional Java transport. It executes no author code and supplies no Kubernetes supervisor or execution engine. Its trusted loopback tests are protocol tests, not qualification evidence. Deployments remain unavailable until their independently qualified supervisor/runtime and current evidence are configured. Existing `WorkerProvider` methods and inner `qbit-scripting-worker/1` messages are unchanged. + +## Host configuration and cold preparation + +Construct `Configuration` with an operator-owned `SSLContext`, fixed execution endpoint, optional compile endpoint, authorization namespace, runtime profile, toolchain, matching externally qualified/required evidence IDs, deadline and finite capacities. TLS1.3, HTTPS endpoint hostname verification and a negotiated local client certificate are required before HELLO. Configure trusted client key material as well as server trust; the supervisor must verify that certificate and authorize its namespace. A client-side certificate-presence check cannot attest the server's authorization policy. No production keys or trust-all manager are provided. Hostnames and ports are configuration, never source-record fields or request URLs. + +The provider constructor opens no connections. Normal test/publish/bind/enable authoring now invokes the optional preparation SPI with verified immutable code; see [authoring preparation](AUTHORING-PREPARATION.md). The original constructor prepares one copy; the additive `(Configuration, int preparationCopies)` overload uses a bounded operator count. Trusted host activation/reconciliation code supplies a verified `PreparedArtifact(sha256, entryPoint, bundle)` and calls `prepare(artifact, copies)` outside the record-save path. This hashes bounded code, reserves finite cold capacity, connects/authenticates and prepares the requested number of distinct connections. It retains only digest/entrypoint-to-ready-ID mappings and byte counts, not invocation context or capability tokens. Successful partial preparation may remain if a later copy fails. The lifecycle owner must close the provider at shutdown. + +`execute` first takes an immediate execution-count permit before parsing/copying its input, then requires a connection prepared for exactly that digest and entrypoint within the configured namespace/profile/toolchain/evidence/session. A cache miss, exhausted execution admission, or a slot held by execution/cold preparation refuses immediately. If an exact matching prepared connection is leased by heartbeat maintenance, execution may wait for that same epoch to retire, within its original whole-operation deadline. Waiting holds an existing bounded execution permit; it does not create an unbounded request queue. It never connects, uploads code, compiles, evicts or retries. The existing host call still includes and verifies the bundle locally; only the remote INVOKE omits it. No claim is made that Java artifact lookup/hashing/JSON allocation disappears. The caller has already constructed its input before this provider seam; upstream request admission remains responsible for that allocation. + +Maintenance handoff is cancellation-aware. Interruption, closure, loss of the matching epoch, failed heartbeat retirement or the original deadline refuses without sending INVOKE. A waiting execution has priority over another heartbeat, cold preparation or reclamation of its reserved slot. Handoff time and the subsequent wire exchange share one deadline, including the final trusted RETIRED acknowledgment. There is no replay after send and no waiting behind another real invocation. Heartbeat latency can therefore contribute to an invocation's latency; this correctness guarantee is not a measured latency target. Operators still need measured transport latency and sufficient warm capacity. + +Code bytes and entries are bounded per connection; connection count bounds their aggregate. There is no automatic eviction. A full cache refuses new preparation; replace/drain the provider under trusted host lifecycle control to change its working set. Active code is never silently evicted. Remote readiness expiry/restart/eviction must produce a refusal; a later explicit cold preparation is needed after the affected connection is discarded. Readiness is neither execution authorization nor permission to reuse initialized guest state. Existing service and broker authorization still run per invocation. + +Compilation has separate bounded cold admission and its own configured endpoint/connection. A null compile endpoint refuses compile. It never borrows an execution slot. The supervisor must also separate cold work from execution resource reservations; Java pooling alone cannot impose global pod capacity or stop an operator routing both workloads to the same saturated server. + +## Wire protocol: qbit-scripting-supervisor/1 + +TLS application records carry frames with a four-byte big-endian positive signed length followed by exactly that many UTF-8 JSON bytes. A frame is 2..4,194,304 bytes; the entire operation counts both directions plus frame headers against8MiB. Invalid UTF-8, duplicate keys, unquoted keys, trailing JSON data and depth above16 refuse. Outer envelopes have exact key sets and exact scalar types. Inner WorkerProtocol limits remain in force, including128KiB execution results and64KiB broker messages; caller-specified smaller output/call limits also apply. Envelope overhead counts toward outer limits. + +Handshake messages: + +| Type | Exact fields in addition to protocol/type | +| --- | --- | +| `HELLO` | `namespace`, `profile`, `toolchain`, `evidence`, `nonce` | +| `ACCEPT` | the same five strings, exactly echoed, plus a bounded opaque `session` | + +The client nonce is fresh per connection. The authenticated supervisor allocates a new session/epoch; it must never reassign a prior session to an unrelated credential, namespace or runtime. Identity fields are1..128 ASCII letters/digits or `._:-`. Matching configuration strings alone do not establish qualification. + +Every subsequent frame has `protocol`, `type`, `session`, `sequence`, `requestId`. Sequence is a positive connection-local integer, incremented once per operation. Request IDs and sequence/session must match exactly on every response. Only one operation is active per connection, with one outstanding broker request. Broker answers are uncorrelated in the unchanged inner protocol, so multiplexing is deliberately unsupported. + +| Type | Additional exact fields | Meaning | +| --- | --- | --- | +| `PREPARE` | `sha256`, `entryPoint`, `bundle` | Cold immutable code preparation; no record payload or capability. | +| `READY` | `sha256`, `entryPoint`, `readyId` | Exact digest/entrypoint acknowledgment, scoped to this authenticated session. | +| `INVOKE` | `readyId`, `request` | Existing execute request object **without bundle**, retaining exact digest, entrypoint, fresh token, event, payload and limits. | +| `COMPILE` | `request` | Existing strict compile request, on the cold connection only. | +| `BROKER` | `message` | Existing inner broker request; token and request ID must match active invocation. | +| `BROKER_REPLY` | `message` | Existing host-authorized broker response, generated on the initiating QQQ thread. | +| `RESULT` | `message` | Existing compileResult or executeResult, validated against current request and limits. | +| `RETIRED` | `retired` exactly boolean true | Trusted supervisor has stopped/retired this operation's invocation/transient state. | + +PREPARE requires READY followed by RETIRED before readiness is admitted. INVOKE/COMPILE require RESULT followed by RETIRED before any result is returned. Preparation retirement retains only the admitted immutable code representation. Guest output must never have access to the supervisor framing channel: guest `retired` fields, a guest-shaped result or a duplicate RESULT cannot substitute for RETIRED. The supervisor must enforce this separation; a protocol peer capable of lying is not a qualified runtime. + +Guest invocation state must be fresh and disposed according to the existing WorkerProvider contract. This implementation neither changes that contract nor qualifies an isolate reset strategy. There is no bundled server that interprets these envelopes yet. + +## Cancellation, deadlines and uncertainty + +One whole-operation deadline covers socket connect/handshake, frames, broker exchange and retirement. A shared per-provider scheduler closes the active socket to bound blocked network I/O; read polling also observes caller interruption. Timer callbacks carry an operation epoch so a dequeued stale timer cannot kill a reused session. Cancellation/failure closes and discards that connection and all of its readiness; there is no replay after send, reconnection in execute or inferred completion. Provider close cancels its owned sockets/operations and scheduler only. + +**DNS qualification limit:** JDK hostname resolution performed by `InetSocketAddress` on the cold path is synchronous and not bounded by socket connect timeout or socket closure. It consumes one of the finite cold permits while blocked; this implementation creates no unbounded resolver threads. Operator DNS must be configured/qualified with bounded lookup behavior. No hard overall cold-preparation deadline is claimed across a stalled operating-system DNS resolver. The hot path performs no DNS lookup. + +Synchronous native broker work may also outlive the socket deadline until its own native database deadline/callback return. The provider checks deadline/closed/interruption before sending a late answer; it does not forcibly terminate the QQQ thread or pretend to roll back caller work. The executor still closes the capability token in its finally block. Remote supervision must independently terminate guest execution on deadline/disconnect even while the client is blocked or dead. Closing a socket is not remote termination evidence. + +All uncertain, malformed, out-of-sequence or mismatched transcripts discard the connection. Failures expose value-free diagnostics; source, tokens, endpoint/TLS exception detail and guest messages are not logged by this provider. Local and remote bounded admission, cleanup under pod/client death, rolling drain, namespace isolation and runtime resource/fresh-state enforcement remain deployment acceptance gates. + +## Verification scope + +Trusted ephemeral mutual-TLS fixtures exercise sequential code reuse with fresh invocation tokens, caller-thread broker, hot miss with zero connections, entry/byte/copy admission, active-slot refusal, cold compilation separation, exact namespace/profile/toolchain/evidence/entrypoint identity, malformed length/UTF-8/protocol messages, lost retirement, interruption and deadlines. A deterministic captured-scheduler test runs an old watchdog after a later operation begins. Test key material is generated for the temporary fixture by the current JDK; no production credentials or trust bypass are used. + +Initial full base verification passed161 tests with zero failures/errors/skips, Checkstyle and85/65 coverage at09:49:10. The final pre-parse admission refinement passed11 focused test methods at09:50:03, including refusal-before-parse and permit reuse after validation failure. Final full verification passed161 tests with zero failures/errors/skips, Checkstyle and85/65 coverage at09:55:44 (`persistent-transport-final-verification.log`), including the admission and mutual-TLS refinements. Parent scoped source review accepted the transport boundary; no separate full independent provider audit is claimed. The immutable source/JAR/docs checkpoint is `docs/evidence/persistent-transport-20261031-0955`, separate from the sealed JFR evidence. These tests do not execute submitted TypeScript, measure Kubernetes latency or assert a real supervisor retires hostile code. + +## Native cold maintenance — 2026-10-31 + +Native warm lifecycle verification completed 2026-10-31 at11:40:55: parent handle45711 terminal0, full192 tests with zero failures/errors/skips, Checkstyle0 and unchanged coverage gates PASS. Focused42 (lifecycle14/TLS17/authoring11) passed under normal loopback access. Main SHA256 `89B52743EC2EA4CB5670D69BF5927D632FC80A660E4A1093101AE2E3CC07506B`; tests SHA256 `D907AA9CC171A33A7ECC9359D16A4E2ECF980520E7E10073BCD497855F89221C`. No install, shared-cache mutation or deployment occurred. Independent review accepted the five scoped corrections; runtime/supervisor deployment qualification remains separate. The optional maintenance SPI adds idle PING/PONG/RETIRED, acknowledgment freshness and atomic bounded required-set readiness under the existing slots lock. No code, inventory, context or capability token enters heartbeat. Busy slots are skipped. See [WARM-LIFECYCLE.md](WARM-LIFECYCLE.md). diff --git a/docs/PHASE-TIMING.md b/docs/PHASE-TIMING.md new file mode 100644 index 0000000..3073b1f --- /dev/null +++ b/docs/PHASE-TIMING.md @@ -0,0 +1,50 @@ +# Optional scripting phase timing + +`com.kingsrook.qqq.ScriptingPhase` is a JDK Flight Recorder event, disabled by default. It adds no dependency, runtime qualification, permission or audit exemption. After one-time registration, disabled calls reuse one stateless no-op span without a per-call allocation, clock read or I/O. A custom Java runtime missing `jdk.jfr` uses the same no-op path. Recording setup/commit failures are isolated from application results and required audit persistence. + +The only application fields are closed `phase` and `outcome` labels. No source, context, identifiers, actor, credentials, exception type/message or record values are attached. Stack capture defaults off. Standard JFR event metadata still includes duration and thread information; operators must protect recordings and avoid sensitive thread names. Operators can change JFR settings, including stacks, so the default privacy scope does not describe arbitrary additional recordings. + +## Enable a bounded recording + +Use the standard JDK API from trusted host/operator code; no scripting API exposes recording controls: + +```java +try (var recording = new jdk.jfr.Recording()) { + recording.enable("com.kingsrook.qqq.ScriptingPhase") + .withThreshold(java.time.Duration.ZERO) + .withoutStackTrace(); + recording.setMaxSize(32L * 1024 * 1024); + recording.start(); + // Exercise the actual qualified host path for a bounded observation window. + recording.stop(); + recording.dump(java.nio.file.Path.of("scripting-phases.jfr")); +} +``` + +Inspect with JDK Mission Control or `jfr print --events com.kingsrook.qqq.ScriptingPhase scripting-phases.jfr`. Recording output belongs in operator-controlled storage. A threshold drops shorter events and biases latency analysis; use zero when measuring the complete distribution. Recording has its own cost and bounded storage can discard older events. Compare enabled/disabled runs on the real deployment when evaluating overhead. + +## Exact measurement boundaries + +| Phase | Scope | +| --- | --- | +| `HOOK_CALLBACK` | Synchronous native hook callback, including binding lookup, authorized context preparation and applying the decision to the input records. | +| `BINDING_LOOKUP` | Current binding lookup and its existing checks. This also occurs in service calls outside a native save. | +| `AUTHORIZATION_PUBLICATION` | Synchronous `runHook` current-actor, exact binding, publication/build and grant checks before executor entry. | +| `AUTHORIZED_RECORD_READ` | Actual enforced native hook record read, field projection and read-recursion cleanup. | +| `EXECUTION_TOTAL` | Executor entry through result return/propagation, including recursion guard, STARTED/outcome audit and cleanup. | +| `ARTIFACT_VERIFICATION` | Stored build/artifact retrieval and signed-manifest/artifact verification. | +| `WORKER_DISPATCH` | Actual `WorkerProvider.execute` call, including synchronous broker callbacks. Transport startup is included if performed by that provider inside the call. | +| `BROKER_CALLBACK` | Callback thread/session and budget checks plus broker handling. | +| `AUDIT_PERSISTENCE` | Each `ScriptingStore.history` append, including required independent transaction commit and restoring the caller transaction where configured. This is not every draft/publication mutation. | + +`RETURNED` means the boundary returned normally; it does **not** mean ALLOW, successful script execution or committed source mutation. `THREW` means a throwable propagated out of that measured boundary. For example, a provider can throw while the executor returns an audited UNAVAILABLE result: dispatch is THREW and execution total is RETURNED. A native hook can return record validation errors normally. + +Spans are nested and inclusive. Do not sum their durations or their p95 values: worker dispatch includes broker time, and execution total includes dispatch and audit. There is deliberately no request/actor correlation field. Analyze per-phase distributions separately; do not infer one-to-one relationships among interleaved events. + +`HOOK_CALLBACK` excludes the outer native record action, database DML and caller commit. `EXECUTION_TOTAL` is not full record-save latency. Context and durable AFTER execution share executor/artifact/worker/broker/history boundaries, but do not necessarily pass the synchronous hook callback/publication phases. Compilation, queued delay, Jobs identity restoration, native transaction admission, network outside provider dispatch and outer commit require host measurements at their actual boundaries. No full-save or Kubernetes latency result is claimed here. + +## Verification + +`ScriptTimingTest` records actual native insert/update routing through the existing protocol-only FakeWorker and checks all nine phases, exact event fields, default-off reuse, no recorded source/message marker, and unchanged audit rows. A throwing worker verifies native write refusal and durable-history API outcomes while distinguishing THREW dispatch from RETURNED execution. A trusted child JVM with `--limit-modules java.base` verifies graceful absence of JFR. A stopped recording verifies that span cleanup preserves the original application exception. These tests execute no author code and do not qualify a runtime or measure production performance. + +The first focused gate passed 38 tests. Final focused/full verification status was recorded in the pre-extraction session notes (not published); earlier failed import/style gates remain in their original logs. No shared Maven artifact is installed by this change. diff --git a/docs/PRODUCTION-CONNECTION.md b/docs/PRODUCTION-CONNECTION.md new file mode 100644 index 0000000..42b0195 --- /dev/null +++ b/docs/PRODUCTION-CONNECTION.md @@ -0,0 +1,138 @@ +# Connecting a QQQ application to external scripting services + +Current work is **local development**. Use the existing local Docker/native tool workflow for fixed trusted development fixtures. This document describes the existing application connection APIs and the operator work needed when services are deployed in Kubernetes; it does not provision a cluster, require a local VM, or qualify a production deployment. Successful local examples do not satisfy the original requirement to execute fully untrusted author code safely. + +The application can connect to the execution supervisor and cold compiler over persistent mutual TLS. It does not need Kubernetes API access, kubeconfig, service-account administration credentials, a Kubernetes SDK, or per-record pod creation. Operator deployment and secret delivery happen outside record execution. The current renderer bridge has a different interface and is described separately below. + +## Existing application API + +[`PersistentWorkerProvider.Configuration`](../src/main/java/com/kingsrook/qbits/scripting/worker/PersistentWorkerProvider.java) is a Java record supplied by host composition. There is **no generic production application JSON/YAML/environment loader in this QBit**. The [Java example](production-connection-examples/ApplicationConnection.java.example) uses actual constructors/setters; the host supplies its own existing secret/configuration integration. It is documentation, not a new runtime entrypoint. + +| Constructor field | Existing meaning and enforced bounds | +| --- | --- | +| `sslContext` | Host-owned client identity and server trust. Must support TLS 1.3 and present a client certificate. No trust-all fallback. | +| `executionEndpoint` | Required `Endpoint(host, port)`. Host is a bounded DNS name/IPv4 literal; port 1–65535. It is not a URL, Kubernetes selector, Unix socket or credential. IPv6 literals are not accepted by the current host grammar. | +| `compileEndpoint` | Separate fixed cold endpoint, same `Endpoint` type. `null` makes compilation unavailable; it does not enable local compilation or an execution-endpoint fallback. | +| `namespace` | Exact application namespace checked against the authenticated client leaf-certificate fingerprint on both services. This is an application isolation label, **not automatically the Kubernetes namespace**. | +| `profile`, `toolchain` | Exact peer-agreed identifiers. Current compiler requires `typescript-6.0.3-node-24`; the shipped compiler-image verifier additionally pins Node 24.22.2 and TypeScript 6.0.3. A profile string alone does not configure or prove resource isolation. | +| `qualificationId`, `requiredQualificationId` | Required matching evidence IDs. Host `ScriptingRuntime.withRequiredQualificationId` must match too. These are operator trust inputs, not secrets and not automatic certification. Never reuse a trusted-lab identity as production evidence. | +| `deadline` | Whole transport exchange budget, positive and at most 120 seconds. Separate from the guest CPU/wall/resource caps. | +| `executionConnections` | 1–64 application connections/admitted executions. Capacity is finite; misses or busy slots do not create a hot queue. | +| `coldOperations` | 1–8 concurrent cold preparation/compilation operations. | +| `entriesPerConnection`, `codeBytesPerConnection` | 1–256 entries, 1–64 MiB of code per connection. Must fit the server's independently configured limits. | + +Namespace/profile/toolchain/evidence identifiers use `[A-Za-z0-9._:-]{1,128}`. The optional second `PersistentWorkerProvider(configuration, preparationCopies)` constructor requires copies in `1..executionConnections`; the compatible single-argument constructor uses one copy. Each copy occupies a distinct authenticated session/cache epoch. Connection capacity, aggregate JVM/native memory and all application replicas must be sized together; the example numbers are illustrative, not a throughput or latency claim. + +`SupervisorConnection` enables TLS 1.3 and HTTPS-style hostname verification even though this is a framed TCP protocol, not HTTP. The server certificate SAN must match the exact configured service DNS name. Do not terminate TLS at an HTTP proxy that removes the client's identity: the supervisor/compiler require the actual client certificate and map its lowercase SHA-256 DER leaf fingerprint to a single application namespace. Multiple fingerprints can be declared during an operator-controlled certificate transition; there is no wildcard namespace rule or automatic certificate reload API. + +## Service-side configuration already implemented + +Both entrypoints take **one absolute path** to an operator JSON file of at most 64 KiB. Unknown/missing keys fail closed. The example files contain deliberately invalid `<...>` placeholders for hashes and evidence; there is no interpolation engine. Replace them with reviewed deployment inputs before use, without changing the schema. + +- [`SupervisorMain`](../supervisor/src/main/java/com/kingsrook/scripting/supervisor/SupervisorMain.java): [exact JSON example](production-connection-examples/supervisor.json.example). +- [`CompilerMain`](../supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerMain.java): [exact JSON example](production-connection-examples/compiler.json.example). + +Both accept `bind`, `port`, `keyStore`, `trustStore`, `certificateNamespaces`, `profile`, `toolchain`, `evidence`, `requiredEvidence`. `bind` is the server listen address; a literal avoids cold-start DNS dependence. `keyStore` and `trustStore` are absolute paths to PKCS12 files, each at most 1 MiB. `certificateNamespaces` contains 1–256 exact lowercase 64-hex leaf fingerprints mapped to namespace identifiers. Both enforce TLS 1.3 client authentication. + +The existing entrypoints read passwords only from **`QQQ_SUPERVISOR_KEYSTORE_PASSWORD`** and **`QQQ_SUPERVISOR_TRUSTSTORE_PASSWORD`**, including the compiler. There are no corresponding JSON password fields and no password-file option. For Kubernetes, the operator can project PKCS12 files as read-only secret volumes and supply these exact environment variables through secret references such as `env.valueFrom.secretKeyRef`. Those references belong to the host deployment, not the QBit config parser or guest context. Keep the runtime/compiler tree separate from secret mounts. The app-side `SSLContext` is built by host code from its own mounted client keys/trust; these service-entrypoint env variables are not automatically read by `PersistentWorkerProvider`. + +Additional supervisor keys: + +| Key | Existing requirement | +| --- | --- | +| `runtimePath`, `runtimeSha256` | Absolute fixed Wasm runtime path and exact lowercase SHA-256. Not author-provided bytecode. | +| `nativePath`, `nativeSha256` | Absolute fixed native library path and exact SHA-256. Pin the matching reviewed binary/OS/architecture closure. | +| `slots`, `connections` | 1–64 engine slots; connections must be at least slots and at most 128. | +| `coldOperations`, `entries`, `codeBytes` | 1–8, 1–256, 1–64 MiB respectively. | +| `memoryBytes` | 64–256 MiB. Invocation memory must exactly match this runtime profile; smaller requested limits are not silently rounded. | +| `idleMillis`, `fuel` | 100–60,000 ms; positive fuel at most 1,000,000,000. These are not substitutes for outer process/container enforcement. | + +Additional compiler keys: + +| Key | Existing requirement | +| --- | --- | +| `nodePath`, `nodeSha256` | Absolute fixed Node executable and exact SHA-256; no PATH lookup or author-selected command. | +| `compilerRoot`, `manifestSha256` | Absolute fixed image tree and exact `compiler-manifest.json` hash. The manifest uses `qqq-typescript-image/1`, Node `24.22.2`, TypeScript `6.0.3`, and the explicitly allowed compiler/TypeScript file closure. Extra files/symlinks fail verification. | +| `heapMiB` | 64–512 MiB Node heap cap. | +| `slots`, `connections` | 1–8 child-compile slots; 1–16 compiler connections. | +| `cpuMillis`, `wallMillis` | 1–30,000 ms CPU; 100–60,000 ms wall. Source compile remains on this cold lane. | + +Mount pinned runtime/compiler inputs read-only: startup hashes cannot stop privileged post-check file replacement. The host/operator selects immutable image digests and the corresponding source/binary evidence; the connection API does not query a registry or deploy an image. Existing frame/session/sequence/request checks and `RETIRED` cleanup acknowledgment remain mandatory. See [supervisor contract](../supervisor/README.md) for exact cancellation, native cleanup and CPU accounting limits. + +## Stable signing, persistence and current authority + +Configure `ScriptingRuntime.withCoordinator(new BuildCoordinator(coordinatorId, key, toolchainId))`, `withWorker`, `withRequiredQualificationId`, and `withTransactionPolicy(new RdbmsScriptingTransactionPolicy())`. The coordinator key must contain at least 32 bytes and must be held only in application secret-backed configuration; it is never sent to workers. Preserve the correct coordinator ID/key for existing immutable HMAC-signed artifacts across application restarts. There is no built-in key-ring/rotation migration or Kubernetes Secret loader: changing to an unrelated key makes existing verification fail, rather than silently resigning history. + +The host's existing native backend is `RDBMSBackendMetaData` (`com.kingsrook.qqq.backend.module.rdbms.model.metadata`), configured with `withName`, `withVendor`, `withJdbcUrl`, `withUsername` and `withPassword`; `ScriptingQBitConfig.withBackendName` selects that registered backend. Resolve database credentials through host secrets, not a new scripting property. For MySQL, configure finite driver `connectTimeout` and `socketTimeout`, `connectionTimeZone=UTC` and `forceConnectionTimeZoneToSession=true` in the native JDBC/pool configuration; preserve authenticated TLS rather than copying the test fixtures' disabled TLS. Session lock timeout must be 1–5 seconds and UTC, with a writable non-autocommit transaction. MySQL qualification also requires InnoDB, exact binary text comparison/keys, `innodb_flush_log_at_trx_commit=1`, recovery-capable `innodb_doublewrite`, and `sync_binlog=1` when binary logging is enabled. File H2 uses an actual `jdbc:h2:file:` backend with live `WRITE_DELAY=0`; memory H2 is test-only. These are requirements to configure/verify separately, not settings this QBit changes for the host. + +The optional [RDBMS adapter](../rdbms/README.md) is mandatory for its supported durable deployments; absence refuses persistence/history, not just publication. It requalifies the actual transaction, mapped tables, identity keys and operation-ID capacity. Current support is qualified MySQL 8.4 and file H2 under the documented settings; Aurora/MariaDB/PostgreSQL are not implied compatible. Apply shipped migrations with a separate migration identity; application identity stays DML-only. Configure finite native JDBC connect/socket/query timeouts and bounded lock waits. The adapters do not create schemas or change server settings on the hot path. Native AFTER hooks additionally need the existing core durable-Jobs configuration/migrations, described in [DURABLE-AFTER-HOOKS.md](DURABLE-AFTER-HOOKS.md). + +Keep ordinary invocation/broker authorization tied to the current user, current credential/channel ceilings, immutable publication and declared capabilities. Worker connection identity authorizes a transport namespace; it does not grant a source-record read or bypass tenant policy. + +## Per-node preparation and readiness + +Configure [`WarmReconciliationConfig`](../src/main/java/com/kingsrook/qbits/scripting/service/WarmReconciliationConfig.java) on `ScriptingRuntime.withWarmReconciliation` before producing the QBit. The native producer registers `WarmRuntimeService` automatically for maintenance-capable providers. It registers the **`scripting.warm`** health check with readiness inclusion, 10-second check interval, 5-second check timeout, failure threshold one and no probe-cache reuse. + +| Field | Existing bound/policy | +| --- | --- | +| `principal` | Declared native service-identity name, 1–100 characters from `[A-Za-z0-9_.-]`. Fresh `ServiceIdentities` session per pass/check; never a saved interactive session. | +| `targetTables` | Explicit nonempty set, at most 64 existing native table names; no `scripting_` target names. No implicit all-table scope. | +| `maxBindings`, `maxCodeBytes` | 1–1,000 active scoped bindings; 1–64 MiB total unique code. Overflow refuses readiness, not truncation. | +| `interval`, `passBudget`, `maxAge` | 1–300 seconds; 1–120 seconds; 1–600 seconds. `maxAge >= interval + passBudget`. | + +Declare the principal through `ServiceIdentitiesMetaData.ofOrWithNew(instance).withPrincipal(...)` using `QServicePrincipalMetaData`. It needs `scripting.warm`, `scripting.view` and current authorized script-project/source-metadata visibility under the host's tenant/security-key/credential policy. Those two named grants alone do not override project access. Do not grant record data privileges simply to warm code. The example accepts this explicitly configured principal from the host rather than inventing an all-access identity. + +The service owns one bounded coalesced fixed-delay maintenance future per QQQ instance. It verifies current binding/publication/build/HMAC identity before transferring immutable code, then rechecks it. Idle authenticated `PING/PONG` plus `RETIRED` confirms the same cache epoch without code, context, token or guest execution. Busy valid invocations are not evicted. Readiness checks all required exact prepared identities/copy counts/profile in one local snapshot, verifies age, then re-reads current pins. Missing principal, target, artifact, freshness, capacity or configuration means DOWN. TCP connection alone is insufficient. Readiness covers active bindings plus explicitly inventoried context publications in the configured scope; it does not enumerate every publication automatically. + +User-initiated test/publish/bind already use the cold preparation SPI. An execute/save hook never connects, compiles, uploads or retries on a cache miss: it fails explicitly. Worker restart closes cache sessions; the cold reconciler reconnects and prepares the currently authorized scoped pins. Application restart must construct a **fresh runtime/provider**; reuse of a stopped instance is unsupported. Steady-state inventory work is linear; first-cold repeated pin validation can cost more, including quadratic metadata work. No p95/p99 or startup-time result follows from these configuration examples. + +The host must actually run QQQ's native runtime-service lifecycle and expose its native readiness result through its existing deployment health integration. This module does not supply a new HTTP health endpoint, Kubernetes probe manifest, readiness gate controller or process manager. Cooperative pass cancellation cannot bound an unbounded JDBC/DNS/authentication call; configure those deadlines independently. Readiness should control admission without treating temporary cold recovery as a reason for an endless restart loop. + +Optional `ScriptingRuntime.withWarmPublicationSource(source)` includes required unbound CONTEXT publications in the same cold lifecycle, for example immutable references maintained by an independently installed consumer adapter. The source receives configured target tables and the remaining combined pin limit under the fresh WARM + VIEW service identity. It must return all currently required authorized `WarmPublicationSource.Pin(publicationId, artifactId, bundleSha256)` values or throw, never silently truncate. The foundation revalidates current project/isolation, exact signed pins and successful exact-build CONTEXT testing. Configure the consuming adapter explicitly; otherwise null preserves hook-only behavior and unbound contexts are not automatically restored after restart. This addition is staged pending its root-owned verification; it does not qualify any deployment. + +## Renderer connections: local adapter and new remote transport + +The original optional Generation Docker renderer bridge remains available. Its [`RendererConfiguration`](../../qbit-document-generation/bridges/renderer/src/main/java/com/kingsrook/qbits/documentgeneration/renderer/RendererConfiguration.java) fields are: + +`dockerExecutable`, `image`, `imageId`, `qualificationId`, `requiredQualificationId`, `ownerId`, `stateDirectory`, `renderDeadline`, `maximumBytes`, `maximumPages`, `maximumConcurrent`. + +That constructor requires an existing absolute Docker executable, existing absolute non-symlink state directory, digest-pinned `image` (`...@sha256:...`), exact `imageId` (`sha256:...`), matching evidence IDs and bounded owner ID. Deadline is positive and at most30 seconds; output1–100 MiB; pages1–500; concurrency1–16 (compatible constructor defaults to one). It uses disposable Docker/runsc and the existing durable owned-container lifecycle/reaper. Its fields are not remote endpoint settings. + +A **new optional [remote-renderer module](../../qbit-document-generation/bridges/remote-renderer/README.md)** now provides the service connection implementation. It adds `qqq-remote-renderer-transport` plus `qbit-document-generation-remote-renderer`, without a renderer engine in the application or a foundation dependency on transport. `RemoteRendererBridge(RendererLimits, IsolatedRendererBridge.AssetReader, RemoteRendererClient.Configuration)` reuses current asset authorization, immutable request/catalog checks and output hash/length/PDF/ZPL policy. Close the adapter during normal application shutdown. + +The exact remote application settings are: + +| Existing API | Fields and contract | +| --- | --- | +| `DeploymentIdentity` | `mode`, `namespace`, `profile`, `evidence`, `requiredEvidence`, `imageId`. Explicit mode is `TRUSTED_LAB` or `QUALIFIED_DEPLOYMENT`; matching evidence and immutable `sha256:` image ID are required. Qualified clients refuse trusted-lab service mode; choosing qualified mode does not establish qualification. | +| `RemoteRendererClient.Configuration` | `tls` (`SSLContext`), `endpoint` (`URI`), `address` (already-resolved `InetSocketAddress`), `identity`, `deadlineMillis` (100–30000). Endpoint must be HTTPS `/render`, with no userinfo/query/fragment; address port must match. Certificate hostname verification uses the URI hostname independently of the resolved address. Resolve/refresh service DNS outside requests and recreate the operator configuration when routing changes. | +| `RendererLimits` | `admissionIdentity`, `maximumBytes`, `maximumPages`, `maximumConcurrent`. Keep a stable deployment identity and its immutable JVM concurrency limit across instances. Server slots/deployment enforce separate service/cluster capacity. | + +`RendererServiceMain` consumes one absolute operator JSON file; use its [actual example schema](../../qbit-document-generation/bridges/remote-renderer/config.example.json). Required keys are `bind`, `port`, `keyStore`, `trustStore`, `identity`, `certificateNamespaces`, `slots`, `deadlineMillis`, `maximumOutput`, `javaExecutable`, `classpath`, `fileHashes`, and `temporaryRoot`. Supply the complete fixed ordered renderer JAR classpath and exact executable/JAR/native-provider hashes; there is no arbitrary author command. Passwords come from **`QQQ_RENDERER_KEYSTORE_PASSWORD`** and **`QQQ_RENDERER_TRUSTSTORE_PASSWORD`**, not JSON. Use host-owned secret references and immutable inputs as described in that module's README. + +The private `qqq-remote-renderer-1` transport is one TLS1.3 HTTP/1.1 `POST /render` per connection, with actual client-certificate fingerprint→namespace authorization, exact identity/exchange headers and service-generated retirement evidence. It has no redirects, automatic retry, compression, chunking, cookies or generic business REST endpoint. Direct TCP forwarding can preserve this contract; an ingress that changes framing or terminates client TLS is not implicitly supported. The application needs neither a Docker socket nor Kubernetes administrative credentials. + +**Current evidence and limits:** parent reports the initial 41-test remote-module gate passed. Two deadline-edge corrections remain pending their verification at this documentation checkpoint. Do not infer final corrected-source/image/end-to-end approval from the baseline. The implementation is not claimed hostile-input or Kubernetes production-qualified. In particular, the service and renderer child can have the **same UID/filesystem authority**: filtering the child's environment does not prevent an escaped child from reading mounted service secrets. Production requires separately demonstrated credential/filesystem separation. `ProcessHandle` tracks the fixed child and observed descendants but is **not atomic containment of arbitrary hostile forking processes**; actual OS/cgroup containment, aggregate memory/CPU/PID/tmp limits and termination behavior require qualification. Image/evidence headers are operator pins, not self-attestation. Fresh per-request children provide no warm-pool or latency guarantee. The existing local Docker path is not silently replaced or weakened. + +Continue local development with the existing local Docker/native tools and explicitly scoped trusted fixtures. Document and connect a production deployment only against its reviewed service/secret/containment evidence; no cluster provisioning or production mode was enabled by this documentation update. + +## Deployment boundary and remaining evidence + +Operators provide service routing/DNS, actual TLS identities and rotation, immutable images/native libraries, resource/process isolation, egress restrictions, secret delivery, finite database/auth/DNS deadlines, capacity and graceful shutdown/forced termination policies. Service caches are connection-local; ordinary TCP routing must preserve established sessions. Every application replica must reconcile its own connection/copy capacity after restart. No cluster access or provisioning was performed for this document. + +Current local trusted-fixture successes establish their stated source, transport, lifecycle and integration behavior only. Before admitting fully untrusted authors in production, qualify the exact image/native/Wasm/compiler/renderer tuple and deployment containment, resource cancellation/cleanup under load, restart/recovery behavior, authority isolation, durable storage and operational limits. Matching evidence strings do not perform that qualification. The proposed record-save overhead budget remains a target, not an accepted/measured production SLA. + +Documentation checked against the actual source APIs on 2026-10-31. These examples contain no credentials or ready-to-use qualification identity. No runtime code, default, image, cluster, shared cache or installed artifact was changed. + +### Warm capacity integration update (verified source, 2026-11-01) + +The optional Generation supplier now requires the declared maintenance principal's explicit +`document-generation.warm` grant in addition to view. Its active/draft/unfinished-work inventory +and batch-completion migration are documented in +[the optional bridge](../../qbit-document-generation/bridges/scripting/README.md#cold-maintenance-for-document-context-publications). +Cold idle-session retirement and bounded authoring-use leases are described in +[WARM-LIFECYCLE.md](WARM-LIFECYCLE.md#capacity-reclamation-and-authoring-use-2026-11-01-verified-source). +Keep configured capacity/headroom and finite JDBC/transport deadlines explicit. The source passed full Scripting227 and Generation focused67/Checkstyle (including H2 migration); +actual MySQL migration delta qualification remains separate. Before adopting new application code, +apply the appropriate additive migration once and retain NULL for existing batches. No backfill to +true is authorized by this change. These gates do not establish production containment or deployment qualification. diff --git a/docs/RELEASE-STATUS.md b/docs/RELEASE-STATUS.md new file mode 100644 index 0000000..ada4ec3 --- /dev/null +++ b/docs/RELEASE-STATUS.md @@ -0,0 +1,27 @@ +# Scripting — current status + +Updated: 2026-11-04. **Work stopped at the user's request. Unreleased work in progress; not a production 1.0 release.** + +## Purpose + +Shared TypeScript authoring and compilation, immutable published scripts, scoped API capabilities, record lifecycle hooks and isolated execution integration. + +## Repository layout + +`base/`, `rdbms/`, `middleware/` and `github/` are the Java reactor modules. `worker/`, `wasm-runtime/`, `supervisor/` and `deploy/` contain separate runtime components and deployment material. A Java package build does not build or qualify every runtime image. + +## Building and installation + +Use Java 21 and the included Maven wrapper. Once compatible Core artifacts are available, use `./mvnw verify` for the Java reactor or `./mvnw -pl base -am verify` for the base subset. Runtime components have separate build instructions. Current source is a checkpoint, not a supported production installation. + +## Evidence retained + +The initial packaging checks passed style validation; deployment configuration tests passed 13 cases. Public Core 4.1.0-RC.21 could not compile the required APIs. A separate Core integration candidate compiled selected modules under a private qualification version and passed 15 transaction-callback tests. The complete Scripting reactor and runtime were not verified before work stopped. + +## Source and release state + +The starting source was extracted from Website-Backend commit b6fb23825a87412397dec703b9617148ae7eb601, then adapted for standalone packaging. This repository preserves the resulting source and existing licenses. It does not declare a published artifact cohort, completed deployment or complete acceptance. + +See [known issues](KNOWN-ISSUES.md) and the README for the existing API and integration documentation. No further implementation, tests, releases or deployment should be started without explicit user instruction. The outstanding dependency order is Core, Collaboration, Scripting/Record Files, then the integrated Generation bridges and matching UI. + +The prepared release caller is saved as an inactive reference in docs/release/qrun-release.reference.yaml. No new workflow is being activated by this checkpoint; shared CI/CD design and enrollment remain unfinished. diff --git a/docs/SOURCE-PROVENANCE.md b/docs/SOURCE-PROVENANCE.md new file mode 100644 index 0000000..539c059 --- /dev/null +++ b/docs/SOURCE-PROVENANCE.md @@ -0,0 +1,5 @@ +# Source provenance and checkpoint boundaries + +The initial source was extracted from Website-Backend commit b6fb23825a87412397dec703b9617148ae7eb601, subtree corresponding to this QBit. Existing license and copyright files were retained. Subsequent changes adapt the source to a standalone Maven layout; the source is preserved unfinished, not advertised as a published release. + +The current status and known issues document the actual validation and limitations. Local build caches, credentials, tool transcripts and machine-specific operational handoffs are excluded from the repository. Git history records the closeout snapshot. No current artifact coordinate or deployment should be inferred from a private historical build version. \ No newline at end of file diff --git a/docs/STATUS.md b/docs/STATUS.md new file mode 100644 index 0000000..d4498d1 --- /dev/null +++ b/docs/STATUS.md @@ -0,0 +1,29 @@ +# qbit-scripting standalone status + +This replaces the pre-extraction session notes, TODO list and private-host evidence logs, which were not carried into this repository (see `PROVENANCE.md`). + +## Release blocker: core API gap + +The source in this repository was developed and reviewed against a reviewed private QQQ core/RDBMS build. It does **not** compile against public `qqq-backend-core:4.1.0-RC.21`. Missing from RC.21 (found by compiling, not by inspection): `ScriptExecutionIsolation` / `ScriptExecutionMode`, `TableCustomizerInterface.whenToRunPreUpdate` / `PreUpdatePhase`, `TransactionalRecordChangeMetaData` / `TransactionalRecordChangeScope`, `DurableProcessMetaData.get/withTransactionalRevision`, `BoundedIntentData`, `FixedProcessRequest`, `ScriptTableCustomizer`, `requireEnlistedExecution`. Eight main source files are affected (hooks, service and the QBit producer). Publishing the Maven modules therefore needs a public QQQ release that contains those APIs, or a reviewed source change; neither is done here. + +## What the code is + +A control plane: script projects, immutable strict-TypeScript builds, capability broker, native record hooks. It never compiles or runs script code in the JVM. With no qualified worker configured, every build and execution fails closed. + +## Runtime source (not part of the Maven QBit artifacts) + +`worker/` (Node/TypeScript), `supervisor/` (Java, Extism/Javy host), `wasm-runtime/` (Rust, pinned Wasm), `deploy/` (qualification packaging). They are source only. No worker image was built, rebuilt, signed or deployed for this extraction, and `qualified=false` / `productionQualified=false` / `redistributionCleared=false` stay as written. See `worker/QUALIFICATION.md`, `deploy/NOTICE-GAPS.md`, `wasm-runtime/docs/release-license-review.md`. + +## Production gates still open + +- Independent qualification of a real worker image and runsc deployment (escape/exhaustion/CPU/memory/process/fd/output/wall controls, no ambient network/secrets/mounts). +- Durable ownership journal and reaper proven across coordinator/host death. +- Host-managed HMAC key lifecycle (rotation, revocation); no signing keys in records or workers. +- Supported schema is MySQL 8.4 / file H2 only; PostgreSQL and Aurora are unsupported by the current adapter. Immutable-trigger migration and restore need qualification on the target host. +- Audit pool capacity and reconciliation of unresolved STARTED events; no exactly-once execution guarantee. +- Broker integration review under permission changes; GitHub access wiring to a managed credential provider. +- Upstream SPDX attestation and residual advisory/license review for the worker image. + +## Build + +Use the wrapper. `./mvnw verify` builds the base module (Checkstyle incl. tests, tests, JaCoCo 85%/65%). Optional modules are separate POMs that need the base module installed first: `./mvnw install`, then `./mvnw -f rdbms/pom.xml verify` (and `middleware`, `github`). `supervisor/pom.xml` is a standalone, non-QBit runtime build. diff --git a/docs/WARM-LIFECYCLE.md b/docs/WARM-LIFECYCLE.md new file mode 100644 index 0000000..f6015c7 --- /dev/null +++ b/docs/WARM-LIFECYCLE.md @@ -0,0 +1,88 @@ +# Per-node warm code lifecycle + +Native warm lifecycle verification completed 2026-10-31 at11:40:55: parent handle45711 terminal0, full192 tests with zero failures/errors/skips, Checkstyle0 and unchanged coverage gates PASS. Focused42 (lifecycle14/TLS17/authoring11) passed under normal loopback access. Main SHA256 `89B52743EC2EA4CB5670D69BF5927D632FC80A660E4A1093101AE2E3CC07506B`; tests SHA256 `D907AA9CC171A33A7ECC9359D16A4E2ECF980520E7E10073BCD497855F89221C`. No install, shared-cache mutation or deployment occurred. Independent review accepted the five scoped corrections; runtime/supervisor deployment qualification remains separate. The immutable173 proof remains historical and untouched. Earlier handoff notes below are retained as history. + +`PersistentWorkerProvider` implements the optional `WarmWorkerMaintenance` interface. When the producer sees that interface it registers native `QRuntimeServiceInterface` and readiness health metadata. Other providers retain their existing behavior. The cache provider is not ready without explicit `WarmReconciliationConfig` and a declared native service principal. + +The host supplies a principal name, explicit target-table set, maximum active bindings (and every configured target table must exist), maximum unique code bytes, fixed delay, whole-pass budget and maximum confirmed age. The service owns one daemon scheduled future per QQQ instance, with coalesced non-overlap and cancellation on native stop. It closes its provider only at `afterApplicationStop`. No Jobs queue, new registry, record store, global timer or caller session is created. + +Declare the principal through native `ServiceIdentitiesMetaData`; grant `scripting.warm`, `scripting.view`, current native access to the script projects and read access under any credential/channel ceilings. WARM permits bounded code-metadata maintenance only; it does not imply execution, publication, binding or record privileges. Scope the principal's project security keys narrowly. Each pass and readiness check obtains a fresh native service identity in a temporary context with no inherited transaction, authorized action scope or user backend variants. There is no system-session fallback, user delegation or reuse of binding creators' permissions. Ordinary invocation still requires current initiating-user and broker authority. + +An inventory query reads enabled bindings only for the host's explicit target scope, with `maxBindings + 1` limit. Overflow refuses rather than silently truncating readiness. Every selected binding must have current project visibility and an intact publication/build/artifact association verified by the existing HMAC coordinator. The service bounds aggregate unique code bytes, keeps immutable code only during the pass, and stores only bounded identity digests afterward. A changed binding, publication, signed artifact, service declaration, context or policy during preparation invalidates readiness. No source record values or capability tokens are sent to preparation. + +Cold `PING` has exactly protocol/type/session/sequence/requestId. The authenticated peer replies with exact PONG then normal RETIRED(true), without guest allocation, code transfer, inventory or cache mutation. The same live connection/session must retain its ready-code epoch; a supervisor that loses its cache must close the session. Successful PREPARE/INVOKE retirement also refreshes confirmation age. Busy execution slots are skipped, never stolen by heartbeats. Missing code is transferred only on the cold maintenance lane. Ordinary execution never opens a connection, prepares or compiles on a cache miss. + +Readiness independently re-reads current required scoped pins using a fresh metadata identity and verifies all exact local ready IDs/copy counts/profile under one cache lock and one timestamp, plus bounded acknowledgment age. A final metadata re-read follows maintenance callbacks before readiness is accepted. It does not prepare or execute code. An unavailable principal, hidden required project, metadata overflow, changed pin, expired acknowledgment, missing worker or stopped lifecycle reports a generic DOWN outcome with no code/identity details. Readiness concerns this configured scope, not arbitrary inactive publications or other QQQ instances. + +Native database connect/socket/query timeouts and bounded authentication-provider/DNS behavior are host deployment prerequisites. Cooperative deadline checks and cancellation cannot interrupt an unbounded JDBC handshake, authentication call or operating-system DNS resolver. No additional resolver threads are introduced. Real supervisor session/cache-loss behavior, cold compiler availability, multiple-node startup/restart, workload capacity and hostile-runtime qualification remain separate deployment gates. + +Lifecycle restart scope: a stopped lifecycle/provider is terminal. Application restart must construct a fresh ScriptingRuntime and provider; arbitrary stop/start reuse of the same objects is not supported or qualified. Worker connection/cache loss during a running lifecycle is handled by cold reconnection/reconciliation. + +Independent corrected-source review accepted the five scoped corrections (native health timing, target existence, final pin check, aggregate readiness atomicity and volatile lifecycle publication). The new TLS aggregate regression checks partial retirement and recovery; it does not force a concurrent interleaving. Atomicity is established by the single existing slots lock and timestamp in source. Focused42 execution remains pending; no passing result is inferred from review. + +Verification update: parent handle96235 completed with42 focused tests, zero failures/errors/skips and Checkstyle0 (lifecycle14, TLS17, authoring11). Full main handle45711 is running against the same frozen source/private cache with normal loopback access; no full-suite success or install is claimed yet. Prior failed style/compile/native-metadata gates remain preserved. + +Cold-start scaling: steady-state reconciliation performs a fixed number of bounded full inventory/signature scans, linear in the selected bindings plus code bytes. Every newly cold artifact requires additional complete pin/current-project validation immediately before and after transfer. First-cold startup can therefore cost substantially more (up to quadratic metadata work within the configured count and cooperative pass budget). Partial prepared code is reusable on a later pass; no throughput/startup latency/SLA claim is made. Choose bounded scope, capacity and pass budgets using actual deployment measurements. Native JDBC connection/socket/query and authentication/DNS deadlines must be finite independently; thread cancellation is not a substitute. + +Final verification: Native warm lifecycle verification completed 2026-10-31 at11:40:55: parent handle45711 terminal0, full192 tests with zero failures/errors/skips, Checkstyle0 and unchanged coverage gates PASS. Focused42 (lifecycle14/TLS17/authoring11) passed under normal loopback access. Main SHA256 `89B52743EC2EA4CB5670D69BF5927D632FC80A660E4A1093101AE2E3CC07506B`; tests SHA256 `D907AA9CC171A33A7ECC9359D16A4E2ECF980520E7E10073BCD497855F89221C`. No install, shared-cache mutation or deployment occurred. Independent review accepted the five scoped corrections; runtime/supervisor deployment qualification remains separate. + +Immutable proof: [warm-lifecycle-20261031-114806](evidence/warm-lifecycle-20261031-114806/PROOF.md),116 sealed files/92 source inputs. Manifest SHA256 `3B9ACBD28461EDD5FDA0DD17542FC7956FCB06AE523BB52E8C866579E7CA8757`. The snapshot includes filtered summaries and hashes of all six preserved raw gate logs; prior173 evidence is unchanged. + + +## Additional context publications (staged; verification pending) + +A restart previously restored only active hook bindings. Optional `ScriptingRuntime.withWarmPublicationSource(WarmPublicationSource)` adds other required immutable CONTEXT publication pins to the same cold inventory. Its `publications(Set targetTables, int limit)` runs under the freshly declared WARM + VIEW metadata identity, not a publication author or invoking user. The callback must authorize its own current owner metadata, return every required positive distinct publication/artifact/SHA-256 pin, and throw on uncertainty, denied metadata or overflow. An empty list means an authoritatively empty additional inventory; null is an error. The supplied limit is the remaining combined admission budget, possibly zero. No dependency on a consuming QBit is introduced. + +The foundation independently checks current script-project access/isolation, exact publication/build/artifact identity, signed code and a successful TEST_CONTEXT on that exact build. It does not grant EXECUTE to the metadata identity. `maxBindings` retains its configuration name and now bounds active hook rows plus additional distinct publication IDs; identical code is counted/transferred once by hash/entry/toolchain within the shared byte budget. A publication used by both inventories consumes two pin slots but one code allocation. Removed consumer pins stop contributing to readiness; existing worker capacity/epoch eviction remains unchanged. + +The callback object, native context and authority are checked before/after external work. Existing final snapshot rereads and atomic aggregate readiness cover both inventories. Health may read owner metadata but never warms; execute/save hooks never enumerate, connect, prepare or retry. Source changes do not affect immutable publications; consumer pin changes invalidate readiness until cold reconciliation. Owner queries, like native JDBC, need finite deadlines independently of the cooperative pass budget. + +Eleven added protocol-fixture lifecycle tests cover unbound cold startup/reconnect, no hot preparation, changed/removed pins, malformed/uncertain inventories, context qualification/current project access, HMAC/missing artifact refusal, combined count/byte limits, code deduplication and callback authority/final-snapshot changes. They are staged for the root-owned gate; no new passing result or deployed-runtime qualification is claimed. The 192-test sealed baseline above is historical and unchanged. + +## Capacity reclamation and authoring use (2026-11-01, verified source) + +Cold maintenance now retires idle sessions containing code outside the exact required inventory. +A session is the existing cache epoch: removing one discards all its entries, including any still +required entries. The same pass must reprepare and revalidate the complete inventory before UP. +Busy invocation sessions are never stolen. No hot execute/readiness path connects, prepares, +retries or performs reclamation. Changed inventory or maintenance authority aborts the pass. + +`WarmWorkerPreparation.prepareForUse(Artifact)` returns an AutoCloseable local `UseLease`. +Authoring test/context-test/publish/bind/enable holds it from cold preparation through dispatch +or commit; every success/failure closes it. Persistent provider admission is immediate and bounded +by `Configuration.coldOperations` (no wait queue). Code protection uses the existing provider's +slot lock and bounded reference counts; there is no durable registry or timer. All sessions holding +leased code are ineligible for cold retirement. A failed preparation releases its reservation; +release is idempotent and performs no I/O. Existing non-reclaiming providers retain the compatible +default preparation implementation. Actual transport/cache loss can still make execution unavailable. + +Operators need capacity for required pins plus concurrent authoring code and finite native JDBC/ +transport deadlines. A long synchronous authoring operation retains its bounded lease until it +returns; maintenance may defer reclamation or remain DOWN meanwhile. No lease expires underneath +an active operation. First-cold rebuilding can transfer the required set again; there is no claimed +latency or throughput guarantee. Parent full Scripting227 and Generation focused67 gates passed (zero failures/errors/skips, +Checkstyle0, scripting coverage PASS), including TLS and actual H2 migration/replay. The inherited +MySQL migration case remains separate. The earlier sealed192-test result remains evidence only +for that earlier snapshot. See Generation docs/warm-capacity-source-handoff.json for source/log hashes. + + +## Bounded idle-heartbeat skip and local development capacity + +`WarmWorkerMaintenance.heartbeat(budget, confirmedWithin)` lets the native lifecycle avoid a wire +`PING` for an idle session whose last completed exchange already confirmed the same cache epoch. +Only an open, idle slot with no waiting execution whose `PREPARE`, `PING` or `INVOKE` finished with +`RETIRED` inside the window is skipped. Unconfirmed, stale, busy, closed or partial slots are never +treated as ready, a late or incomplete `PONG` is still a failure, deadlines are unchanged and no +invocation is retried. The default method performs the full heartbeat, so existing providers are +unchanged. `WarmReconciliationConfig.heartbeatSkipWindow()` supplies the window: +`min(maxAge / 4, maxAge - interval - passBudget)`, never negative, so a skipped slot can still be +refreshed before readiness age expires. Readiness `maxAge` checks are unchanged. + +Local Docker Desktop finding (not a production claim): a trivial JSSE TLS 1.3 echo through a +Docker-published loopback port showed roughly 10% of 1 Hz exchanges delayed by about 300 ms (some +seconds), while the same exchange with plain TCP, or TLS directly on the host, showed none. +Retransmit-counter correlation and controls are in the runtime lane status; no packet capture was +possible. The supervisor `slots=1` / `idleMillis=60000` worker configuration also meant that one +dropped host session held the only engine for up to a minute. The documented local development +worker uses `slots=2`, `connections=4`, `idleMillis=15000` (heartbeat cadence stays well below the +idle limit). More slots do not replace the handoff/heartbeat code and are not a qualification. diff --git a/docs/production-connection-examples/ApplicationConnection.java.example b/docs/production-connection-examples/ApplicationConnection.java.example new file mode 100644 index 0000000..8028991 --- /dev/null +++ b/docs/production-connection-examples/ApplicationConnection.java.example @@ -0,0 +1,49 @@ +// Documentation example only: not a new configuration loader or application entrypoint. +// The host supplies a verified SSLContext, secret-backed stable coordinator key, +// reviewed deployment evidence, existing backend, and declared least-authority identity. +import javax.net.ssl.SSLContext; +import java.time.Duration; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingQBitConfig; +import com.kingsrook.qbits.scripting.build.BuildCoordinator; +import com.kingsrook.qbits.scripting.rdbms.RdbmsScriptingTransactionPolicy; +import com.kingsrook.qbits.scripting.service.ScriptingRuntime; +import com.kingsrook.qbits.scripting.service.WarmReconciliationConfig; +import com.kingsrook.qbits.scripting.worker.PersistentWorkerProvider; +import com.kingsrook.qbits.scripting.worker.WorkerMessages.ExecutionLimits; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.serviceidentity.QServicePrincipalMetaData; +import com.kingsrook.qqq.backend.core.serviceidentity.ServiceIdentitiesMetaData; + +final class ApplicationConnection +{ + static ScriptingQBitConfig configure(QInstance instance, String backendName, + SSLContext applicationMtls, byte[] coordinatorKey, String coordinatorId, + String reviewedEvidenceId, QServicePrincipalMetaData warmPrincipal) + { + String toolchain = "typescript-6.0.3-node-24"; + // These are example operator DNS names, not automatic Kubernetes discovery. + // Server certificate SANs must match the exact names passed here. + var transport = new PersistentWorkerProvider.Configuration(applicationMtls, + new PersistentWorkerProvider.Endpoint("scripting-execution.workers.svc.cluster.local", 18443), + new PersistentWorkerProvider.Endpoint("scripting-compiler.workers.svc.cluster.local", 18444), + "application-v1", "wasm-64m-v1", toolchain, reviewedEvidenceId, reviewedEvidenceId, + Duration.ofSeconds(30), 4, 2, 64, 8L * 1024 * 1024); + var worker = new PersistentWorkerProvider(transport, 2); + // Host must grant this declared principal scripting.warm, scripting.view and + // current project/source-metadata visibility under its tenant/credential policy. + // It must not inherit an initiating user's session or source-record privileges. + ServiceIdentitiesMetaData.ofOrWithNew(instance).withPrincipal(warmPrincipal); + var runtime = new ScriptingRuntime().withWorker(worker) + .withCoordinator(new BuildCoordinator(coordinatorId, coordinatorKey, toolchain)) + .withRequiredQualificationId(reviewedEvidenceId) + .withTransactionPolicy(new RdbmsScriptingTransactionPolicy()) + .withLimits(ExecutionLimits.hookDefaults()) + .withWarmReconciliation(new WarmReconciliationConfig(warmPrincipal.getName(), + Set.of("purchaseOrder"), 100, 8L * 1024 * 1024, + Duration.ofSeconds(10), Duration.ofSeconds(20), Duration.ofSeconds(60))); + // Existing producer/native application lifecycle owns registration/start/stop. + // Do not connect/prepare from record execution or start a separate timer. + return (new ScriptingQBitConfig().withBackendName(backendName).withRuntime(runtime)); + } +} diff --git a/docs/production-connection-examples/compiler.json.example b/docs/production-connection-examples/compiler.json.example new file mode 100644 index 0000000..dcfe720 --- /dev/null +++ b/docs/production-connection-examples/compiler.json.example @@ -0,0 +1,22 @@ +{ + "bind": "0.0.0.0", + "port": 18444, + "keyStore": "/run/secrets/scripting/server.p12", + "trustStore": "/run/secrets/scripting/client-trust.p12", + "certificateNamespaces": { + "": "application-v1" + }, + "profile": "wasm-64m-v1", + "toolchain": "typescript-6.0.3-node-24", + "evidence": "", + "requiredEvidence": "", + "nodePath": "/opt/compiler/node", + "nodeSha256": "", + "compilerRoot": "/opt/compiler/image", + "manifestSha256": "", + "heapMiB": 256, + "slots": 2, + "connections": 4, + "cpuMillis": 5000, + "wallMillis": 10000 +} diff --git a/docs/production-connection-examples/supervisor.json.example b/docs/production-connection-examples/supervisor.json.example new file mode 100644 index 0000000..148f384 --- /dev/null +++ b/docs/production-connection-examples/supervisor.json.example @@ -0,0 +1,25 @@ +{ + "bind": "0.0.0.0", + "port": 18443, + "keyStore": "/run/secrets/scripting/server.p12", + "trustStore": "/run/secrets/scripting/client-trust.p12", + "certificateNamespaces": { + "": "application-v1" + }, + "profile": "wasm-64m-v1", + "toolchain": "typescript-6.0.3-node-24", + "evidence": "", + "requiredEvidence": "", + "runtimePath": "/opt/scripting/runtime/qqq-script-runtime.wasm", + "runtimeSha256": "", + "nativePath": "/opt/scripting/native/libextism.so", + "nativeSha256": "", + "slots": 8, + "connections": 16, + "coldOperations": 2, + "entries": 64, + "codeBytes": 8388608, + "memoryBytes": 67108864, + "idleMillis": 30000, + "fuel": 10000000 +} diff --git a/docs/release/qrun-release.reference.yaml b/docs/release/qrun-release.reference.yaml new file mode 100644 index 0000000..a063d1a --- /dev/null +++ b/docs/release/qrun-release.reference.yaml @@ -0,0 +1,22 @@ +# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository. +name: qrun-release +on: + push: {} + workflow_dispatch: {} +permissions: + contents: read + actions: read +jobs: + release: + uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-maven.yaml@qrun-release-v1 + with: + runner: linux-small + postpublish: + if: ${{ needs.release.outputs.capture == 'true' }} + needs: release + permissions: + contents: write + actions: write + uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1 + with: + runner: linux-small diff --git a/github/LICENSE b/github/LICENSE new file mode 100644 index 0000000..4bb040e --- /dev/null +++ b/github/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, source + code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 Kingsrook, LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/github/NOTICE b/github/NOTICE new file mode 100644 index 0000000..6fbe46b --- /dev/null +++ b/github/NOTICE @@ -0,0 +1,6 @@ +QBit Scripting +Copyright 2026 Kingsrook, LLC + +This product includes software developed at Kingsrook, LLC (https://kingsrook.com/). + +Licensed under the Apache License, Version 2.0. diff --git a/github/README.md b/github/README.md new file mode 100644 index 0000000..95b1fde --- /dev/null +++ b/github/README.md @@ -0,0 +1,33 @@ +# GitHub source importer + +Optional `com.kingsrook.qbits:qbit-scripting-github:0.1.0-SNAPSHOT` adapter implements the base QBit's `GitSourceImporter` port. It has no dependency on another QBit's credential registry, UI or runtime. Install it explicitly with `runtime.withImporter(new GitHubSourceImporter(hostManagedGitHubAccess))`. + +`GitHubAccess.tokenFor(Actor, scriptId, repository)` must authorize the current actor, project and exact repository on every call, and obtain a read-only short-lived credential from the host's existing managed credential provider. Throw to deny. A null result means the host explicitly authorized anonymous public access. This is not automatic anonymous fallback. Credential text is used only in the trusted Authorization header and is never persisted, logged, returned to a browser or passed to a worker. No new OAuth application, token store or environment-token guessing is implemented. + +Imports require a full lowercase 40-character commit SHA. The adapter uses GitHub.com's fixed HTTPS API host, never follows redirects or response-provided URLs, checks returned commit/tree identities, rejects truncated trees, verifies every blob's Git SHA-1 and declared size, and decodes strict UTF-8. It uses the documented [Git commit](https://docs.github.com/en/rest/git/commits), [tree](https://docs.github.com/en/rest/git/trees) and [blob](https://docs.github.com/en/rest/git/blobs) response shapes. Transport trust is GitHub HTTPS; the adapter does not claim verification of a commit author's cryptographic signature. + +The selected directory must contain ordinary non-executable `.ts` files and directories only. Symlinks, submodules, archives, manifests, JavaScript, unsupported paths and duplicate case-insensitive paths fail the whole import. Use a dedicated source directory in repositories containing README files or package manifests. No clone, checkout, package installation, build hooks or CI scripts run. Branches and tags are not accepted. Import/sync is an explicit action against an exact new commit; there is no automatic webhook or background fetch. + +Bounds: 45-second whole import deadline, at most 15 seconds per HTTP request including body delivery, 1 MiB per encoded response, 1,000 tree entries, 200 source files, 256 KiB per file and 2 MiB decoded source total. Oversized streaming responses cancel before JSON allocation. Hosts still need normal request admission/rate limits. Current repository authorization is reevaluated before each HTTP request; current draft permission and project authorization are reevaluated after network work and before the database transaction commits. + +The service writes a core source revision/files and `scripting_import` receipt in one transaction. The receipt stores repository, commit, selected directory, canonical SHA-256, current actor and timestamp. It is provenance, not another byte registry. Use the matching owner schema/migrations for the explicitly qualified H2 or MySQL deployment; archived PostgreSQL SQL is not production backend qualification. A JDBC test proves receipt failure rolls back every source row. An import never compiles, tests, publishes or binds automatically. + +From the base directory: `./mvnw -f github/pom.xml verify`. Seven trusted HTTP-response/stream tests pass, including mode/path denial, truncation, tampered blob data, repository denial and subtree selection. No private repository or real external HTTP request was used in tests. Host credential integration and production GitHub transport qualification remain deployment work. The existing semantic UI/UX API and Next editor expose this import through the reviewed core isolation authority. + +## Native process clients + +The producer registers `scripting_import_commit` on a selected Script record. Its ordinary +EDIT_FORM asks for `owner/repository`, the full lowercase 40-character commit SHA and an optional +TypeScript-only subdirectory. No token, provider URL or actor field is accepted. With no configured +importer preparation refuses; repository access still belongs to the host-managed `GitHubAccess`. + +The server retains the expected current revision between prepare and apply. Submit delegates to +the same `ScriptingService.importDraft` used by the semantic API and editor. Stale source, revoked +authoring permission and narrowed credential/channel write ceilings refuse before retrieval; +authority and optimistic revision are checked again in the final transaction. Import creates a +new immutable full draft file set and receipt, preserving prior revisions. The result links that +revision and displays its repository/commit/directory/source hash. Check, test and publish remain +separate actions. CLI, TUI and other QQQ process clients need no Next-specific import workflow. + +Native process tests use a trusted source importer fixture with actual owner persistence and +metadata. They do not contact GitHub or establish production credential/transport qualification. diff --git a/github/pom.xml b/github/pom.xml new file mode 100644 index 0000000..1d90a70 --- /dev/null +++ b/github/pom.xml @@ -0,0 +1,49 @@ + + + 4.0.0 + com.kingsrookqbit-build-parent2.0.0 + com.kingsrook.qbitsqbit-scripting-github${revision} + 0.1.0-SNAPSHOT21none4.1.0-RC.214.1.0-RC.21 + + com.kingsrook.qqqqqq-backend-core${qqq.core.version} + com.kingsrook.qqqqqq-backend-module-rdbms${qqq.core.version} + com.kingsrook.qqqqqq-bom-pom${qqq.version}pomimport + org.junitjunit-bom6.0.1pomimport + + + com.kingsrook.qbitsqbit-scripting${revision} + org.junit.jupiterjunit-jupiter6.0.1test + + + + src/main/resourcesfalse + + + + + org.apache.maven.pluginsmaven-resources-plugin3.3.1 + package-license-noticesprocess-resourcescopy-resources + ${project.build.outputDirectory}/META-INF + ${project.basedir}falseLICENSENOTICE + + + + org.apache.maven.pluginsmaven-compiler-plugin3.14.1 + org.apache.maven.pluginsmaven-surefire-plugin3.5.4 + org.apache.maven.pluginsmaven-checkstyle-plugin3.3.1 + com.puppycrawl.toolscheckstyle10.12.6 + ${project.basedir}/../checkstyle/config.xmlcheckstyle.reactor.root=${maven.multiModuleProjectDirectory}truetruewarningtrue + validatecheck + + + + org.codehaus.mojoflatten-maven-plugin1.7.3 + trueossrh + + flattenprocess-resourcesflatten + flatten.cleancleanclean + + + + + diff --git a/github/src/main/java/com/kingsrook/qbits/scripting/github/BoundedBody.java b/github/src/main/java/com/kingsrook/qbits/scripting/github/BoundedBody.java new file mode 100644 index 0000000..88f0188 --- /dev/null +++ b/github/src/main/java/com/kingsrook/qbits/scripting/github/BoundedBody.java @@ -0,0 +1,108 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.github; + +import java.io.ByteArrayOutputStream; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; + +/******************************************************************************* + ** Enforce body size while HTTP bytes arrive, before JSON allocation. + *******************************************************************************/ +final class BoundedBody implements HttpResponse.BodySubscriber +{ + private final int maximum; + private final ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + private final CompletableFuture result = new CompletableFuture<>(); + private Flow.Subscription subscription; + + /*************************************************************************** + ** Per-request maximum includes encoded JSON overhead. + ***************************************************************************/ + BoundedBody(int maximum) + { + this.maximum = maximum; + } + + /*************************************************************************** + ** The HTTP future completes only after bounded body delivery. + ***************************************************************************/ + @Override + public CompletionStage getBody() + { + return (result); + } + + /*************************************************************************** + ** Pull one chunk at a time to bound outstanding input. + ***************************************************************************/ + @Override + public void onSubscribe(Flow.Subscription value) + { + subscription = value; + value.request(1); + } + + /*************************************************************************** + ** Reject oversized content without retaining the remaining body. + ***************************************************************************/ + @Override + public void onNext(List chunks) + { + for(ByteBuffer chunk : chunks) + { + if(chunk.remaining() > maximum - bytes.size()) + { + subscription.cancel(); + result.completeExceptionally(new IllegalArgumentException("GitHub response exceeds limit")); + return; + } + byte[] value = new byte[chunk.remaining()]; + chunk.get(value); + bytes.writeBytes(value); + } + subscription.request(1); + } + + /*************************************************************************** + ** Propagate transport failure without logging its details. + ***************************************************************************/ + @Override + public void onError(Throwable error) + { + result.completeExceptionally(error); + } + + /*************************************************************************** + ** Final immutable response bytes. + ***************************************************************************/ + @Override + public void onComplete() + { + result.complete(bytes.toByteArray()); + } +} + diff --git a/github/src/main/java/com/kingsrook/qbits/scripting/github/GitHubAccess.java b/github/src/main/java/com/kingsrook/qbits/scripting/github/GitHubAccess.java new file mode 100644 index 0000000..ae7ca37 --- /dev/null +++ b/github/src/main/java/com/kingsrook/qbits/scripting/github/GitHubAccess.java @@ -0,0 +1,41 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.github; + +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.service.Actor; + +/******************************************************************************* + ** Host-managed repository authorization and credential lookup. Implementations + ** authorize the current actor/project/repository on EVERY call; deny by throwing. + ** Return null only for explicitly authorized anonymous public access. Tokens + ** stay in trusted HTTP requests, never source records, contexts or diagnostics. + *******************************************************************************/ +@FunctionalInterface +public interface GitHubAccess +{ + /*************************************************************************** + ** Resolve a short-lived read-only credential from the host's existing store. + ***************************************************************************/ + String tokenFor(Actor actor, Integer scriptId, String repository) throws ScriptingException; +} + diff --git a/github/src/main/java/com/kingsrook/qbits/scripting/github/GitHubSourceImporter.java b/github/src/main/java/com/kingsrook/qbits/scripting/github/GitHubSourceImporter.java new file mode 100644 index 0000000..d9d0fec --- /dev/null +++ b/github/src/main/java/com/kingsrook/qbits/scripting/github/GitHubSourceImporter.java @@ -0,0 +1,285 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.github; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.nio.ByteBuffer; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.source.GitCommitRef; +import com.kingsrook.qbits.scripting.source.GitSourceImporter; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import org.json.JSONArray; +import org.json.JSONObject; + +/******************************************************************************* + ** Explicit GitHub.com exact-commit import. No clone, archives, redirects, + ** moving refs, URLs from responses, package installation or repository hooks. + ** Selected subtree must contain only ordinary UTF-8 .ts files and directories. + *******************************************************************************/ +public final class GitHubSourceImporter implements GitSourceImporter +{ + private static final int MAX_RESPONSE = 1024 * 1024; + private static final int MAX_TREE_ENTRIES = 1000; + private final GitHubAccess access; + private final Reader reader; + + /*************************************************************************** + ** Trusted fixture seam; production requests always use the fixed GitHub host. + ***************************************************************************/ + @FunctionalInterface + interface Reader + { + /*********************************************************************** + ** Read one bounded response within the remaining operation deadline. + ***********************************************************************/ + byte[] get(String path, String token, int maximum, Duration timeout) throws Exception; + } + + /*************************************************************************** + ** Host must explicitly install its existing repository authorization policy. + ***************************************************************************/ + public GitHubSourceImporter(GitHubAccess access) + { + this(access, GitHubSourceImporter::read); + } + + /*************************************************************************** + ** Package-private deterministic HTTP-response fixtures, never a browser input. + ***************************************************************************/ + GitHubSourceImporter(GitHubAccess access, Reader reader) + { + this.access = Objects.requireNonNull(access); + this.reader = Objects.requireNonNull(reader); + } + + /*************************************************************************** + ** Verify exact commit identity, selected tree, blob size/hash and source policy. + ***************************************************************************/ + @Override + public SourceSnapshot importSnapshot(Actor actor, Integer scriptId, GitCommitRef ref) throws ScriptingException + { + Actor.requireCurrentIdentity(actor.id()); + long deadline = System.nanoTime() + Duration.ofSeconds(45).toNanos(); + String base = "/repos/" + ref.repository() + "/git/"; + try + { + JSONObject commit = get(actor, scriptId, ref, base + "commits/" + ref.commitSha(), deadline); + require(ref.commitSha().equals(commit.getString("sha")), "Commit identity mismatch"); + String tree = sha(commit.getJSONObject("tree").getString("sha")); + if(ref.subdirectory() != null) + { + for(String segment : ref.subdirectory().split("/")) + { + JSONArray entries = tree(actor, scriptId, ref, base, tree, false, deadline); + String next = null; + for(int i = 0; i < entries.length(); i++) + { + JSONObject entry = entries.getJSONObject(i); + if(segment.equals(entry.getString("path"))) + { + require(next == null && "tree".equals(entry.getString("type")) && "040000".equals(entry.getString("mode")), "Subdirectory must be an ordinary tree"); + next = sha(entry.getString("sha")); + } + } + require(next != null, "Selected source directory is missing"); + tree = next; + } + } + JSONArray entries = tree(actor, scriptId, ref, base, tree, true, deadline); + List blobs = new ArrayList<>(); + Set paths = new HashSet<>(); + long total = 0; + for(int i = 0; i < entries.length(); i++) + { + JSONObject entry = entries.getJSONObject(i); + String path = entry.getString("path"); + require(paths.add(path.toLowerCase(java.util.Locale.ROOT)), "Duplicate source tree path"); + String type = entry.getString("type"); + String mode = entry.getString("mode"); + sha(entry.getString("sha")); + if("tree".equals(type) && "040000".equals(mode)) + { + SourceFile.validatePath(path + "/placeholder.ts"); + continue; + } + require("blob".equals(type) && "100644".equals(mode), "Only ordinary non-executable source files are accepted"); + SourceFile.validatePath(path); + long size = size(entry); + require(size <= SourceFile.MAX_CONTENT_BYTES, "Source file exceeds limit"); + total += size; + require(total <= SourceSnapshot.MAX_TOTAL_BYTES && blobs.size() < SourceSnapshot.MAX_FILES, "Source snapshot exceeds limit"); + blobs.add(entry); + } + List files = new ArrayList<>(); + for(JSONObject entry : blobs) + { + String expected = entry.getString("sha"); + JSONObject blob = get(actor, scriptId, ref, base + "blobs/" + expected, deadline); + require(expected.equals(blob.getString("sha")) && "base64".equals(blob.getString("encoding")), "Invalid blob identity or encoding"); + byte[] bytes = Base64.getDecoder().decode(blob.getString("content").replace("\n", "").replace("\r", "")); + require(bytes.length == size(entry) && bytes.length == size(blob), "Blob length mismatch"); + MessageDigest hash = MessageDigest.getInstance("SHA-1"); + hash.update(("blob " + bytes.length + "\0").getBytes(StandardCharsets.US_ASCII)); + require(expected.equals(SourceSnapshot.hex(hash.digest(bytes))), "Blob content hash mismatch"); + files.add(new SourceFile(entry.getString("path"), utf8(bytes))); + } + return (SourceSnapshot.of(ref.origin(), files)); + } + catch(ScriptingException error) + { + throw error; + } + catch(Exception error) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, "Repository response violates the source import policy"); + } + } + + /*************************************************************************** + ** Reject partial trees; truncation must never silently drop source files. + ***************************************************************************/ + private JSONArray tree(Actor actor, Integer project, GitCommitRef ref, String base, String sha, boolean recursive, long deadline) throws ScriptingException + { + JSONObject response = get(actor, project, ref, base + "trees/" + sha + (recursive ? "?recursive=1" : ""), deadline); + require(sha.equals(response.getString("sha")) && Boolean.FALSE.equals(response.get("truncated")), "Incomplete or mismatched source tree"); + JSONArray entries = response.getJSONArray("tree"); + require(entries.length() <= MAX_TREE_ENTRIES, "Tree entry count exceeds limit"); + return (entries); + } + + /*************************************************************************** + ** Each request reauthorizes current actor/repository; no response URL is used. + ***************************************************************************/ + private JSONObject get(Actor actor, Integer project, GitCommitRef ref, String path, long deadline) throws ScriptingException + { + Actor.requireCurrentIdentity(actor.id()); + String token = access.tokenFor(actor, project, ref.repository()); + long remaining = deadline - System.nanoTime(); + if(remaining <= 0) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Source import deadline exceeded"); + } + try + { + byte[] bytes = reader.get(path, token, MAX_RESPONSE, Duration.ofNanos(Math.min(remaining, Duration.ofSeconds(15).toNanos()))); + require(bytes.length <= MAX_RESPONSE, "Repository response exceeds limit"); + return (new JSONObject(utf8(bytes))); + } + catch(ScriptingException error) + { + throw error; + } + catch(Exception error) + { + if(error instanceof InterruptedException) + { + Thread.currentThread().interrupt(); + } + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Repository content is unavailable"); + } + } + + /*************************************************************************** + ** Fixed public API host; managed credentials cannot follow redirects elsewhere. + ***************************************************************************/ + private static byte[] read(String path, String token, int maximum, Duration timeout) throws Exception + { + try(HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).connectTimeout(Duration.ofSeconds(5)).build()) + { + var request = HttpRequest.newBuilder(URI.create("https://api.github.com" + path)).timeout(timeout) + .header("Accept", "application/vnd.github+json").header("X-GitHub-Api-Version", "2026-03-10") + .header("User-Agent", "QQQ-Scripting").GET(); + if(token != null) + { + request.header("Authorization", "Bearer " + token); + } + var future = client.sendAsync(request.build(), info -> new BoundedBody(maximum)); + try + { + var response = future.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + require(response.statusCode() == 200, "Repository request refused"); + return (response.body()); + } + finally + { + future.cancel(true); + } + } + } + + /*************************************************************************** + ** Numeric byte counts cannot be coerced from strings or fractional JSON. + ***************************************************************************/ + private static long size(JSONObject value) throws ScriptingException + { + Object size = value.get("size"); + require(size instanceof Integer || size instanceof Long, "Invalid blob size"); + long result = ((Number) size).longValue(); + require(result >= 0 && result <= SourceFile.MAX_CONTENT_BYTES, "Blob size exceeds limit"); + return (result); + } + + /*************************************************************************** + ** API object names are full immutable SHA-1 identifiers, never paths or refs. + ***************************************************************************/ + private static String sha(String value) throws ScriptingException + { + require(value.matches("[0-9a-f]{40}"), "Invalid Git object identity"); + return (value); + } + + /*************************************************************************** + ** Reject malformed UTF-8 instead of changing source bytes during decoding. + ***************************************************************************/ + private static String utf8(byte[] bytes) throws Exception + { + return (StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(bytes)).toString()); + } + + /*************************************************************************** + ** Fail the complete import; never return a filtered partial source snapshot. + ***************************************************************************/ + private static void require(boolean condition, String message) throws ScriptingException + { + if(!condition) + { + throw new ScriptingException(ScriptingException.Kind.INVALID, message); + } + } +} + diff --git a/github/src/test/java/com/kingsrook/qbits/scripting/github/GitHubSourceImporterTest.java b/github/src/test/java/com/kingsrook/qbits/scripting/github/GitHubSourceImporterTest.java new file mode 100644 index 0000000..1a3ec49 --- /dev/null +++ b/github/src/test/java/com/kingsrook/qbits/scripting/github/GitHubSourceImporterTest.java @@ -0,0 +1,283 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.github; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Flow; +import java.util.concurrent.atomic.AtomicInteger; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.source.GitCommitRef; +import com.kingsrook.qbits.scripting.source.SourceSnapshot; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/******************************************************************************* + ** Trusted GitHub HTTP response fixtures. No repository is fetched or executed. + *******************************************************************************/ +class GitHubSourceImporterTest +{ + /*************************************************************************** + ** Establish an identified local fixture session without repository access. + ***************************************************************************/ + @BeforeEach + void setup() + { + QContext.init(null, new QSession().withUser(new QUser().withIdReference("author"))); + } + + /*************************************************************************** + ** Drop fixture identity after each test. + ***************************************************************************/ + @AfterEach + void cleanup() + { + QContext.clear(); + } + + /*************************************************************************** + ** Actor is always derived from the actual current fixture session. + ***************************************************************************/ + private Actor everything() throws ScriptingException + { + return (Actor.fromSession(QContext.getQSession())); + } + + private static final String COMMIT = "a".repeat(40); + private static final String TREE = "b".repeat(40); + private static final GitCommitRef REF = new GitCommitRef("acme/scripts", COMMIT, null); + + /*************************************************************************** + ** Exact commit/tree/blob provenance, canonical files and per-request access. + ***************************************************************************/ + @Test + void importsPinnedSourceWithCurrentRepositoryAuthorization() throws Exception + { + Map responses = fixture("export const value = 1;"); + AtomicInteger calls = new AtomicInteger(); + var importer = importer(responses, calls); + var first = importer.importSnapshot(everything(), 42, REF); + var second = importer.importSnapshot(everything(), 42, REF); + assertEquals(first.sha256(), second.sha256()); + assertEquals(REF.origin(), first.origin()); + assertEquals("src/main.ts", first.files().getFirst().path()); + assertTrue(first.verify()); + assertEquals(6, calls.get()); + } + + /*************************************************************************** + ** Unknown files, executable files, symlinks and submodules are all refused. + ***************************************************************************/ + @Test + void refusesUnsafeTreeEntriesBeforeFetchingAnyBlob() throws Exception + { + for(String mode : List.of("100755", "120000", "160000")) + { + var fixture = fixture("source"); + fixture.get("trees/" + TREE + "?recursive=1").getJSONArray("tree").getJSONObject(0).put("mode", mode); + AtomicInteger calls = new AtomicInteger(); + assertThrows(ScriptingException.class, () -> importer(fixture, calls).importSnapshot(everything(), 42, REF)); + assertEquals(2, calls.get()); + } + for(String path : List.of("package.json", "../main.ts", "/main.ts", "src\\main.ts", "file.zip")) + { + var fixture = fixture("source"); + fixture.get("trees/" + TREE + "?recursive=1").getJSONArray("tree").getJSONObject(0).put("path", path); + assertThrows(ScriptingException.class, () -> importer(fixture, new AtomicInteger()).importSnapshot(everything(), 42, REF)); + } + } + + /*************************************************************************** + ** Missing immutable identity or partial trees cannot become usable snapshots. + ***************************************************************************/ + @Test + void rejectsTruncationCommitMismatchAndOversize() throws Exception + { + var truncated = fixture("source"); + truncated.get("trees/" + TREE + "?recursive=1").put("truncated", true); + assertThrows(ScriptingException.class, () -> importer(truncated, new AtomicInteger()).importSnapshot(everything(), 42, REF)); + var mismatch = fixture("source"); + mismatch.get("commits/" + COMMIT).put("sha", TREE); + assertThrows(ScriptingException.class, () -> importer(mismatch, new AtomicInteger()).importSnapshot(everything(), 42, REF)); + var oversized = fixture("source"); + oversized.get("trees/" + TREE + "?recursive=1").getJSONArray("tree").getJSONObject(0).put("size", 300000); + assertThrows(ScriptingException.class, () -> importer(oversized, new AtomicInteger()).importSnapshot(everything(), 42, REF)); + } + + /*************************************************************************** + ** Bytes must match their Git blob hash, declared size and strict UTF-8. + ***************************************************************************/ + @Test + void rejectsChangedBlobAndEncoding() throws Exception + { + for(String field : List.of("content", "size", "encoding", "sha")) + { + var fixture = fixture("source"); + var blob = fixture.entrySet().stream().filter(entry -> entry.getKey().startsWith("blobs/")).findFirst().orElseThrow().getValue(); + blob.put(field, switch(field) + { + case "content" -> Base64.getEncoder().encodeToString("change".getBytes(StandardCharsets.UTF_8)); + case "size" -> 1; + case "encoding" -> "utf-8"; + default -> COMMIT; + }); + assertThrows(ScriptingException.class, () -> importer(fixture, new AtomicInteger()).importSnapshot(everything(), 42, REF)); + } + } + + /*************************************************************************** + ** Managed repository denial prevents all HTTP traffic and cannot expose tokens. + ***************************************************************************/ + @Test + void deniedAccessDoesNotFetch() throws Exception + { + var importer = new GitHubSourceImporter((actor, project, repository) -> + { + throw new ScriptingException(ScriptingException.Kind.DENIED, "Repository is not authorized"); + }, (path, token, maximum, timeout) -> + { + throw new AssertionError("Denied actor must not fetch"); + }); + assertThrows(ScriptingException.class, () -> importer.importSnapshot(everything(), 42, REF)); + } + + /*************************************************************************** + ** Select an exact ordinary subtree; siblings are never downloaded or imported. + ***************************************************************************/ + @Test + void selectsPinnedSubtreeAndRejectsDirectorySymlink() throws Exception + { + var fixture = fixture("source"); + String child = "c".repeat(40); + var original = fixture.remove("trees/" + TREE + "?recursive=1").put("sha", child); + fixture.put("trees/" + child + "?recursive=1", original); + fixture.put("trees/" + TREE, new JSONObject().put("sha", TREE).put("truncated", false).put("tree", List.of( + Map.of("path", "scripts", "type", "tree", "mode", "040000", "sha", child), + Map.of("path", "README.md", "type", "blob", "mode", "100644", "sha", COMMIT)))); + var ref = new GitCommitRef("acme/scripts", COMMIT, "scripts"); + assertEquals(ref.origin(), importer(fixture, new AtomicInteger()).importSnapshot(everything(), 42, ref).origin()); + fixture.get("trees/" + TREE).getJSONArray("tree").getJSONObject(0).put("mode", "120000"); + assertThrows(ScriptingException.class, () -> importer(fixture, new AtomicInteger()).importSnapshot(everything(), 42, ref)); + } + + /*************************************************************************** + ** Transport bounds apply while streaming, rather than after unbounded allocation. + ***************************************************************************/ + @Test + void boundedBodyCancelsOversizedStream() + { + var body = new BoundedBody(2); + AtomicInteger canceled = new AtomicInteger(); + body.onSubscribe(new Flow.Subscription() + { + /******************************************************************* + ** Fixture supplies chunks explicitly. + *******************************************************************/ + @Override + public void request(long count) {} + + /******************************************************************* + ** Observe immediate cancellation. + *******************************************************************/ + @Override + public void cancel() + { + canceled.incrementAndGet(); + } + }); + body.onNext(List.of(ByteBuffer.wrap(new byte[3]))); + assertTrue(body.getBody().toCompletableFuture().isCompletedExceptionally()); + assertEquals(1, canceled.get()); + var small = new BoundedBody(4); + small.onSubscribe(new Flow.Subscription() + { + /******************************************************************* + ** Fixture supplies chunks explicitly. + *******************************************************************/ + @Override + public void request(long count) {} + + /******************************************************************* + ** Successful fixture does not need cancellation. + *******************************************************************/ + @Override + public void cancel() {} + }); + small.onNext(List.of(ByteBuffer.wrap(new byte[2]))); + small.onComplete(); + assertFalse(small.getBody().toCompletableFuture().isCompletedExceptionally()); + assertEquals(2, small.getBody().toCompletableFuture().join().length); + } + + /*************************************************************************** + ** Fixture object responses use the actual commit/tree/blob API shapes. + ***************************************************************************/ + private Map fixture(String content) throws Exception + { + byte[] bytes = content.getBytes(StandardCharsets.UTF_8); + MessageDigest hash = MessageDigest.getInstance("SHA-1"); + hash.update(("blob " + bytes.length + "\0").getBytes(StandardCharsets.US_ASCII)); + String blob = SourceSnapshot.hex(hash.digest(bytes)); + Map result = new HashMap<>(); + result.put("commits/" + COMMIT, new JSONObject().put("sha", COMMIT).put("tree", Map.of("sha", TREE))); + result.put("trees/" + TREE + "?recursive=1", new JSONObject().put("sha", TREE).put("truncated", false).put("tree", + List.of(Map.of("path", "src/main.ts", "mode", "100644", "type", "blob", "sha", blob, "size", bytes.length)))); + result.put("blobs/" + blob, new JSONObject().put("sha", blob).put("encoding", "base64").put("size", bytes.length) + .put("content", Base64.getEncoder().encodeToString(bytes))); + return (result); + } + + /*************************************************************************** + ** Assert the trusted credential stays only at the HTTP boundary. + ***************************************************************************/ + private GitHubSourceImporter importer(Map responses, AtomicInteger calls) + { + return (new GitHubSourceImporter((actor, project, repository) -> + { + assertEquals(42, project); + assertEquals(REF.repository(), repository); + calls.incrementAndGet(); + return ("fixture-credential"); + }, (path, token, maximum, timeout) -> + { + assertEquals("fixture-credential", token); + assertTrue(timeout.toSeconds() <= 15); + return (responses.get(path.substring(("/repos/" + REF.repository() + "/git/").length())).toString().getBytes(StandardCharsets.UTF_8)); + })); + } +} + diff --git a/middleware/LICENSE b/middleware/LICENSE new file mode 100644 index 0000000..4bb040e --- /dev/null +++ b/middleware/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, source + code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 Kingsrook, LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/middleware/NOTICE b/middleware/NOTICE new file mode 100644 index 0000000..6fbe46b --- /dev/null +++ b/middleware/NOTICE @@ -0,0 +1,6 @@ +QBit Scripting +Copyright 2026 Kingsrook, LLC + +This product includes software developed at Kingsrook, LLC (https://kingsrook.com/). + +Licensed under the Apache License, Version 2.0. diff --git a/middleware/README.md b/middleware/README.md new file mode 100644 index 0000000..500a325 --- /dev/null +++ b/middleware/README.md @@ -0,0 +1,42 @@ +# Scripting middleware + +Optional native QQQ UI API extension. The base QBit depends only on core; this artifact supplies authenticated transport and generated OpenAPI, without another source registry or runtime. + +GAV: `com.kingsrook.qbits:qbit-scripting-middleware:0.1.0-SNAPSHOT`. +After producing the base QBit, the host calls `ScriptingApi.register(qInstance)` before middleware startup. The native `/version` discovery response advertises extension `scripting` version `1.0.0`. + +All routes are relative to the host's standard UI API prefix: + +| Method | Path | Purpose | +| --- | --- | --- | +| POST | `/qbit/scripting/projects` | Create a server-marked isolated project; body contains name only | +| GET | `/qbit/scripting/projects/{scriptId}` | Authorized project, recent revisions/builds/publications, available actions | +| GET | `/qbit/scripting/projects/{scriptId}/revisions/{revisionId}` | Immutable source files | +| POST | `/qbit/scripting/projects/{scriptId}/drafts` | Append draft using expected `baseRevisionId` | +| POST | `/qbit/scripting/projects/{scriptId}/imports` | Explicit exact-commit import through the configured host provider | +| POST | `/qbit/scripting/projects/{scriptId}/builds` | Compile saved revision and declared capabilities | +| GET | `/qbit/scripting/projects/{scriptId}/builds/{buildId}` | Bounded compiler diagnostics | +| POST | `/qbit/scripting/projects/{scriptId}/builds/{buildId}/test-context` | Test compiled artifact with bounded JSON context | +| POST | `/qbit/scripting/projects/{scriptId}/builds/{buildId}/publications` | Publish tested immutable artifact | +| POST | `/qbit/scripting/projects/{scriptId}/typings` | Permission-filtered editor declarations | +| GET | `/qbit/scripting/projects/{scriptId}/hooks` | Authorized host phases and project binding page; optional `beforeBindingId` cursor | +| POST | `/qbit/scripting/projects/{scriptId}/bindings` | Create an enabled binding pinned to a publication | +| POST | `/qbit/scripting/projects/{scriptId}/bindings/{bindingId}/state` | Explicit boolean enable/disable | +| POST | `/qbit/scripting/projects/{scriptId}/builds/{buildId}/test-hook` | Read-only selected-record event simulation | + +Current authenticated QQQ session supplies identity; clients cannot supply actors, providers, commands or runtime configuration. Every action rechecks project association and its service permission. Source intake rejects legacy projects and requires the coordinated core isolation guard. This artifact selectively pins reviewed core/RDBMS 4.1.0-isolated-record-files-SNAPSHOT while other QQQ modules remain RC20. Creation and publication use typed core authority; no reflective class-presence gate remains. This local artifact is not a remote release. + +Request limit is 3 MiB before JSON decoding, in addition to source, capability, context and worker bounds enforced by the service. Host ingress/Javalin limits should be configured consistently and remain bounded. Source/build/publication histories expose the latest 100 entries each; hook bindings use bounded keyset pages across all project publications, without bundles, signing keys, signatures or provider settings. Responses use the normal QQQ error envelope; unexpected exceptions return generic text. A timeout does not cancel or undo committed work. Refresh history before deciding whether to retry. + +The generated fragment is `src/main/resources/openapi/qqq-v1/scripting.yaml`. From the base QBit directory: + +```powershell +.\mvnw.cmd -B -ntp install +.\mvnw.cmd -B -ntp -f middleware/pom.xml verify +# Intentional schema changes only: +.\mvnw.cmd -B -ntp -f middleware/pom.xml '-Dscripting.openapi.update=true' test +``` + +Verification: seven tests, including six actual Javalin HTTP tests for create/save/build/test/publish, hook binding controls, current session/association/legacy denials, injected context rejection, generic source-count refusal, worker failure-detail suppression and committed schema comparison for fourteen routes; full wrapper install passes. No sandbox, production database, deployment or full language-service IDE qualification is implied. The optional [GitHub adapter](../github/README.md) is independently tested with trusted response fixtures. + +Hook test bodies contain tableName, event, string recordId and scalar changes only. The server loads prior/current values, validates current permissions and never performs record writes. Binding mutation receipts share the mutation transaction. Worker execution audit commits independently and does not prove business-record commit. Failure responses expose host-owned outcome text, never untrusted worker error details. diff --git a/middleware/pom.xml b/middleware/pom.xml new file mode 100644 index 0000000..018b9c2 --- /dev/null +++ b/middleware/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + com.kingsrookqbit-build-parent2.0.0 + com.kingsrook.qbitsqbit-scripting-middleware${revision} + 0.1.0-SNAPSHOT21none4.1.0-RC.214.1.0-RC.21 + + com.kingsrook.qqqqqq-backend-core${qqq.core.version} + com.kingsrook.qqqqqq-backend-module-rdbms${qqq.core.version} + com.kingsrook.qqqqqq-bom-pom${qqq.version}pomimport + org.junitjunit-bom6.0.1pomimport + + + com.kingsrook.qbitsqbit-scripting${revision} + com.kingsrook.qqqqqq-middleware-javalin${qqq.version} + com.kingsrook.qbitsqbit-scripting${revision}test-jarteststest + org.junit.jupiterjunit-jupiter6.0.1test + + + + src/main/resourcesfalse + + + + + org.apache.maven.pluginsmaven-resources-plugin3.3.1 + package-license-noticesprocess-resourcescopy-resources + ${project.build.outputDirectory}/META-INF + ${project.basedir}falseLICENSENOTICE + + + + org.apache.maven.pluginsmaven-compiler-plugin3.14.1 + org.apache.maven.pluginsmaven-surefire-plugin3.5.4 + org.apache.maven.pluginsmaven-checkstyle-plugin3.3.1 + com.puppycrawl.toolscheckstyle10.12.6 + ${project.basedir}/../checkstyle/config.xmlcheckstyle.reactor.root=${maven.multiModuleProjectDirectory}truetruewarningtrue + validatecheck + + + + org.codehaus.mojoflatten-maven-plugin1.7.3 + trueossrh + + flattenprocess-resourcesflatten + flatten.cleancleanclean + + + + diff --git a/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingApi.java b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingApi.java new file mode 100644 index 0000000..8ac5efa --- /dev/null +++ b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingApi.java @@ -0,0 +1,69 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.api; + +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtension; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.MiddlewareSpecExtensionsMetaData; + + +/******************************************************************************* + ** Optional middleware integration. The base Scripting QBit stays core-only. + *******************************************************************************/ +public final class ScriptingApi +{ + /*************************************************************************** + ** Static registration utility. + ***************************************************************************/ + private ScriptingApi() {} + + /*************************************************************************** + ** Host registers after the base producer, before middleware startup. + ***************************************************************************/ + public static void register(QInstance instance) + { + MiddlewareSpecExtensionsMetaData.ofOrWithNew(instance).withExtension(extension()); + } + + /*************************************************************************** + ** Same descriptor drives discovery, routing and generated OpenAPI. + ***************************************************************************/ + public static MiddlewareSpecExtension extension() + { + return (new MiddlewareSpecExtension().withName("scripting").withSemanticVersion("1.0.0") + .withDescription("Isolated TypeScript source authoring, strict builds, tests and immutable publication") + .withSpec(ScriptingSpecs.CreateProject.class) + .withSpec(ScriptingSpecs.Project.class) + .withSpec(ScriptingSpecs.Revision.class) + .withSpec(ScriptingSpecs.SaveDraft.class) + .withSpec(ScriptingSpecs.ImportDraft.class) + .withSpec(ScriptingSpecs.Check.class) + .withSpec(ScriptingSpecs.Build.class) + .withSpec(ScriptingSpecs.TestContext.class) + .withSpec(ScriptingSpecs.Publish.class) + .withSpec(ScriptingSpecs.Types.class) + .withSpec(ScriptingHookSpecs.Hooks.class) + .withSpec(ScriptingHookSpecs.Bind.class) + .withSpec(ScriptingHookSpecs.SetEnabled.class) + .withSpec(ScriptingHookSpecs.TestHook.class)); + } +} diff --git a/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingHookSpecs.java b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingHookSpecs.java new file mode 100644 index 0000000..f8cddf7 --- /dev/null +++ b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingHookSpecs.java @@ -0,0 +1,283 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.api; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.service.ScriptingResults.BindingSpec; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation; +import com.kingsrook.qqq.openapi.model.HttpMethod; +import com.kingsrook.qqq.openapi.model.Schema; + + +/******************************************************************************* + ** Authenticated project-scoped record hook administration and read-only tests. + *******************************************************************************/ +public final class ScriptingHookSpecs +{ + /*************************************************************************** + ** Holder for native semantic route specifications. + ***************************************************************************/ + private ScriptingHookSpecs() {} + + /*************************************************************************** + ** Discover host-installed current-authority phases and recent bindings. + ***************************************************************************/ + public static final class Hooks extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native route and generated contract share this registration. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (operation(HttpMethod.GET, "/hooks", "Read authorized hook targets and recent bindings")); + } + + /*********************************************************************** + ** Optional positive keyset cursor from the prior binding page. + ***********************************************************************/ + @Override + public java.util.List defineRequestParameters() + { + var parameters = new java.util.ArrayList<>(super.defineRequestParameters()); + parameters.add(new com.kingsrook.qqq.openapi.model.Parameter().withName("beforeBindingId").withIn(com.kingsrook.qqq.openapi.model.In.QUERY) + .withRequired(false).withDescription("Exclusive binding id from nextBindingId.").withSchema(new Schema().withType(com.kingsrook.qqq.openapi.model.Type.INTEGER))); + return (parameters); + } + + /*********************************************************************** + ** Bounded view with explicit recent-history limitation. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingHooksV1"); + } + + /*********************************************************************** + ** The service applies project and native target visibility. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + String cursor = input.getQueryParam("beforeBindingId"); + return (service().hooks().view(actor(), id(input.getPathParam("scriptId")), cursor == null ? null : id(cursor))); + } + } + + /*************************************************************************** + ** Create an immutable pinned binding with explicit narrowed capabilities. + ***************************************************************************/ + public static final class Bind extends ScriptingJsonSpec + { + /*********************************************************************** + ** Binding is separate from source publication. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (operation(HttpMethod.POST, "/bindings", "Bind a publication to a host-installed record phase")); + } + + /*********************************************************************** + ** Newly created binding identity. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingIdV1"); + } + + /*********************************************************************** + ** Exact semantic body, never generic service-owned table writes. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingBindRequestV1")); + } + + /*********************************************************************** + ** Identity, event, publication and all capabilities remain server checked. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "name", "tableName", "event", "publicationId", "capabilities", "patchableFields"); + Object value = input.getBody().get("patchableFields"); + if(!(value instanceof List fields) || fields.size() > 100 || fields.stream().anyMatch(field -> !(field instanceof String text) || !text.matches("[A-Za-z_][A-Za-z0-9_]{0,99}"))) + { + throw new QBadRequestException("Bounded patchable field names are required"); + } + Set patchable = fields.stream().map(String.class::cast).collect(java.util.stream.Collectors.toSet()); + return (Map.of("id", service().hooks().bind(actor(), id(input.getPathParam("scriptId")), + new BindingSpec(text(input.getBody(), "name", 200), text(input.getBody(), "tableName", 200), event(input), + bodyId(input.getBody(), "publicationId", false), capabilities(input), patchable)))); + } + } + + /*************************************************************************** + ** Change only an existing binding's enabled state through its owning service. + ***************************************************************************/ + public static final class SetEnabled extends ScriptingJsonSpec + { + /*********************************************************************** + ** Every mutation is scoped to the source project and concrete binding. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (operation(HttpMethod.POST, "/bindings/{bindingId}/state", "Enable or disable a pinned hook binding")); + } + + /*********************************************************************** + ** Acknowledges the exact binding whose state was committed. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingIdV1"); + } + + /*********************************************************************** + ** Boolean is required without coercion. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingBindingStateRequestV1")); + } + + /*********************************************************************** + ** Current bind permission, project, target and activation checks apply. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "enabled"); + if(!(input.getBody().get("enabled") instanceof Boolean enabled)) + { + throw new QBadRequestException("An explicit enabled boolean is required"); + } + Integer binding = id(input.getPathParam("bindingId")); + service().hooks().setEnabled(actor(), id(input.getPathParam("scriptId")), binding, enabled); + return (Map.of("id", binding)); + } + } + + /*************************************************************************** + ** Test a built event using a real authorized record without applying decisions. + ***************************************************************************/ + public static final class TestHook extends ScriptingJsonSpec + { + /*********************************************************************** + ** A distinct dry-run route never invokes a native mutation. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (operation(HttpMethod.POST, "/builds/{buildId}/test-hook", "Dry-run a hook against an authorized selected record")); + } + + /*********************************************************************** + ** A bounded decision and audit identity, not a mutation result. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingHookTestV1"); + } + + /*********************************************************************** + ** No supplied prior record, actor, runtime or raw source is accepted. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingHookTestRequestV1")); + } + + /*********************************************************************** + ** Selected record and proposed scalar changes are bounded independently. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "tableName", "event", "recordId", "changes"); + Object record = input.getBody().get("recordId"); + if(!(record instanceof String) || record.toString().length() > 1024 || record.toString().isBlank()) + { + throw new QBadRequestException("A bounded scalar record identity is required"); + } + if(!(input.getBody().get("changes") instanceof Map values) || values.size() > 100) + { + throw new QBadRequestException("A bounded proposed changes object is required"); + } + Map changes = new HashMap<>(); + for(Map.Entry entry : values.entrySet()) + { + if(!(entry.getKey() instanceof String field) || entry.getValue() != null && !(entry.getValue() instanceof String || entry.getValue() instanceof Boolean || entry.getValue() instanceof Number)) + { + throw new QBadRequestException("Proposed changes require named JSON scalar fields"); + } + changes.put(field, entry.getValue()); + } + return (service().hooks().test(actor(), id(input.getPathParam("scriptId")), id(input.getPathParam("buildId")), + text(input.getBody(), "tableName", 200), event(input), (Serializable) record, changes)); + } + } + + /*************************************************************************** + ** Consistent authenticated project route declaration. + ***************************************************************************/ + private static BasicOperation operation(HttpMethod method, String suffix, String summary) + { + return (new BasicOperation().withHttpMethod(method).withPath("/qbit/scripting/projects/{scriptId}" + suffix) + .withShortSummary(summary).withLongDescription(summary + ". Current project and native target permissions are enforced; no script receives credentials.")); + } + + /*************************************************************************** + ** Only the six supported event identifiers are accepted. + ***************************************************************************/ + private static HookEvent event(ScriptingJsonSpec.JsonInput input) throws QBadRequestException + { + try + { + Object value = input.getBody().get("event"); + if(!(value instanceof String text)) + { + throw new IllegalArgumentException(); + } + return (HookEvent.valueOf(text)); + } + catch(IllegalArgumentException invalid) + { + throw new QBadRequestException("A supported hook event is required"); + } + } +} diff --git a/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingJsonSpec.java b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingJsonSpec.java new file mode 100644 index 0000000..be7f37a --- /dev/null +++ b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingJsonSpec.java @@ -0,0 +1,233 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.api; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingQBitConfig; +import com.kingsrook.qbits.scripting.capability.CapabilityDeclaration; +import com.kingsrook.qbits.scripting.service.Actor; +import com.kingsrook.qbits.scripting.service.ScriptingService; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QAuthenticationException; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; +import com.kingsrook.qqq.middleware.javalin.QJavalinUtils; +import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse; +import com.kingsrook.qqq.middleware.javalin.specs.extensions.AbstractJsonEndpointSpec; +import com.kingsrook.qqq.openapi.model.In; +import com.kingsrook.qqq.openapi.model.Parameter; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; +import io.javalin.http.Context; +import io.javalin.http.HttpStatus; +import org.json.JSONArray; + + +/******************************************************************************* + ** Native authenticated transport. Services own permissions and source state; + ** no actor, worker configuration, executable or arbitrary process is accepted. + *******************************************************************************/ +public abstract class ScriptingJsonSpec extends AbstractJsonEndpointSpec +{ + /*************************************************************************** + ** Bound request bytes before the framework parses the JSON object. + ***************************************************************************/ + @Override + public JsonInput buildInput(Context context) throws Exception + { + Actor.fromSession(QContext.getQSession()); + if(context.bodyAsBytes().length > 3 * 1024 * 1024) + { + throw new QBadRequestException("Scripting request exceeds 3 MiB"); + } + return (super.buildInput(context)); + } + + /*************************************************************************** + ** Every route is explicitly scoped to one positive core Script identity. + ***************************************************************************/ + @Override + public List defineRequestParameters() + { + var result = new java.util.ArrayList(); + for(String name : List.of("scriptId", "revisionId", "buildId", "bindingId")) + { + if(defineBasicOperation().getPath().contains("{" + name + "}")) + { + result.add(new Parameter().withName(name).withIn(In.PATH).withRequired(true) + .withDescription("Positive immutable " + name + ".").withSchema(new Schema().withType(Type.INTEGER))); + } + } + return (result); + } + + /*************************************************************************** + ** Standard error envelope including qualified-runtime unavailability. + ***************************************************************************/ + @Override + public List defineAdditionalBasicResponses() + { + return (List.of(new BasicResponse(HttpStatus.BAD_REQUEST, "Invalid request", "BasicErrorResponseV1"), + new BasicResponse(HttpStatus.UNAUTHORIZED, "Authentication required", "BasicErrorResponseV1"), + new BasicResponse(HttpStatus.FORBIDDEN, "Current permission required", "BasicErrorResponseV1"), + new BasicResponse(HttpStatus.NOT_FOUND, "Project or revision unavailable", "BasicErrorResponseV1"), + new BasicResponse(HttpStatus.CONFLICT, "Source or artifact integrity conflict", "BasicErrorResponseV1"), + new BasicResponse(HttpStatus.INTERNAL_SERVER_ERROR, "Request could not be completed", "BasicErrorResponseV1"), + new BasicResponse(HttpStatus.SERVICE_UNAVAILABLE, "Isolated runtime unavailable", "BasicErrorResponseV1"))); + } + + /*************************************************************************** + ** Preserve standard error shape and omit unexpected implementation details. + ***************************************************************************/ + @Override + protected void handleException(Context context, Exception error) + { + if(error instanceof QAuthenticationException || error instanceof QPermissionDeniedException || error instanceof QBadRequestException) + { + super.handleException(context, error); + return; + } + if(error instanceof ScriptingException scripting) + { + org.eclipse.jetty.http.HttpStatus.Code status = switch(scripting.getKind()) + { + case DENIED -> org.eclipse.jetty.http.HttpStatus.Code.FORBIDDEN; + case NOT_FOUND -> org.eclipse.jetty.http.HttpStatus.Code.NOT_FOUND; + case INVALID, LIMIT -> org.eclipse.jetty.http.HttpStatus.Code.BAD_REQUEST; + case UNAVAILABLE -> org.eclipse.jetty.http.HttpStatus.Code.SERVICE_UNAVAILABLE; + case INTEGRITY -> org.eclipse.jetty.http.HttpStatus.Code.CONFLICT; + }; + QJavalinUtils.respondWithError(context, status, scripting.getMessage()); + return; + } + QJavalinUtils.respondWithError(context, org.eclipse.jetty.http.HttpStatus.Code.INTERNAL_SERVER_ERROR, "Scripting request could not be completed"); + } + + /*************************************************************************** + ** Resolve only the runtime already installed by the host QBit producer. + ***************************************************************************/ + protected static ScriptingService service() throws ScriptingException + { + var service = ScriptingQBitConfig.serviceFor(QContext.getQInstance()); + if(service == null) + { + throw new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Scripting is not configured"); + } + return (service); + } + + /*************************************************************************** + ** Reject unknown keys, including attempted actor/configuration overrides. + ***************************************************************************/ + protected static void keys(JsonInput input, String... allowed) throws QBadRequestException + { + if(!Set.of(allowed).containsAll(input.getBody().keySet())) + { + throw new QBadRequestException("Unexpected scripting request field"); + } + } + + /*************************************************************************** + ** Strict text validation does not coerce numbers or objects. + ***************************************************************************/ + protected static String text(Map body, String key, int max) throws QBadRequestException + { + Object value = body.get(key); + if(!(value instanceof String string) || string.getBytes(StandardCharsets.UTF_8).length > max) + { + throw new QBadRequestException("A bounded string is required for " + key); + } + return (string); + } + + /*************************************************************************** + ** Path ids are canonical positive integers, without coercing body values. + ***************************************************************************/ + protected static Integer id(String value) throws QBadRequestException + { + try + { + if(value == null || !value.matches("[1-9][0-9]{0,9}")) + { + throw new NumberFormatException(); + } + return (Integer.valueOf(value)); + } + catch(NumberFormatException invalid) + { + throw new QBadRequestException("A positive integer identifier is required"); + } + } + + /*************************************************************************** + ** Body ids must be integer JSON scalars; null is allowed for the first draft. + ***************************************************************************/ + protected static Integer bodyId(Map body, String key, boolean nullable) throws QBadRequestException + { + Object value = body.get(key); + if(value == null && nullable) + { + return (null); + } + if(!(value instanceof Integer integer) || integer < 1) + { + throw new QBadRequestException("A positive integer is required for " + key); + } + return (integer); + } + + /*************************************************************************** + ** Capability lists retain the host's existing bounded typed decoder. + ***************************************************************************/ + protected static List capabilities(JsonInput input) throws Exception + { + Object value = input.getBody().get("capabilities"); + if(!(value instanceof List list) || list.size() > CapabilityDeclaration.MAX_DECLARATIONS) + { + throw new QBadRequestException("A bounded capabilities array is required"); + } + for(Object entry : list) + { + if(!(entry instanceof Map capability) || !capability.keySet().equals(Set.of("operation", "table", "fields", "maxRows", "maxCalls")) + || !"READ".equals(capability.get("operation")) || !(capability.get("table") instanceof String) + || !(capability.get("maxRows") instanceof Integer) || !(capability.get("maxCalls") instanceof Integer) + || !(capability.get("fields") instanceof List fields) || fields.stream().anyMatch(field -> !(field instanceof String))) + { + throw new QBadRequestException("Capabilities require exact typed READ declarations"); + } + } + return (CapabilityDeclaration.listFromJson(new JSONArray(list).toString())); + } + + + /*************************************************************************** + ** Authentication always comes from the middleware-established QQQ session. + ***************************************************************************/ + protected static Actor actor() throws ScriptingException + { + return (Actor.fromSession(QContext.getQSession())); + } +} diff --git a/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingSchemas.java b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingSchemas.java new file mode 100644 index 0000000..d05af02 --- /dev/null +++ b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingSchemas.java @@ -0,0 +1,145 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.api; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import com.kingsrook.qqq.middleware.javalin.specs.v1.responses.BasicErrorResponseV1; +import com.kingsrook.qqq.openapi.model.Schema; +import com.kingsrook.qqq.openapi.model.Type; + + +/******************************************************************************* + ** Explicit versioned authoring wire schemas, independent of source entities. + *******************************************************************************/ +final class ScriptingSchemas +{ + /*************************************************************************** + ** Static contract holder. + ***************************************************************************/ + private ScriptingSchemas() {} + + /*************************************************************************** + ** Every endpoint references the same exact generated schema definitions. + ***************************************************************************/ + static Map schemas() + { + Map all = new TreeMap<>(); + all.put("BasicErrorResponseV1", new BasicErrorResponseV1().toSchema()); + all.put("ScriptingCreateProjectRequestV1", object("Create an isolated project; execution mode is server-controlled.", "name", string())); + all.put("ScriptingFileV1", object("One bounded TypeScript source file.", "path", string(), "content", string())); + all.put("ScriptingRevisionV1", object("Immutable source revision summary.", "id", integer(), "sequence", integer(), "message", string(), "author", string())); + all.put("ScriptingStatusV1", object("Recent lifecycle status, no executable content.", "id", integer(), "revisionId", integer(), "artifactId", integer(), + "version", integer(), "status", string(), "actor", string(), "createdAt", string())); + all.put("ScriptingProjectV1", object("Recent100 revisions/builds/publications and current authorized actions.", "scriptId", integer(), + "executionMode", string(), "latestRevisionId", integer(), "revisions", array("ScriptingRevisionV1"), "builds", array("ScriptingStatusV1"), + "publications", array("ScriptingStatusV1"), "actions", new Schema().withType(Type.ARRAY).withItems(string()), "workerAvailable", bool(), "importAvailable", bool())); + all.put("ScriptingImportProvenanceV1", object("Retained immutable repository source receipt.", "repository", string(), "commitSha", string(), + "subdirectory", string(), "sourceSha256", string(), "importedBy", string())); + all.put("ScriptingDraftV1", object("Authorized source snapshot.", "revisionId", integer(), "sourceSha256", string(), "files", array("ScriptingFileV1"), + "provenance", new Schema().withRefToSchema("ScriptingImportProvenanceV1").withNullable(true))); + all.put("ScriptingImportRequestV1", object("Explicit exact-commit import as a new draft, never build or publish.", + "baseRevisionId", integer(), "repository", string(), "commitSha", string(), "subdirectory", string())); + all.put("ScriptingDraftRequestV1", object("Save appends a revision; baseRevisionId must match the latest revision (null for first save).", + "baseRevisionId", integer(), "message", string(), "files", array("ScriptingFileV1"))); + all.put("ScriptingIdV1", object("Immutable result identity.", "id", integer())); + all.put("ScriptingCapabilityV1", object("Read-only declared capability, checked again on every broker call.", "operation", + string().withEnumValues(List.of("READ")), "table", string(), "fields", new Schema().withType(Type.ARRAY).withItems(string()), + "maxRows", integer(), "maxCalls", integer())); + all.put("ScriptingCheckRequestV1", object("Compile a saved revision with a strict pinned toolchain.", "revisionId", integer(), "entryPoint", string(), + "capabilities", array("ScriptingCapabilityV1"))); + all.put("ScriptingDiagnosticV1", object("Bounded compiler location and message.", "file", string(), "line", integer(), "column", integer(), "code", string(), "message", string())); + all.put("ScriptingBuildV1", object("Compile status; success alone is not publication.", "buildId", integer(), "success", bool(), + "diagnostics", array("ScriptingDiagnosticV1"), "artifactId", integer(), "artifactVersion", integer())); + all.put("ScriptingTestRequestV1", object("Test one built context script with bounded JSON and narrowed read grants.", "input", + new Schema().withType(Type.OBJECT).withAdditionalProperties(true), "capabilities", array("ScriptingCapabilityV1"))); + all.put("ScriptingTestV1", object("Audited context test outcome; no hook shape is accepted.", "executionId", integer(), "status", string(), + "hook", new Schema().withType(Type.OBJECT).withNullable(true), "message", string(), "contextJson", string())); + all.put("ScriptingPublishRequestV1", object("Publish an already built and tested immutable artifact.", "note", string())); + all.put("ScriptingTypesRequestV1", object("Generate scoped editor typings for explicit capabilities.", "capabilities", array("ScriptingCapabilityV1"))); + all.put("ScriptingTypesV1", object("Editor assistance; runtime broker authorization is independent.", "declarations", string())); + all.put("ScriptingHookPhaseV1", object("Installed phase and current permitted patch fields.", "event", string(), + "patchableFields", new Schema().withType(Type.ARRAY).withItems(string()))); + all.put("ScriptingHookTargetV1", object("Host-installed target visible to this actor.", "tableName", string(), "label", string(), "phases", array("ScriptingHookPhaseV1"))); + all.put("ScriptingBindingV1", object("Pinned binding and current enabled state.", "id", integer(), "name", string(), "tableName", string(), "event", string(), + "publicationId", integer(), "enabled", bool(), "grants", array("ScriptingCapabilityV1"), "patchableFields", new Schema().withType(Type.ARRAY).withItems(string()))); + all.put("ScriptingHooksV1", object("Up to200 targets and100 bindings across all publications; nextBindingId continues retained binding history.", + "targets", array("ScriptingHookTargetV1"), "bindings", array("ScriptingBindingV1"), "historyLimited", bool(), "nextBindingId", integer())); + all.put("ScriptingBindRequestV1", object("Create an enabled pinned hook binding, separate from publication.", "name", string(), "tableName", string(), "event", string(), + "publicationId", integer(), "capabilities", array("ScriptingCapabilityV1"), "patchableFields", new Schema().withType(Type.ARRAY).withItems(string()))); + all.put("ScriptingBindingStateRequestV1", object("Change enabled state through the binding service and atomic audit.", "enabled", bool())); + all.put("ScriptingHookTestRequestV1", object("Read-only selected-record simulation; no mutation or native write validation occurs.", "tableName", string(), "event", string(), + "recordId", new Schema().withType(Type.STRING).withDescription("Scalar record primary key, converted by native field metadata."), + "changes", new Schema().withType(Type.OBJECT).withAdditionalProperties(true))); + all.put("ScriptingHookDecisionV1", object("Bounded simulated decision; patch is never applied by the test endpoint.", "action", string(), "message", string(), + "patch", new Schema().withType(Type.OBJECT).withAdditionalProperties(true).withNullable(true))); + all.put("ScriptingHookTestV1", object("Audited dry-run result, never a record write result.", "executionId", integer(), "status", string(), + "hook", new Schema().withRefToSchema("ScriptingHookDecisionV1").withNullable(true), "message", string(), "contextJson", string())); + return (all); + } + + /*************************************************************************** + ** Schema references keep generated clients consistent across endpoints. + ***************************************************************************/ + private static Schema array(String component) + { + return (new Schema().withType(Type.ARRAY).withItems(new Schema().withRefToSchema(component))); + } + + /*************************************************************************** + ** Nullable string; operation-specific validators enforce required values. + ***************************************************************************/ + private static Schema string() + { + return (new Schema().withType(Type.STRING).withNullable(true)); + } + + /*************************************************************************** + ** Positive identities or bounded integer counts, as described by each body. + ***************************************************************************/ + private static Schema integer() + { + return (new Schema().withType(Type.INTEGER).withNullable(true)); + } + + /*************************************************************************** + ** Explicit booleans do not accept string coercion. + ***************************************************************************/ + private static Schema bool() + { + return (new Schema().withType(Type.BOOLEAN)); + } + + /*************************************************************************** + ** Build an exact object shape from trusted schema declarations only. + ***************************************************************************/ + private static Schema object(String description, Object... fields) + { + Map properties = new TreeMap<>(); + for(int i = 0; i < fields.length; i += 2) + { + properties.put((String) fields[i], (Schema) fields[i + 1]); + } + return (new Schema().withType(Type.OBJECT).withDescription(description).withProperties(properties).withAdditionalProperties(false)); + } +} diff --git a/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingSpecs.java b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingSpecs.java new file mode 100644 index 0000000..109748d --- /dev/null +++ b/middleware/src/main/java/com/kingsrook/qbits/scripting/api/ScriptingSpecs.java @@ -0,0 +1,518 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.api; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.scripting.source.GitCommitRef; +import com.kingsrook.qbits.scripting.source.SourceFile; +import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException; +import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation; +import com.kingsrook.qqq.openapi.model.HttpMethod; +import com.kingsrook.qqq.openapi.model.Schema; +import org.json.JSONObject; + + +/******************************************************************************* + ** Small semantic authoring endpoints; no generic code/process execution route. + *******************************************************************************/ +public final class ScriptingSpecs +{ + /*************************************************************************** + ** Static holder of native registered spec classes. + ***************************************************************************/ + private ScriptingSpecs() {} + + /*************************************************************************** + ** Create a separate isolated project without accepting a caller mode flag. + ***************************************************************************/ + public static final class CreateProject extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native authenticated creation endpoint. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/scripting/projects") + .withShortSummary("Create an isolated TypeScript project").withLongDescription("Requires draft permission and ordinary core project insert authorization. Never migrates legacy source.")); + } + + /*********************************************************************** + ** Created project identity. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingIdV1"); + } + + /*********************************************************************** + ** Only a bounded display name is accepted. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingCreateProjectRequestV1")); + } + + /*********************************************************************** + ** Delegate marker creation to the reviewed typed core authority. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "name"); + return (Map.of("id", service().createProject(actor(), text(input.getBody(), "name", 200)))); + } + } + + /*************************************************************************** + ** Read project status and recent history. + ***************************************************************************/ + public static final class Project extends ScriptingJsonSpec + { + /*********************************************************************** + ** Register shared extension schemas once for all semantic routes. + ***********************************************************************/ + @Override + public Map defineComponentSchemas() + { + return (ScriptingSchemas.schemas()); + } + + /*********************************************************************** + ** Native route declaration, authenticated by the UI API pipeline. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/scripting/projects/{scriptId}") + .withShortSummary("Read project status and recent history").withLongDescription("Read project status and recent history. Requires current action permission and owning-project access.")); + } + + /*********************************************************************** + ** Versioned response schema. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingProjectV1"); + } + + /*********************************************************************** + ** Invoke only the already-installed host service under the current user. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (service().authoring().project(actor(), id(input.getPathParam("scriptId")))); + } + } + + /*************************************************************************** + ** Import an explicitly selected immutable Git commit as an ordinary draft. + ***************************************************************************/ + public static final class ImportDraft extends ScriptingJsonSpec + { + /*********************************************************************** + ** No moving refs, provider names, credential fields or webhooks accepted. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/scripting/projects/{scriptId}/imports") + .withShortSummary("Import exact Git commit as draft").withLongDescription("Explicit repository import requires current draft permission and host-managed repository access.")); + } + + /*********************************************************************** + ** A successful response names the new immutable core source revision. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingIdV1"); + } + + /*********************************************************************** + ** The typed body contains provenance, never authorization material. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingImportRequestV1")); + } + + /*********************************************************************** + ** Host authorization and atomic persistence remain in the shared service. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "baseRevisionId", "repository", "commitSha", "subdirectory"); + GitCommitRef ref; + try + { + String directory = input.getBody().get("subdirectory") == null ? null : text(input.getBody(), "subdirectory", 200); + ref = new GitCommitRef(text(input.getBody(), "repository", 201), text(input.getBody(), "commitSha", 40), directory); + } + catch(IllegalArgumentException invalid) + { + throw new QBadRequestException("Import requires a repository, full immutable commit and safe optional subdirectory"); + } + return (Map.of("id", service().importDraft(actor(), id(input.getPathParam("scriptId")), bodyId(input.getBody(), "baseRevisionId", true), ref))); + } + } + + /*************************************************************************** + ** Read one immutable source revision. + ***************************************************************************/ + public static final class Revision extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native route declaration, authenticated by the UI API pipeline. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/scripting/projects/{scriptId}/revisions/{revisionId}") + .withShortSummary("Read one immutable source revision").withLongDescription("Read one immutable source revision. Requires current action permission and owning-project access.")); + } + + /*********************************************************************** + ** Versioned response schema. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingDraftV1"); + } + + /*********************************************************************** + ** Invoke only the already-installed host service under the current user. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (service().authoring().draft(actor(), id(input.getPathParam("scriptId")), id(input.getPathParam("revisionId")))); + } + } + + /*************************************************************************** + ** Save a new draft without activating it. + ***************************************************************************/ + public static final class SaveDraft extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native route declaration, authenticated by the UI API pipeline. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/scripting/projects/{scriptId}/drafts") + .withShortSummary("Save a new draft without activating it").withLongDescription("Save a new draft without activating it. Requires current action permission and owning-project access.")); + } + + /*********************************************************************** + ** Versioned response schema. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingIdV1"); + } + + /*********************************************************************** + ** Versioned exact request body. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingDraftRequestV1")); + } + + /*********************************************************************** + ** Invoke only the already-installed host service under the current user. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "baseRevisionId", "message", "files"); + Integer project = id(input.getPathParam("scriptId")); + var actor = actor(); + service().authoring().requireIsolated(actor, project); + Object raw = input.getBody().get("files"); + if(!(raw instanceof List entries) || entries.size() > 100) + { + throw new QBadRequestException("A bounded files array is required"); + } + List files = new ArrayList<>(); + for(Object entry : entries) + { + if(!(entry instanceof Map map) || !map.keySet().equals(Set.of("path", "content")) + || !(map.get("path") instanceof String path) || !(map.get("content") instanceof String content)) + { + throw new QBadRequestException("Each file needs a path and content"); + } + try + { + files.add(new SourceFile(path, content)); + } + catch(IllegalArgumentException invalid) + { + throw new QBadRequestException("Invalid TypeScript source file"); + } + } + return (Map.of("id", service().saveDraft(actor, project, bodyId(input.getBody(), "baseRevisionId", true), + text(input.getBody(), "message", 2000), files))); + } + } + + /*************************************************************************** + ** Compile one saved revision. + ***************************************************************************/ + public static final class Check extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native route declaration, authenticated by the UI API pipeline. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/scripting/projects/{scriptId}/builds") + .withShortSummary("Compile one saved revision").withLongDescription("Compile one saved revision. Requires current action permission and owning-project access.")); + } + + /*********************************************************************** + ** Versioned response schema. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingBuildV1"); + } + + /*********************************************************************** + ** Versioned exact request body. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingCheckRequestV1")); + } + + /*********************************************************************** + ** Invoke only the already-installed host service under the current user. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "revisionId", "entryPoint", "capabilities"); + Integer project = id(input.getPathParam("scriptId")); + var actor = actor(); + service().authoring().requireIsolated(actor, project); + Integer revision = bodyId(input.getBody(), "revisionId", false); + service().authoring().draft(actor, project, revision); + return (service().check(actor, revision, text(input.getBody(), "entryPoint", 100), capabilities(input))); + } + } + + /*************************************************************************** + ** Read compile diagnostics. + ***************************************************************************/ + public static final class Build extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native route declaration, authenticated by the UI API pipeline. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.GET).withPath("/qbit/scripting/projects/{scriptId}/builds/{buildId}") + .withShortSummary("Read compile diagnostics").withLongDescription("Read compile diagnostics. Requires current action permission and owning-project access.")); + } + + /*********************************************************************** + ** Versioned response schema. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingBuildV1"); + } + + /*********************************************************************** + ** Invoke only the already-installed host service under the current user. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + return (service().authoring().build(actor(), id(input.getPathParam("scriptId")), id(input.getPathParam("buildId")))); + } + } + + /*************************************************************************** + ** Test a built context script. + ***************************************************************************/ + public static final class TestContext extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native route declaration, authenticated by the UI API pipeline. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/scripting/projects/{scriptId}/builds/{buildId}/test-context") + .withShortSummary("Test a built context script").withLongDescription("Test a built context script. Requires current action permission and owning-project access.")); + } + + /*********************************************************************** + ** Versioned response schema. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingTestV1"); + } + + /*********************************************************************** + ** Versioned exact request body. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingTestRequestV1")); + } + + /*********************************************************************** + ** Invoke only the already-installed host service under the current user. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "input", "capabilities"); + Integer project = id(input.getPathParam("scriptId")); + Integer build = id(input.getPathParam("buildId")); + var actor = actor(); + service().authoring().requireBuild(actor, project, build); + if(!(input.getBody().get("input") instanceof Map context)) + { + throw new QBadRequestException("Context test input must be a JSON object"); + } + return (service().testContext(actor, build, new JSONObject(context).toString(), capabilities(input))); + } + } + + /*************************************************************************** + ** Publish a tested immutable build. + ***************************************************************************/ + public static final class Publish extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native route declaration, authenticated by the UI API pipeline. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/scripting/projects/{scriptId}/builds/{buildId}/publications") + .withShortSummary("Publish a tested immutable build").withLongDescription("Publish a tested immutable build. Requires current action permission and owning-project access.")); + } + + /*********************************************************************** + ** Versioned response schema. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingIdV1"); + } + + /*********************************************************************** + ** Versioned exact request body. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingPublishRequestV1")); + } + + /*********************************************************************** + ** Invoke only the already-installed host service under the current user. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "note"); + Integer project = id(input.getPathParam("scriptId")); + Integer build = id(input.getPathParam("buildId")); + var actor = actor(); + service().authoring().requireBuild(actor, project, build); + return (Map.of("id", service().publish(actor, build, text(input.getBody(), "note", 2000)))); + } + } + + /*************************************************************************** + ** Generate permission-scoped API declarations. + ***************************************************************************/ + public static final class Types extends ScriptingJsonSpec + { + /*********************************************************************** + ** Native route declaration, authenticated by the UI API pipeline. + ***********************************************************************/ + @Override + public BasicOperation defineBasicOperation() + { + return (new BasicOperation().withHttpMethod(HttpMethod.POST).withPath("/qbit/scripting/projects/{scriptId}/typings") + .withShortSummary("Generate permission-scoped API declarations").withLongDescription("Generate permission-scoped API declarations. Requires current action permission and owning-project access.")); + } + + /*********************************************************************** + ** Versioned response schema. + ***********************************************************************/ + @Override + protected String getResponseSchemaName() + { + return ("ScriptingTypesV1"); + } + + /*********************************************************************** + ** Versioned exact request body. + ***********************************************************************/ + @Override + protected Schema defineRequestBodySchema() + { + return (new Schema().withRefToSchema("ScriptingTypesRequestV1")); + } + + /*********************************************************************** + ** Invoke only the already-installed host service under the current user. + ***********************************************************************/ + @Override + protected Object handle(JsonInput input) throws Exception + { + keys(input, "capabilities"); + return (Map.of("declarations", service().authoring().typings(actor(), id(input.getPathParam("scriptId")), capabilities(input)))); + } + } +} diff --git a/middleware/src/main/resources/openapi/qqq-v1/scripting.yaml b/middleware/src/main/resources/openapi/qqq-v1/scripting.yaml new file mode 100644 index 0000000..fc43fa6 --- /dev/null +++ b/middleware/src/main/resources/openapi/qqq-v1/scripting.yaml @@ -0,0 +1,1774 @@ +components: + schemas: + BasicErrorResponseV1: + description: "The error body of every UI API operation (core and extensions)." + properties: + error: + description: "The error message (the same as message; kept for clients of\ + \ UI API 1.0 to 1.5)." + type: "string" + code: + description: "A stable, machine-readable code. Well-known values: BAD_REQUEST\ + \ (400), UNAUTHENTICATED (401),\nPERMISSION_DENIED (403), NOT_FOUND (404),\ + \ CONFLICT (409), RATE_LIMITED (429), INTERNAL_ERROR (500),\nERROR (other).\ + \ Extensions may add codes; treat an unknown code by the HTTP status.\ + \ Since 1.6.0." + type: "string" + message: + description: "A message to show the user. Since 1.6.0." + type: "string" + fieldErrors: + description: "Errors about particular request fields, when known. Since\ + \ 1.6.0." + items: + properties: + field: + description: "The field (or parameter) name." + type: "string" + message: + description: "The error about that field." + type: "string" + code: + description: "An optional machine-readable code, e.g. REQUIRED, TOO_LONG,\ + \ PATTERN." + type: "string" + type: "object" + type: "array" + traceId: + description: "Identifies this request in the server logs (also the X-QQQ-Trace-Id\ + \ response header); quote it when reporting a problem. Since 1.6.0." + type: "string" + type: "object" + ScriptingBindRequestV1: + additionalProperties: false + description: "Create an enabled pinned hook binding, separate from publication." + properties: + capabilities: + items: + $ref: "#/components/schemas/ScriptingCapabilityV1" + type: "array" + event: + nullable: true + type: "string" + name: + nullable: true + type: "string" + patchableFields: + items: + nullable: true + type: "string" + type: "array" + publicationId: + nullable: true + type: "integer" + tableName: + nullable: true + type: "string" + type: "object" + ScriptingBindingStateRequestV1: + additionalProperties: false + description: "Change enabled state through the binding service and atomic audit." + properties: + enabled: + type: "boolean" + type: "object" + ScriptingBindingV1: + additionalProperties: false + description: "Pinned binding and current enabled state." + properties: + enabled: + type: "boolean" + event: + nullable: true + type: "string" + grants: + items: + $ref: "#/components/schemas/ScriptingCapabilityV1" + type: "array" + id: + nullable: true + type: "integer" + name: + nullable: true + type: "string" + patchableFields: + items: + nullable: true + type: "string" + type: "array" + publicationId: + nullable: true + type: "integer" + tableName: + nullable: true + type: "string" + type: "object" + ScriptingBuildV1: + additionalProperties: false + description: "Compile status; success alone is not publication." + properties: + artifactId: + nullable: true + type: "integer" + artifactVersion: + nullable: true + type: "integer" + buildId: + nullable: true + type: "integer" + diagnostics: + items: + $ref: "#/components/schemas/ScriptingDiagnosticV1" + type: "array" + success: + type: "boolean" + type: "object" + ScriptingCapabilityV1: + additionalProperties: false + description: "Read-only declared capability, checked again on every broker call." + properties: + fields: + items: + nullable: true + type: "string" + type: "array" + maxCalls: + nullable: true + type: "integer" + maxRows: + nullable: true + type: "integer" + operation: + enum: + - "READ" + nullable: true + type: "string" + table: + nullable: true + type: "string" + type: "object" + ScriptingCheckRequestV1: + additionalProperties: false + description: "Compile a saved revision with a strict pinned toolchain." + properties: + capabilities: + items: + $ref: "#/components/schemas/ScriptingCapabilityV1" + type: "array" + entryPoint: + nullable: true + type: "string" + revisionId: + nullable: true + type: "integer" + type: "object" + ScriptingCreateProjectRequestV1: + additionalProperties: false + description: "Create an isolated project; execution mode is server-controlled." + properties: + name: + nullable: true + type: "string" + type: "object" + ScriptingDiagnosticV1: + additionalProperties: false + description: "Bounded compiler location and message." + properties: + code: + nullable: true + type: "string" + column: + nullable: true + type: "integer" + file: + nullable: true + type: "string" + line: + nullable: true + type: "integer" + message: + nullable: true + type: "string" + type: "object" + ScriptingDraftRequestV1: + additionalProperties: false + description: "Save appends a revision; baseRevisionId must match the latest\ + \ revision (null for first save)." + properties: + baseRevisionId: + nullable: true + type: "integer" + files: + items: + $ref: "#/components/schemas/ScriptingFileV1" + type: "array" + message: + nullable: true + type: "string" + type: "object" + ScriptingDraftV1: + additionalProperties: false + description: "Authorized source snapshot." + properties: + files: + items: + $ref: "#/components/schemas/ScriptingFileV1" + type: "array" + provenance: + $ref: "#/components/schemas/ScriptingImportProvenanceV1" + nullable: true + revisionId: + nullable: true + type: "integer" + sourceSha256: + nullable: true + type: "string" + type: "object" + ScriptingFileV1: + additionalProperties: false + description: "One bounded TypeScript source file." + properties: + content: + nullable: true + type: "string" + path: + nullable: true + type: "string" + type: "object" + ScriptingHookDecisionV1: + additionalProperties: false + description: "Bounded simulated decision; patch is never applied by the test\ + \ endpoint." + properties: + action: + nullable: true + type: "string" + message: + nullable: true + type: "string" + patch: + additionalProperties: true + nullable: true + type: "object" + type: "object" + ScriptingHookPhaseV1: + additionalProperties: false + description: "Installed phase and current permitted patch fields." + properties: + event: + nullable: true + type: "string" + patchableFields: + items: + nullable: true + type: "string" + type: "array" + type: "object" + ScriptingHookTargetV1: + additionalProperties: false + description: "Host-installed target visible to this actor." + properties: + label: + nullable: true + type: "string" + phases: + items: + $ref: "#/components/schemas/ScriptingHookPhaseV1" + type: "array" + tableName: + nullable: true + type: "string" + type: "object" + ScriptingHookTestRequestV1: + additionalProperties: false + description: "Read-only selected-record simulation; no mutation or native write\ + \ validation occurs." + properties: + changes: + additionalProperties: true + type: "object" + event: + nullable: true + type: "string" + recordId: + description: "Scalar record primary key, converted by native field metadata." + type: "string" + tableName: + nullable: true + type: "string" + type: "object" + ScriptingHookTestV1: + additionalProperties: false + description: "Audited dry-run result, never a record write result." + properties: + contextJson: + nullable: true + type: "string" + executionId: + nullable: true + type: "integer" + hook: + $ref: "#/components/schemas/ScriptingHookDecisionV1" + nullable: true + message: + nullable: true + type: "string" + status: + nullable: true + type: "string" + type: "object" + ScriptingHooksV1: + additionalProperties: false + description: "Up to200 targets and100 bindings across all publications; nextBindingId\ + \ continues retained binding history." + properties: + bindings: + items: + $ref: "#/components/schemas/ScriptingBindingV1" + type: "array" + historyLimited: + type: "boolean" + nextBindingId: + nullable: true + type: "integer" + targets: + items: + $ref: "#/components/schemas/ScriptingHookTargetV1" + type: "array" + type: "object" + ScriptingIdV1: + additionalProperties: false + description: "Immutable result identity." + properties: + id: + nullable: true + type: "integer" + type: "object" + ScriptingImportProvenanceV1: + additionalProperties: false + description: "Retained immutable repository source receipt." + properties: + commitSha: + nullable: true + type: "string" + importedBy: + nullable: true + type: "string" + repository: + nullable: true + type: "string" + sourceSha256: + nullable: true + type: "string" + subdirectory: + nullable: true + type: "string" + type: "object" + ScriptingImportRequestV1: + additionalProperties: false + description: "Explicit exact-commit import as a new draft, never build or publish." + properties: + baseRevisionId: + nullable: true + type: "integer" + commitSha: + nullable: true + type: "string" + repository: + nullable: true + type: "string" + subdirectory: + nullable: true + type: "string" + type: "object" + ScriptingProjectV1: + additionalProperties: false + description: "Recent100 revisions/builds/publications and current authorized\ + \ actions." + properties: + actions: + items: + nullable: true + type: "string" + type: "array" + builds: + items: + $ref: "#/components/schemas/ScriptingStatusV1" + type: "array" + executionMode: + nullable: true + type: "string" + importAvailable: + type: "boolean" + latestRevisionId: + nullable: true + type: "integer" + publications: + items: + $ref: "#/components/schemas/ScriptingStatusV1" + type: "array" + revisions: + items: + $ref: "#/components/schemas/ScriptingRevisionV1" + type: "array" + scriptId: + nullable: true + type: "integer" + workerAvailable: + type: "boolean" + type: "object" + ScriptingPublishRequestV1: + additionalProperties: false + description: "Publish an already built and tested immutable artifact." + properties: + note: + nullable: true + type: "string" + type: "object" + ScriptingRevisionV1: + additionalProperties: false + description: "Immutable source revision summary." + properties: + author: + nullable: true + type: "string" + id: + nullable: true + type: "integer" + message: + nullable: true + type: "string" + sequence: + nullable: true + type: "integer" + type: "object" + ScriptingStatusV1: + additionalProperties: false + description: "Recent lifecycle status, no executable content." + properties: + actor: + nullable: true + type: "string" + artifactId: + nullable: true + type: "integer" + createdAt: + nullable: true + type: "string" + id: + nullable: true + type: "integer" + revisionId: + nullable: true + type: "integer" + status: + nullable: true + type: "string" + version: + nullable: true + type: "integer" + type: "object" + ScriptingTestRequestV1: + additionalProperties: false + description: "Test one built context script with bounded JSON and narrowed read\ + \ grants." + properties: + capabilities: + items: + $ref: "#/components/schemas/ScriptingCapabilityV1" + type: "array" + input: + additionalProperties: true + type: "object" + type: "object" + ScriptingTestV1: + additionalProperties: false + description: "Audited context test outcome; no hook shape is accepted." + properties: + contextJson: + nullable: true + type: "string" + executionId: + nullable: true + type: "integer" + hook: + nullable: true + type: "object" + message: + nullable: true + type: "string" + status: + nullable: true + type: "string" + type: "object" + ScriptingTypesRequestV1: + additionalProperties: false + description: "Generate scoped editor typings for explicit capabilities." + properties: + capabilities: + items: + $ref: "#/components/schemas/ScriptingCapabilityV1" + type: "array" + type: "object" + ScriptingTypesV1: + additionalProperties: false + description: "Editor assistance; runtime broker authorization is independent." + properties: + declarations: + nullable: true + type: "string" + type: "object" + securitySchemes: + sessionUuidCookie: + description: "The session cookie set by POST /qqq/v1/manageSession (web clients)." + in: "cookie" + name: "sessionUUID" + type: "apiKey" + bearerAuth: + bearerFormat: "JWT or session UUID" + description: "`Authorization: Bearer `, where the token is either\n-\ + \ an OAuth2 access token (a signed JWT) issued by the app's identity provider\ + \ to the app's client,\n or to its native client (`values.nativeClientId`\ + \ in GET /qqq/v1/metaData/authentication), for\n example from the device-authorization\ + \ grant or auth-code + PKCE. It is verified against the\n provider's keys\ + \ (issuer, audience, expiry, signature) and gives the same user and permissions\n\ + \ as signing in on the web. When it expires (401), refresh it at the provider\ + \ and retry; or\n- a session UUID returned by POST /qqq/v1/manageSession,\ + \ which the server keeps refreshed." + scheme: "bearer" + type: "http" + oauth2: + description: "The app's OAuth2 / OpenID Connect provider. The real endpoints\ + \ come from the provider's discovery\ndocument: `{values.baseUrl}.well-known/openid-configuration`,\ + \ with values from\nGET /qqq/v1/metaData/authentication. Native clients use\ + \ `values.nativeClientId` (a public client)\nwith auth-code + PKCE (loopback\ + \ or custom-scheme redirect URI registered at the provider) or the\ndevice-authorization\ + \ grant (`device_authorization_endpoint`, RFC 8628), then send the access\ + \ token\nas `Authorization: Bearer`." + flows: + authorizationCode: + authorizationUrl: "/authorize" + scopes: + email: "Email address" + offline_access: "A refresh token, to renew the access token without signing\ + \ in again" + openid: "OpenID Connect sign-in" + profile: "Name, username and groups (permissions come from the groups)" + tokenUrl: "/token" + type: "oauth2" +info: + contact: + email: "contact@kingsrook.com" + description: "Isolated TypeScript source authoring, strict builds, tests and immutable\ + \ publication" + title: "QQQ Middleware API extension: scripting" + version: "1.0.0" +openapi: "3.0.3" +paths: + /qqq/v1/qbit/scripting/projects: + post: + description: "Requires draft permission and ordinary core project insert authorization.\ + \ Never migrates legacy source." + operationId: "postQbitScriptingProjects" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingCreateProjectRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingIdV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Create an isolated TypeScript project" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}: + get: + description: "Read project status and recent history. Requires current action\ + \ permission and owning-project access." + operationId: "getQbitScriptingProjectsByScriptId" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingProjectV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Read project status and recent history" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/revisions/{revisionId}: + get: + description: "Read one immutable source revision. Requires current action permission\ + \ and owning-project access." + operationId: "getQbitScriptingProjectsByScriptIdRevisionsByRevisionId" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + - description: "Positive immutable revisionId." + in: "path" + name: "revisionId" + required: true + schema: + type: "integer" + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingDraftV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Read one immutable source revision" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/drafts: + post: + description: "Save a new draft without activating it. Requires current action\ + \ permission and owning-project access." + operationId: "postQbitScriptingProjectsByScriptIdDrafts" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingDraftRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingIdV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Save a new draft without activating it" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/imports: + post: + description: "Explicit repository import requires current draft permission and\ + \ host-managed repository access." + operationId: "postQbitScriptingProjectsByScriptIdImports" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingImportRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingIdV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Import exact Git commit as draft" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/builds: + post: + description: "Compile one saved revision. Requires current action permission\ + \ and owning-project access." + operationId: "postQbitScriptingProjectsByScriptIdBuilds" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingCheckRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingBuildV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Compile one saved revision" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/builds/{buildId}: + get: + description: "Read compile diagnostics. Requires current action permission and\ + \ owning-project access." + operationId: "getQbitScriptingProjectsByScriptIdBuildsByBuildId" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + - description: "Positive immutable buildId." + in: "path" + name: "buildId" + required: true + schema: + type: "integer" + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingBuildV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Read compile diagnostics" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/builds/{buildId}/test-context: + post: + description: "Test a built context script. Requires current action permission\ + \ and owning-project access." + operationId: "postQbitScriptingProjectsByScriptIdBuildsByBuildIdTestContext" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + - description: "Positive immutable buildId." + in: "path" + name: "buildId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingTestRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingTestV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Test a built context script" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/builds/{buildId}/publications: + post: + description: "Publish a tested immutable build. Requires current action permission\ + \ and owning-project access." + operationId: "postQbitScriptingProjectsByScriptIdBuildsByBuildIdPublications" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + - description: "Positive immutable buildId." + in: "path" + name: "buildId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingPublishRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingIdV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Publish a tested immutable build" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/typings: + post: + description: "Generate permission-scoped API declarations. Requires current\ + \ action permission and owning-project access." + operationId: "postQbitScriptingProjectsByScriptIdTypings" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingTypesRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingTypesV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Generate permission-scoped API declarations" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/hooks: + get: + description: "Read authorized hook targets and recent bindings. Current project\ + \ and native target permissions are enforced; no script receives credentials." + operationId: "getQbitScriptingProjectsByScriptIdHooks" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + - description: "Exclusive binding id from nextBindingId." + in: "query" + name: "beforeBindingId" + required: false + schema: + type: "integer" + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingHooksV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Read authorized hook targets and recent bindings" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/bindings: + post: + description: "Bind a publication to a host-installed record phase. Current project\ + \ and native target permissions are enforced; no script receives credentials." + operationId: "postQbitScriptingProjectsByScriptIdBindings" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingBindRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingIdV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Bind a publication to a host-installed record phase" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/bindings/{bindingId}/state: + post: + description: "Enable or disable a pinned hook binding. Current project and native\ + \ target permissions are enforced; no script receives credentials." + operationId: "postQbitScriptingProjectsByScriptIdBindingsByBindingIdState" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + - description: "Positive immutable bindingId." + in: "path" + name: "bindingId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingBindingStateRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingIdV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Enable or disable a pinned hook binding" + tags: + - "scripting" + /qqq/v1/qbit/scripting/projects/{scriptId}/builds/{buildId}/test-hook: + post: + description: "Dry-run a hook against an authorized selected record. Current\ + \ project and native target permissions are enforced; no script receives credentials." + operationId: "postQbitScriptingProjectsByScriptIdBuildsByBuildIdTestHook" + parameters: + - description: "Positive immutable scriptId." + in: "path" + name: "scriptId" + required: true + schema: + type: "integer" + - description: "Positive immutable buildId." + in: "path" + name: "buildId" + required: true + schema: + type: "integer" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingHookTestRequestV1" + required: true + responses: + 400: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Invalid request" + 401: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Authentication required" + 403: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Current permission required" + 500: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Request could not be completed" + 404: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Project or revision unavailable" + 503: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Isolated runtime unavailable" + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptingHookTestV1" + description: "Successful Response" + 409: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Source or artifact integrity conflict" + 429: + content: + application/json: + schema: + $ref: "#/components/schemas/BasicErrorResponseV1" + description: "Too many requests (code RATE_LIMITED); retry later." + security: + - sessionUuidCookie: + - "N/A" + - bearerAuth: + - "N/A" + - oauth2: + - "openid" + - "profile" + - "email" + summary: "Dry-run a hook against an authorized selected record" + tags: + - "scripting" +servers: +- description: "The QQQ server that serves this document (paths include /qqq/v1)." + url: "/" +tags: +- description: "Operations of the scripting extension." + name: "scripting" diff --git a/middleware/src/test/java/com/kingsrook/qbits/scripting/ScriptingApiHttpTest.java b/middleware/src/test/java/com/kingsrook/qbits/scripting/ScriptingApiHttpTest.java new file mode 100644 index 0000000..389f166 --- /dev/null +++ b/middleware/src/test/java/com/kingsrook/qbits/scripting/ScriptingApiHttpTest.java @@ -0,0 +1,293 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting; + +import java.net.ServerSocket; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qbits.scripting.api.ScriptingApi; +import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleCustomizerInterface; +import com.kingsrook.qqq.middleware.javalin.QApplicationJavalinServer; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Real registered HTTP pipeline and QQQ sessions, never direct handler calls. + ** Worker is protocol-only; old core must refuse HTTP source intake. + *******************************************************************************/ +class ScriptingApiHttpTest extends ScriptingTestBase +{ + private QApplicationJavalinServer server; + private HttpClient client; + private String base; + private String suffix; + private Integer project; + private Integer revision; + private Integer legacy; + + /*************************************************************************** + ** Install native extension under strict route auditing with fixture auth. + ***************************************************************************/ + @BeforeEach + void start() throws Exception + { + legacy = newLegacyProject(); + project = newProject(); + revision = draft(project, "source"); + instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK) + .withCustomizer(new QCodeReference(Sessions.class))); + ScriptingApi.register(instance); + int port; + try(ServerSocket socket = new ServerSocket(0)) + { + port = socket.getLocalPort(); + } + suffix = UUID.randomUUID().toString(); + base = "http://127.0.0.1:" + port + "/qqq/v1"; + client = HttpClient.newHttpClient(); + server = new QApplicationJavalinServer(new AbstractQQQApplication() + { + /*********************************************************************** + ** Return this test's already-populated real metadata instance. + ***********************************************************************/ + @Override + public QInstance defineQInstance() + { + return (instance); + } + }).withPort(port).withServeFrontendNext(false).withServeFrontendMaterialDashboard(false) + .withServeLegacyUnversionedMiddlewareAPI(false).withStrictUiApiRoutes(true); + server.start(); + } + + /*************************************************************************** + ** Release real HTTP resources after every fixture. + ***************************************************************************/ + @AfterEach + void stop() + { + if(server != null) + { + server.stop(); + } + if(client != null) + { + client.close(); + } + } + + /*************************************************************************** + ** Session permissions, project scoping and standard discovery work together. + ***************************************************************************/ + @Test + void projectAndSourceUseCurrentSession() throws Exception + { + var response = request("GET", "/qbit/scripting/projects/" + project, null, "alice"); + assertEquals(200, response.statusCode(), response.body()); + JSONObject json = new JSONObject(response.body()); + assertEquals(revision, json.getInt("latestRevisionId")); + assertFalse(response.body().contains("attestation")); + var source = request("GET", "/qbit/scripting/projects/" + project + "/revisions/" + revision, null, "alice"); + assertEquals(200, source.statusCode(), source.body()); + assertTrue(source.body().contains("source")); + assertEquals(403, request("GET", "/qbit/scripting/projects/" + project, null, "reader").statusCode()); + assertEquals(400, request("GET", "/qbit/scripting/projects/0", null, "alice").statusCode()); + assertTrue(request("GET", "/version", null, "alice").body().contains("scripting")); + } + + /*************************************************************************** + ** Typed JSON rejects actor injection and numeric capability coercion. + ***************************************************************************/ + @Test + void semanticRequestsRejectExtraAndCoercedFields() throws Exception + { + String path = "/qbit/scripting/projects/" + project + "/typings"; + var ok = request("POST", path, "{\"capabilities\":[]}", "alice"); + assertEquals(200, ok.statusCode(), ok.body()); + assertTrue(ok.body().contains("@qqq/sdk")); + assertEquals(400, request("POST", path, "{\"capabilities\":[],\"actorId\":\"admin\"}", "alice").statusCode()); + assertEquals(400, request("POST", path, "{\"capabilities\":[{\"operation\":\"READ\",\"table\":\"purchase_order\",\"fields\":[\"id\"],\"maxRows\":\"1\",\"maxCalls\":1}]}", "alice").statusCode()); + assertEquals(403, request("POST", path, "{\"capabilities\":[]}", "reader").statusCode()); + } + + /*************************************************************************** + ** Current published RC20 lacks the new guard. No HTTP source is persisted + ** until the coordinated core guard is installed and integrated. + ***************************************************************************/ + @Test + void legacyProjectCannotEnterIsolatedAuthoring() throws Exception + { + String path = "/qbit/scripting/projects/" + legacy + "/drafts"; + var answer = request("POST", path, "{\"baseRevisionId\":null,\"message\":\"draft\",\"files\":[]}", "alice"); + assertTrue(answer.statusCode() == 503 || answer.statusCode() == 403, answer.body()); + assertEquals(0, worker.executeCalls); + } + + /*************************************************************************** + ** Generic source count routes cannot disclose rows before parent filtering. + ***************************************************************************/ + @Test + void genericSourceTotalsAreDenied() throws Exception + { + for(String table : java.util.List.of("scriptRevision", "scriptRevisionFile")) + { + var response = request("POST", "/table/" + table + "/count?includeDistinct=false", "{}", "alice"); + assertEquals(403, response.statusCode(), response.body()); + assertFalse(response.body().contains("\"count\":")); + } + } + + /*************************************************************************** + ** Native API exercises the reviewed core marker and every authoring stage. + ***************************************************************************/ + @Test + void isolatedLifecycleRunsThroughNativeHttp() throws Exception + { + var created = request("POST", "/qbit/scripting/projects", "{\"name\":\"Context formatter\"}", "alice"); + assertEquals(200, created.statusCode(), created.body()); + int id = new JSONObject(created.body()).getInt("id"); + String path = "/qbit/scripting/projects/" + id; + assertTrue(request("GET", path, null, "alice").body().contains("ISOLATED")); + String source = new JSONObject().put("baseRevisionId", JSONObject.NULL).put("message", "First draft") + .put("files", new org.json.JSONArray().put(new JSONObject().put("path", ENTRY).put("content", "export async function main() { return {}; }"))).toString(); + var saved = request("POST", path + "/drafts", source, "alice"); + assertEquals(200, saved.statusCode(), saved.body()); + int savedId = new JSONObject(saved.body()).getInt("id"); + var checked = request("POST", path + "/builds", new JSONObject().put("revisionId", savedId).put("entryPoint", ENTRY).put("capabilities", new org.json.JSONArray()).toString(), "alice"); + assertEquals(200, checked.statusCode(), checked.body()); + int built = new JSONObject(checked.body()).getInt("buildId"); + String buildPath = path + "/builds/" + built; + assertEquals(400, request("POST", buildPath + "/publications", "{\"note\":\"Premature\"}", "alice").statusCode()); + assertEquals(403, request("POST", buildPath + "/test-context", "{\"input\":{},\"capabilities\":[]}", "reader").statusCode()); + var tested = request("POST", buildPath + "/test-context", "{\"input\":{},\"capabilities\":[]}", "alice"); + assertEquals(200, tested.statusCode(), tested.body()); + assertEquals("OK", new JSONObject(tested.body()).getString("status")); + var published = request("POST", buildPath + "/publications", "{\"note\":\"Reviewed\"}", "alice"); + assertEquals(200, published.statusCode(), published.body()); + assertEquals(savedId, rows("script").stream().filter(row -> row.getValueInteger("id").equals(id)).findFirst().orElseThrow().getValueInteger("currentScriptRevisionId")); + assertEquals(403, request("GET", "/qbit/scripting/projects/" + project + "/builds/" + built, null, "alice").statusCode()); + assertEquals(400, request("POST", "/qbit/scripting/projects", "{\"name\":\"Bad\",\"executionMode\":\"LEGACY\"}", "alice").statusCode()); + assertEquals(403, request("POST", "/qbit/scripting/projects", "{\"name\":\"Denied\"}", "reader").statusCode()); + } + + /*************************************************************************** + ** Hook tests read a selected record; binding changes use authenticated services. + ***************************************************************************/ + @Test + void nativeHookTestAndBindingControlsAreScoped() throws Exception + { + Integer publication = publishNew(project, "hook-http"); + Integer built = rows(ScriptingTables.PUBLICATION).getLast().getValueInteger("build_id"); + var row = new com.kingsrook.qqq.backend.core.actions.tables.InsertAction().execute( + new com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput(HOOK_TABLE).withRecords(java.util.List.of( + new com.kingsrook.qqq.backend.core.model.data.QRecord().withValue("status", "OLD")))).getRecords().getFirst(); + String path = "/qbit/scripting/projects/" + project; + var catalog = request("GET", path + "/hooks", null, "alice"); + assertEquals(200, catalog.statusCode(), catalog.body()); + assertTrue(catalog.body().contains("PRE_INSERT")); + worker.hookJson = "{\"action\":\"PATCH\",\"patch\":{\"status\":\"RESULT\"}}"; + String test = new JSONObject().put("tableName", HOOK_TABLE).put("event", "PRE_INSERT").put("recordId", row.getValueString("id")) + .put("changes", new JSONObject().put("status", "PROPOSED")).toString(); + var tested = request("POST", path + "/builds/" + built + "/test-hook", test, "alice"); + assertEquals(200, tested.statusCode(), tested.body()); + assertEquals("PATCH", new JSONObject(tested.body()).getJSONObject("hook").getString("action")); + assertEquals("OLD", rows(HOOK_TABLE).getFirst().getValueString("status")); + assertEquals(403, request("POST", path + "/builds/" + built + "/test-hook", test, "reader").statusCode()); + assertEquals(400, request("POST", path + "/builds/" + built + "/test-hook", new JSONObject(test).put("prior", new JSONObject()).toString(), "alice").statusCode()); + worker.failureCode = "PRIVATE-CODE"; + worker.failureMessage = "PRIVATE-RECORD-VALUE"; + var failed = request("POST", path + "/builds/" + built + "/test-hook", test, "alice"); + assertEquals(200, failed.statusCode(), failed.body()); + assertEquals("SCRIPT_ERROR", new JSONObject(failed.body()).getString("status")); + assertFalse(failed.body().contains("PRIVATE-CODE")); + assertFalse(failed.body().contains("PRIVATE-RECORD-VALUE")); + assertFalse(rows(ScriptingTables.EXECUTION).toString().contains("PRIVATE")); + worker.failureCode = null; + String binding = new JSONObject().put("name", "HTTP hook").put("tableName", HOOK_TABLE).put("event", "PRE_INSERT").put("publicationId", publication) + .put("capabilities", new org.json.JSONArray(com.kingsrook.qbits.scripting.capability.CapabilityDeclaration.listToJson(declared()))).put("patchableFields", new org.json.JSONArray().put("status")).toString(); + var created = request("POST", path + "/bindings", binding, "alice"); + assertEquals(200, created.statusCode(), created.body()); + int bindingId = new JSONObject(created.body()).getInt("id"); + assertEquals("BINDING_CHANGE", rows(ScriptingTables.EXECUTION).getLast().getValueString("kind")); + var listed = new JSONObject(request("GET", path + "/hooks", null, "alice").body()).getJSONArray("bindings").getJSONObject(0); + assertEquals("READ", listed.getJSONArray("grants").getJSONObject(0).getString("operation")); + assertEquals(HOOK_TABLE, listed.getJSONArray("grants").getJSONObject(0).getString("table")); + String state = path + "/bindings/" + bindingId + "/state"; + assertEquals(403, request("POST", state, "{\"enabled\":false}", "reader").statusCode()); + assertEquals(400, request("POST", state, "{\"enabled\":\"false\"}", "alice").statusCode()); + assertEquals(200, request("POST", state, "{\"enabled\":false}", "alice").statusCode()); + assertTrue(service.findBinding(HOOK_TABLE, com.kingsrook.qbits.scripting.hooks.HookEvent.PRE_INSERT).isEmpty()); + Integer other = newProject(); + assertEquals(403, request("POST", "/qbit/scripting/projects/" + other + "/bindings/" + bindingId + "/state", "{\"enabled\":true}", "alice").statusCode()); + assertEquals(200, request("POST", state, "{\"enabled\":true}", "alice").statusCode()); + } + + /*************************************************************************** + ** Real JSON request with a stable fixture session cookie. + ***************************************************************************/ + private HttpResponse request(String method, String path, String body, String caller) throws Exception + { + return (client.send(HttpRequest.newBuilder(URI.create(base + path)).timeout(Duration.ofSeconds(15)) + .header("Cookie", "sessionUUID=" + caller + "-" + suffix).header("Content-Type", "application/json") + .method(method, body == null ? HttpRequest.BodyPublishers.noBody() : HttpRequest.BodyPublishers.ofString(body)).build(), + HttpResponse.BodyHandlers.ofString())); + } + + /*************************************************************************** + ** Only an identified fixture author has lifecycle permissions. + ***************************************************************************/ + public static class Sessions implements QAuthenticationModuleCustomizerInterface + { + /*********************************************************************** + ** Establish server-owned identity from a test-only mock authenticator. + ***********************************************************************/ + @Override + public void customizeSession(QInstance instance, QSession session, Map context) + { + boolean author = session.getUuid().startsWith("alice-"); + session.withUser(new QUser().withIdReference(author ? "alice@example.com" : "reader")); + if(author) + { + session.withPermissions(ScriptingPermissions.VIEW, ScriptingPermissions.DRAFT, ScriptingPermissions.CHECK, + ScriptingPermissions.TEST, ScriptingPermissions.PUBLISH, ScriptingPermissions.BIND); + } + } + } +} diff --git a/middleware/src/test/java/com/kingsrook/qbits/scripting/api/ScriptingApiContractTest.java b/middleware/src/test/java/com/kingsrook/qbits/scripting/api/ScriptingApiContractTest.java new file mode 100644 index 0000000..0110785 --- /dev/null +++ b/middleware/src/test/java/com/kingsrook/qbits/scripting/api/ScriptingApiContractTest.java @@ -0,0 +1,61 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.scripting.api; + +import java.nio.file.Files; +import java.nio.file.Path; +import com.kingsrook.qqq.middleware.javalin.specs.v1.MiddlewareVersionV1; +import com.kingsrook.qqq.middleware.javalin.tools.UiApiContract; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** Registered semantic specs are the source of truth for the checked-in schema. + *******************************************************************************/ +class ScriptingApiContractTest +{ + /*************************************************************************** + ** Explicit regeneration flag writes the same deterministic OpenAPI fragment. + ***************************************************************************/ + @Test + void generatedContractMatches() throws Exception + { + var version = new MiddlewareVersionV1(); + var extension = ScriptingApi.extension(); + String generated = UiApiContract.generateExtensionYaml(version, extension); + Path path = Path.of("src/main/resources/openapi/qqq-v1/scripting.yaml"); + if(Boolean.getBoolean("scripting.openapi.update")) + { + Files.createDirectories(path.getParent()); + Files.writeString(path, generated); + } + assertTrue(UiApiContract.check(generated, Files.readString(path), null, 1, "mvnw -f middleware/pom.xml -Dscripting.openapi.update=true test").isEmpty()); + assertEquals(14, extension.getSpecs().size()); + assertTrue(generated.contains("test-hook")); + assertTrue(generated.contains("ScriptingBindingV1")); + assertTrue(generated.contains("ScriptingProjectV1")); + assertTrue(generated.contains("/qbit/scripting/projects/")); + assertTrue(generated.contains("test-context")); + } +} diff --git a/mvnw b/mvnw new file mode 100644 index 0000000..bd8896b --- /dev/null +++ b/mvnw @@ -0,0 +1,295 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.4 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +scriptDir="$(dirname "$0")" +scriptName="$(basename "$0")" + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +actualDistributionDir="" + +# First try the expected directory name (for regular distributions) +if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then + if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then + actualDistributionDir="$distributionUrlNameMain" + fi +fi + +# If not found, search for any directory with the Maven executable (for snapshots) +if [ -z "$actualDistributionDir" ]; then + # enable globbing to iterate over items + set +f + for dir in "$TMP_DOWNLOAD_DIR"/*; do + if [ -d "$dir" ]; then + if [ -f "$dir/bin/$MVN_CMD" ]; then + actualDistributionDir="$(basename "$dir")" + break + fi + fi + done + set -f +fi + +if [ -z "$actualDistributionDir" ]; then + verbose "Contents of $TMP_DOWNLOAD_DIR:" + verbose "$(ls -la "$TMP_DOWNLOAD_DIR")" + die "Could not find Maven distribution directory in extracted archive" +fi + +verbose "Found extracted Maven distribution directory: $actualDistributionDir" +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 0000000..92450f9 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,189 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.4 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' + +$MAVEN_M2_PATH = "$HOME/.m2" +if ($env:MAVEN_USER_HOME) { + $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME" +} + +if (-not (Test-Path -Path $MAVEN_M2_PATH)) { + New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null +} + +$MAVEN_WRAPPER_DISTS = $null +if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) { + $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists" +} else { + $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists" +} + +$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain" +$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +$actualDistributionDir = "" + +# First try the expected directory name (for regular distributions) +$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain" +$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD" +if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) { + $actualDistributionDir = $distributionUrlNameMain +} + +# If not found, search for any directory with the Maven executable (for snapshots) +if (!$actualDistributionDir) { + Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object { + $testPath = Join-Path $_.FullName "bin/$MVN_CMD" + if (Test-Path -Path $testPath -PathType Leaf) { + $actualDistributionDir = $_.Name + } + } +} + +if (!$actualDistributionDir) { + Write-Error "Could not find Maven distribution directory in extracted archive" +} + +Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir" +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..8474695 --- /dev/null +++ b/pom.xml @@ -0,0 +1,21 @@ + + + 4.0.0 + + com.kingsrook.qbits + qbit-scripting-reactor + ${revision} + pom + QBit Scripting (reactor) + Apache License, Version 2.0https://www.apache.org/licenses/LICENSE-2.0repo + 0.1.0-SNAPSHOTUTF-8 + + + base + rdbms + middleware + github + + diff --git a/rdbms/LICENSE b/rdbms/LICENSE new file mode 100644 index 0000000..4bb040e --- /dev/null +++ b/rdbms/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, source + code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 Kingsrook, LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/rdbms/NOTICE b/rdbms/NOTICE new file mode 100644 index 0000000..6fbe46b --- /dev/null +++ b/rdbms/NOTICE @@ -0,0 +1,6 @@ +QBit Scripting +Copyright 2026 Kingsrook, LLC + +This product includes software developed at Kingsrook, LLC (https://kingsrook.com/). + +Licensed under the Apache License, Version 2.0. diff --git a/rdbms/README.md b/rdbms/README.md new file mode 100644 index 0000000..271c086 --- /dev/null +++ b/rdbms/README.md @@ -0,0 +1,61 @@ +# Scripting persistence adapter + +The base Scripting QBit depends only on QQQ core. Persistence is disabled unless a trusted host supplies `ScriptingTransactionPolicy`. This optional module supplies `RdbmsScriptingTransactionPolicy`; it inspects the exact opened transaction before authoring changes or independently committed execution history can proceed. + +```java +runtime.withTransactionPolicy(new RdbmsScriptingTransactionPolicy()); +``` + +The base service preserves the authenticated caller's native credential and client-channel limits before switching to its least-authority storage session. Named scripting grants still authorize each lifecycle action; the adapter does not grant permissions or introduce a second source registry. + +## Qualified connection requirements + +Every transaction rechecks the actual connection and all nine core/source and scripting-owned tables. There is no cached database qualification, reflective connection probing, mutation of pooled session settings, or fallback to a no-op transaction. + +- MySQL: native `mysql`, actual MySQL 8.4, configured single target/catalog, non-autocommit writable connection, all tables InnoDB with `utf8mb4_0900_bin` text comparisons, exact source/owner unique keys, `innodb_flush_log_at_trx_commit=1`, and `sync_binlog=1` when binary logging is enabled. The server must use recovery-capable `innodb_doublewrite` (`ON` or `DETECT_AND_RECOVER`), and the session must not be read-only. The actual session must be UTC and have a lock timeout of 1–5 seconds. The JDBC driver's ordinary timestamp conversion must preserve the server's exact instant. +- File H2: native `h2`, exact `jdbc:h2:file:` target, real writable transaction, live `WRITE_DELAY=0`, and the required identity indexes. In-memory H2 is not a production qualification. +- Aurora MySQL, MariaDB, PostgreSQL and other engines/versions are refused. No provider equivalence is inferred. + +Configure MySQL driver time handling using `connectionTimeZone=UTC&forceConnectionTimeZoneToSession=true`. Configure the bounded session timeout through the host pool/JDBC session variables or a dedicated database default. Keep host credentials and TLS configuration outside this module; the production adapter never disables TLS. Test-only loopback fixtures use disposable credentials and explicitly disabled TLS. + +Required core indexes are `(scriptId, sequenceNo)` on the actual core revision table and `(scriptRevisionId, fileName)` on the actual core revision-file table. Hosts apply them using their native physical column mappings. The adapter checks them even if metadata omitted the uniqueness declaration. Required owner indexes match the shipped `db/mysql/001_scripting.sql`: artifact build/version, publication version, binding active slot and import revision. It never modifies core schemas itself. + +The actual mapped `scripting_execution.operation_id` column must be VARCHAR with capacity at least64 and NOT NULL. This is checked on the same transaction connection before audited work. Fresh001 provides64; existing original001 deployments apply inspected003 to widen36 and enable AFTER phases. No runtime DDL or truncation fallback is used. + +## Migration and deployment authority + +Apply the shipped MySQL migration with a separate migration identity. The application identity receives DML privileges only, without ALTER, DROP or TRIGGER. The migration's append-only history and binding-identity triggers must be checked by that deployment identity, and application DML denial must be tested before release. + +Runtime cannot inspect trigger definitions under a least-privilege DML account: [MySQL requires the TRIGGER privilege to expose that metadata](https://dev.mysql.com/doc/refman/8.4/en/information-schema-triggers-table.html). Granting it merely for inspection would let the application remove the same protections. The adapter therefore checks runtime connection/engine/settings/identity constraints, while the exact trigger migration and DML-denial proof remain deployment requirements. Privileged DDL changes are trusted host actions and require requalification. + +Synchronous settings follow [MySQL's documented commit durability requirements](https://dev.mysql.com/doc/refman/8.4/en/innodb-parameters.html#sysvar_innodb_flush_log_at_trx_commit). Disk flush behavior, backups, restore, replication/failover and host/container death are separate deployment proofs. This adapter is not a claim that every configured host is production-ready. + +## Tests and current evidence + +The base's explicit memory/in-memory-JDBC test policy is intentionally volatile and belongs only to its test classifier. It is not shipped in the production artifact and does not attest durable storage. The real provider subclasses here replace it with the production adapter. + +Build the base main/test artifacts first, then run the module through the repository wrapper: + +```powershell +.\mvnw.cmd verify install +.\mvnw.cmd -f rdbms/pom.xml verify install +``` + +The MySQL suite creates a disposable loopback-only container from `mysql@sha256:a0225d1ea6631c8bc2385dc4c2faba17bbf1834ecf369e1659a9a382bd8d5dd7`, with random synthetic migration/application credentials, per-test schemas and a DML-only application identity. Docker wire logs are suppressed. A restart refreshes the actual dynamic port before reconnecting. The H2 suite uses temporary file databases and closes their native providers before cleanup. + +The inherited suites cover native transaction rollback, separate committed history, publication/binding concurrency and native record-hook behavior. New cases cover unsafe core engines and commit settings, missing source uniqueness/case folding, concurrent draft saves, committed publication after an actual server restart, file-H2 delayed-flush refusal and no-op transaction refusal. + +Current evidence: base full 124 tests and quality gates passed after caller-ceiling/default-deny changes. The optional adapter passed all 29 real-provider tests (17 MySQL and 12 file H2), with zero failures/errors, and was packaged/installed after that unchanged verified snapshot. This includes actual database restart, rollback/concurrency, DML-only migration guards, live durability refusal, doublewrite/read-only rejection and source-key qualification. The first H2 unsafe-setting test exposed a fixture URL that reset WRITE_DELAY; it now changes the actual native connection URL and proves refusal. Neither the protocol-level fake worker nor these database tests qualify hostile script execution. + +## AFTER and audit-schema compatibility — 2026-10-31 + +Current corrected evidence supersedes the historical29-test snapshot above: base145 +passed all quality gates; this optional module passed44 tests (13 file-H2,14 actual +AFTER/MySQL,17 baseline MySQL), zero failures/errors/skips, with Checkstyle. See +`after-hooks-operation64-provider-verification.log` and the base +[AFTER report](../docs/DURABLE-AFTER-HOOKS.md) for exact hashes, schema upgrade +proof and preserved earlier failures. Existing36 and nullable-column refusals are +covered; historical audit rows and immutable triggers survive actual003 migration. +The optional POM's BOM imports now use dependencyManagement and its duplicate +RDBMS dependency is removed; retained dependency versions/scopes are unchanged. +No shared artifact install or host deployment follows from this verification. diff --git a/rdbms/pom.xml b/rdbms/pom.xml new file mode 100644 index 0000000..a897f9b --- /dev/null +++ b/rdbms/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + com.kingsrookqbit-build-parent2.0.0 + com.kingsrook.qbitsqbit-scripting-rdbms${revision} + 0.1.0-SNAPSHOT21none4.1.0-RC.214.1.0-RC.214.1.0-RC.21 + + com.kingsrook.qqqqqq-backend-core${qqq.core.version} + + com.kingsrook.qqqqqq-bom-pom${qqq.version}pomimport + org.junitjunit-bom6.0.1pomimport + + + com.kingsrook.qbitsqbit-scripting${revision} + org.junit.jupiterjunit-jupiter6.0.1test + com.kingsrook.qqqqqq-backend-module-rdbms${qqq.core.version} + com.kingsrook.qbitsqbit-scripting${revision}test-jarteststest + com.h2databaseh22.5.252test + com.mysqlmysql-connector-j9.5.0test + org.testcontainerstestcontainers2.0.3test + org.postgresqlpostgresql42.7.8test + com.kingsrook.qqqqqq-backend-module-postgres${qqq.postgres.version}test + com.kingsrook.qqqqqq-backend-module-rdbmscom.kingsrook.qqqqqq-backend-core + + + + src/main/resourcesfalse + + + + + org.apache.maven.pluginsmaven-resources-plugin3.3.1 + package-license-noticesprocess-resourcescopy-resources + ${project.build.outputDirectory}/META-INF + ${project.basedir}falseLICENSENOTICE + + + + org.apache.maven.pluginsmaven-compiler-plugin3.14.1 + org.apache.maven.pluginsmaven-surefire-plugin3.5.4 + org.apache.maven.pluginsmaven-checkstyle-plugin3.3.1 + com.puppycrawl.toolscheckstyle10.12.6 + ${project.basedir}/../checkstyle/config.xmlcheckstyle.reactor.root=${maven.multiModuleProjectDirectory}truetruewarningtrue + validatecheck + + + + org.codehaus.mojoflatten-maven-plugin1.7.3 + trueossrh + + flattenprocess-resourcesflatten + flatten.cleancleanclean + + + + + diff --git a/rdbms/src/main/java/com/kingsrook/qbits/scripting/rdbms/RdbmsScriptingTransactionPolicy.java b/rdbms/src/main/java/com/kingsrook/qbits/scripting/rdbms/RdbmsScriptingTransactionPolicy.java new file mode 100644 index 0000000..bfc502d --- /dev/null +++ b/rdbms/src/main/java/com/kingsrook/qbits/scripting/rdbms/RdbmsScriptingTransactionPolicy.java @@ -0,0 +1,445 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting.rdbms; + + +import java.net.URI; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.scripting.ScriptingException; +import com.kingsrook.qbits.scripting.ScriptingTables; +import com.kingsrook.qbits.scripting.service.ScriptingTransactionPolicy; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; + + +/******************************************************************************* + ** Optional native RDBMS qualification for scripting's actual owned transaction. + ** Only file H2 with WRITE_DELAY=0, MySQL 8.4/InnoDB and the native postgres + ** module on PostgreSQL 17 (permanent tables, synchronous commit) are supported. The base + ** QBit owns the policy contract and has no JDBC module or driver dependency. + ** This adapter never mutates session/global settings, schema or caller data. + *******************************************************************************/ +public final class RdbmsScriptingTransactionPolicy implements ScriptingTransactionPolicy +{ + private static final List TABLES = List.of("script", "scriptRevision", "scriptRevisionFile", ScriptingTables.BUILD, + ScriptingTables.ARTIFACT, ScriptingTables.PUBLICATION, ScriptingTables.BINDING, ScriptingTables.EXECUTION, ScriptingTables.IMPORT); + private static final Map>> REQUIRED_KEYS = Map.of( + "scriptRevision", List.of(List.of("scriptId", "sequenceNo")), + "scriptRevisionFile", List.of(List.of("scriptRevisionId", "fileName")), + ScriptingTables.ARTIFACT, List.of(List.of("build_id"), List.of("script_id", "version")), + ScriptingTables.PUBLICATION, List.of(List.of("script_id", "version")), + ScriptingTables.BINDING, List.of(List.of("active_slot")), + ScriptingTables.IMPORT, List.of(List.of("script_revision_id"))); + + + /*************************************************************************** + ** Verify all registry/owner tables, not just the first table of this action. + ** Per-call evidence is never cached across connection or schema changes. + ***************************************************************************/ + @Override + public void validate(QBackendTransaction transaction, List tables) throws ScriptingException + { + try + { + if(transaction == null || transaction.getClass() != RDBMSTransaction.class || tables == null || tables.isEmpty() || !TABLES.containsAll(tables)) + { + throw denied(); + } + var instance = QContext.getQInstance(); + if(!(instance.getBackendForTable(tables.getFirst()) instanceof RDBMSBackendMetaData backend)) + { + throw denied(); + } + Connection connection = ((RDBMSTransaction) transaction).getConnection(); + if(connection.isClosed() || connection.isReadOnly() || connection.getAutoCommit()) + { + throw denied(); + } + boolean postgres = "postgres".equals(backend.getVendor()) && "postgres".equals(backend.getBackendType()); + boolean mysql = !postgres && "mysql".equals(backend.getVendor()); + if(postgres) + { + requirePostgresConnection(connection, backend); + } + else if(mysql) + { + requireMySqlConnection(connection, backend); + } + else + { + requireH2Connection(connection, backend); + } + for(String name : TABLES) + { + QTableMetaData table = instance.getTable(name); + if(table == null || !backend.getName().equals(table.getBackendName())) + { + throw denied(); + } + String physical = physicalTable(table); + if(mysql) + { + requireMySqlTable(connection, physical); + } + if(postgres) + { + requirePostgresTable(connection, physical); + } + requireUniqueKeys(connection, table, physical, mysql, postgres); + if(ScriptingTables.EXECUTION.equals(name)) + { + requireAuditCorrelation(connection, table, physical, postgres ? "\"" : "`"); + } + } + } + catch(ScriptingException failure) + { + throw failure; + } + catch(Exception failure) + { + // Qualification errors never expose URLs, credentials, or schema contents. + throw denied(); + } + } + + + /*************************************************************************** + ** Actual PostgreSQL 17 target (native postgres module), UTC date round trip and + ** bounded durable settings, observed on this transaction's own connection. + ***************************************************************************/ + private void requirePostgresConnection(Connection connection, RDBMSBackendMetaData backend) throws Exception + { + String expected = ConnectionManager.getJdbcUrl(backend).split("\\?", 2)[0]; + String actual = connection.getMetaData().getURL().split("\\?", 2)[0]; + if(!expected.startsWith("jdbc:postgresql://") || !expected.equals(actual) + || !"PostgreSQL".equals(connection.getMetaData().getDatabaseProductName()) || connection.getMetaData().getDatabaseMajorVersion() != 17) + { + throw denied(); + } + URI location = URI.create(expected.substring(5)); + String catalog = location.getPath().substring(1); + if(location.getHost() == null || location.getUserInfo() != null || !catalog.matches("[A-Za-z0-9_]+") || !catalog.equals(connection.getCatalog())) + { + throw denied(); + } + try(Statement statement = connection.createStatement()) + { + statement.setQueryTimeout(10); + try(ResultSet settings = statement.executeQuery("SELECT current_setting('server_version_num')::int, current_setting('transaction_read_only'), pg_is_in_recovery()," + + " current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes'), current_setting('TimeZone')," + + " (SELECT setting::bigint FROM pg_settings WHERE name='lock_timeout'), current_setting('transaction_isolation')," + + " current_database(), now(), extract(epoch from now())")) + { + if(!settings.next() || settings.getInt(1) / 10000 != 17 || !"off".equals(settings.getString(2)) || settings.getBoolean(3) + || !List.of("on", "remote_apply").contains(settings.getString(4)) || !"on".equals(settings.getString(5)) || !"on".equals(settings.getString(6)) + || !List.of("UTC", "Etc/UTC").contains(settings.getString(7)) || settings.getLong(8) < 1 || settings.getLong(8) > 5000 + || !"read committed".equals(settings.getString(9)) || !catalog.equals(settings.getString(10))) + { + throw denied(); + } + var epoch = settings.getBigDecimal(12); + var instant = java.time.Instant.ofEpochSecond(epoch.longValue(), epoch.remainder(java.math.BigDecimal.ONE).movePointRight(9).longValue()); + if(!settings.getTimestamp(11).toInstant().equals(instant)) + { + throw denied(); + } + } + } + } + + + /*************************************************************************** + ** Permanent (logged) ordinary tables only, with deterministic text comparison. + ***************************************************************************/ + private void requirePostgresTable(Connection connection, String table) throws Exception + { + try(PreparedStatement statement = connection.prepareStatement("SELECT c.relkind, c.relpersistence FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=current_schema() AND c.relname=?")) + { + statement.setQueryTimeout(10); + statement.setString(1, table); + try(ResultSet rows = statement.executeQuery()) + { + if(!rows.next() || !"r".equals(rows.getString(1)) || !"p".equals(rows.getString(2)) || rows.next()) + { + throw denied(); + } + } + } + try(PreparedStatement statement = connection.prepareStatement("SELECT COUNT(*) FROM pg_attribute a JOIN pg_class c ON c.oid=a.attrelid JOIN pg_namespace n ON n.oid=c.relnamespace" + + " JOIN pg_collation co ON co.oid=a.attcollation WHERE n.nspname=current_schema() AND c.relname=? AND a.attnum>0 AND NOT a.attisdropped AND NOT co.collisdeterministic")) + { + statement.setQueryTimeout(10); + statement.setString(1, table); + try(ResultSet rows = statement.executeQuery()) + { + if(!rows.next() || rows.getInt(1) != 0) + { + throw denied(); + } + } + } + } + + + /*************************************************************************** + ** Actual MySQL target, native date conversion and bounded durable settings. + ***************************************************************************/ + private void requireMySqlConnection(Connection connection, RDBMSBackendMetaData backend) throws Exception + { + String expected = ConnectionManager.getJdbcUrl(backend).split("\\?", 2)[0]; + String actual = connection.getMetaData().getURL().split("\\?", 2)[0]; + if(!expected.startsWith("jdbc:mysql://") || !expected.equals(actual) + || !"MySQL".equals(connection.getMetaData().getDatabaseProductName()) || !connection.getMetaData().getDatabaseProductVersion().startsWith("8.4.")) + { + throw denied(); + } + URI location = URI.create(expected.substring(5)); + String catalog = location.getPath().substring(1); + if(location.getHost() == null || location.getUserInfo() != null || !catalog.matches("[A-Za-z0-9_]+") || !catalog.equals(connection.getCatalog())) + { + throw denied(); + } + try(Statement statement = connection.createStatement()) + { + statement.setQueryTimeout(10); + try(ResultSet settings = statement.executeQuery("SELECT @@GLOBAL.innodb_flush_log_at_trx_commit, @@GLOBAL.log_bin, @@GLOBAL.sync_binlog, @@GLOBAL.read_only, @@SESSION.innodb_lock_wait_timeout, @@SESSION.time_zone, @@GLOBAL.system_time_zone, CURRENT_TIMESTAMP(6), UNIX_TIMESTAMP(CURRENT_TIMESTAMP(6)), @@GLOBAL.innodb_doublewrite, @@SESSION.transaction_read_only")) + { + if(!settings.next() || !List.of("ON", "DETECT_AND_RECOVER").contains(settings.getString(10)) || settings.getBoolean(11) || settings.getInt(1) != 1 || (settings.getBoolean(2) && settings.getInt(3) != 1) || settings.getBoolean(4) + || settings.getInt(5) < 1 || settings.getInt(5) > 5 || !(List.of("UTC", "+00:00").contains(settings.getString(6)) + || ("SYSTEM".equals(settings.getString(6)) && "UTC".equals(settings.getString(7))))) + { + throw denied(); + } + var epoch = settings.getBigDecimal(9); + var instant = java.time.Instant.ofEpochSecond(epoch.longValue(), epoch.remainder(java.math.BigDecimal.ONE).movePointRight(9).longValueExact()); + if(!settings.getTimestamp(8).toInstant().equals(instant)) + { + throw denied(); + } + } + } + } + + + /*************************************************************************** + ** File-backed H2 only; in-memory fixtures require an explicit test policy. + ***************************************************************************/ + private void requireH2Connection(Connection connection, RDBMSBackendMetaData backend) throws Exception + { + String expected = ConnectionManager.getJdbcUrl(backend).split(";", 2)[0]; + if(!"h2".equals(backend.getVendor()) || !expected.startsWith("jdbc:h2:file:") + || !expected.equals(connection.getMetaData().getURL().split(";", 2)[0]) || !"H2".equals(connection.getMetaData().getDatabaseProductName())) + { + throw denied(); + } + boolean durable = false; + try(Statement statement = connection.createStatement()) + { + statement.setQueryTimeout(10); + try(ResultSet settings = statement.executeQuery("SELECT SETTING_NAME, SETTING_VALUE FROM INFORMATION_SCHEMA.SETTINGS WHERE SETTING_NAME='WRITE_DELAY'")) + { + durable = settings.next() && "0".equals(settings.getString(2)); + } + } + if(!durable) + { + throw denied(); + } + } + + + /*************************************************************************** + ** Allow native table mapping in this catalog, never a cross-schema SQL name. + ***************************************************************************/ + private String physicalTable(QTableMetaData table) throws ScriptingException + { + String physical = table.getName(); + if(table.getBackendDetails() != null) + { + if(!(table.getBackendDetails() instanceof RDBMSTableBackendDetails details)) + { + throw denied(); + } + if(details.getTableName() != null) + { + physical = details.getTableName(); + } + } + if(!physical.matches("[A-Za-z_][A-Za-z0-9_]*")) + { + throw denied(); + } + return (physical); + } + + /*************************************************************************** + ** Native event UUID plus ordinal must fit the actual non-null mapped column. + ** Qualification precedes any authoring or independent execution-audit write. + ***************************************************************************/ + private void requireAuditCorrelation(Connection connection, QTableMetaData table, String physical, String quote) throws Exception + { + var field = table.getField("operation_id"); + if(field == null) + { + throw denied(); + } + String column = field.getBackendName() == null ? field.getName() : field.getBackendName(); + if(!column.matches("[A-Za-z_][A-Za-z0-9_]*")) + { + throw denied(); + } + try(PreparedStatement statement = connection.prepareStatement("SELECT " + quote + column + quote + " FROM " + quote + physical + quote + " WHERE 1=0")) + { + statement.setQueryTimeout(10); + try(ResultSet rows = statement.executeQuery()) + { + ResultSetMetaData metadata = rows.getMetaData(); + if(metadata.getColumnCount() != 1 || metadata.getColumnType(1) != Types.VARCHAR || metadata.getPrecision(1) < 64 + || metadata.isNullable(1) != ResultSetMetaData.columnNoNulls) + { + throw denied(); + } + } + } + } + + + /*************************************************************************** + ** Actual storage engine and exact binary identity/source comparisons. + ***************************************************************************/ + private void requireMySqlTable(Connection connection, String table) throws Exception + { + try(PreparedStatement statement = connection.prepareStatement("SELECT TABLE_NAME, ENGINE, TABLE_COLLATION FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA=? AND TABLE_NAME=?")) + { + statement.setQueryTimeout(10); + statement.setString(1, connection.getCatalog()); + statement.setString(2, table); + try(ResultSet rows = statement.executeQuery()) + { + if(!rows.next() || !table.equals(rows.getString(1)) || !"InnoDB".equals(rows.getString(2)) || !"utf8mb4_0900_bin".equals(rows.getString(3)) || rows.next()) + { + throw denied(); + } + } + } + try(PreparedStatement statement = connection.prepareStatement("SELECT COLLATION_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=? AND TABLE_NAME=? AND COLLATION_NAME IS NOT NULL")) + { + statement.setQueryTimeout(10); + statement.setString(1, connection.getCatalog()); + statement.setString(2, table); + try(ResultSet rows = statement.executeQuery()) + { + while(rows.next()) + { + if(!"utf8mb4_0900_bin".equals(rows.getString(1))) + { + throw denied(); + } + } + } + } + } + + + /*************************************************************************** + ** Require exact native identity keys even if host metadata omitted them. + ***************************************************************************/ + private void requireUniqueKeys(Connection connection, QTableMetaData table, String physical, boolean mysql, boolean postgres) throws Exception + { + Map> actual = new HashMap<>(); + if(postgres) + { + // Only complete, valid, non-partial, non-expression unique indexes count as identity keys. + try(PreparedStatement statement = connection.prepareStatement("SELECT i.relname, a.attname FROM pg_index ix JOIN pg_class i ON i.oid=ix.indexrelid" + + " JOIN pg_class t ON t.oid=ix.indrelid JOIN pg_namespace n ON n.oid=t.relnamespace" + + " CROSS JOIN LATERAL unnest(ix.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord)" + + " JOIN pg_attribute a ON a.attrelid=t.oid AND a.attnum=k.attnum" + + " WHERE n.nspname=current_schema() AND t.relname=? AND ix.indisunique AND ix.indisvalid AND ix.indpred IS NULL AND ix.indexprs IS NULL AND k.ord<=ix.indnkeyatts" + + " ORDER BY i.relname, k.ord")) + { + statement.setQueryTimeout(10); + statement.setString(1, physical); + try(ResultSet indexes = statement.executeQuery()) + { + while(indexes.next()) + { + actual.computeIfAbsent(indexes.getString(1), key -> new ArrayList<>()).add(indexes.getString(2)); + } + } + } + } + else + { + try(ResultSet indexes = connection.getMetaData().getIndexInfo(connection.getCatalog(), mysql ? null : connection.getSchema(), physical, true, false)) + { + while(indexes.next()) + { + String name = indexes.getString("INDEX_NAME"); + String column = indexes.getString("COLUMN_NAME"); + if(name != null && column != null) + { + actual.computeIfAbsent(name, key -> new ArrayList<>()).add(column); + } + } + } + } + for(List required : REQUIRED_KEYS.getOrDefault(table.getName(), List.of())) + { + List columns = new ArrayList<>(); + for(String field : required) + { + if(!table.getFields().containsKey(field)) + { + throw denied(); + } + columns.add(table.getField(field).getBackendName() == null ? field : table.getField(field).getBackendName()); + } + if(!actual.containsValue(columns)) + { + throw denied(); + } + } + } + + + /*************************************************************************** + ** Stable external failure without database details or private credentials. + ***************************************************************************/ + private ScriptingException denied() + { + return (new ScriptingException(ScriptingException.Kind.UNAVAILABLE, "Scripting persistence transaction or deployment schema is not qualified")); + } +} diff --git a/rdbms/src/test/java/com/kingsrook/qbits/scripting/FileH2ScriptingTest.java b/rdbms/src/test/java/com/kingsrook/qbits/scripting/FileH2ScriptingTest.java new file mode 100644 index 0000000..4041c1e --- /dev/null +++ b/rdbms/src/test/java/com/kingsrook/qbits/scripting/FileH2ScriptingTest.java @@ -0,0 +1,119 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.nio.file.Path; +import com.kingsrook.qbits.scripting.rdbms.RdbmsScriptingTransactionPolicy; +import com.kingsrook.qqq.backend.core.actions.QBackendTransaction; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The native transaction regression suite through the optional file-H2 adapter. + ** Volatile H2 remains a separate base fixture and never passes this policy. + *******************************************************************************/ +class FileH2ScriptingTest extends JdbcScriptingTest +{ + @TempDir + Path directory; + + + /*************************************************************************** + ** Create actual file-backed tables and required core identity constraints. + ***************************************************************************/ + @Override + @BeforeEach + void useJdbc() throws Exception + { + ConnectionManager.closeConnectionProviders(); + createJdbc("jdbc:h2:file:" + directory.resolve("scripting").toAbsolutePath().toString().replace('\\', '/') + + ";DATABASE_TO_UPPER=FALSE;WRITE_DELAY=0;LOCK_TIMEOUT=5000"); + sql("CREATE UNIQUE INDEX source_revision_identity ON scriptRevision(scriptId, sequenceNo)"); + sql("CREATE UNIQUE INDEX source_file_identity ON scriptRevisionFile(scriptRevisionId, fileName)"); + sql("ALTER TABLE scripting_execution ALTER COLUMN operation_id VARCHAR(64) NOT NULL"); + runtime.withTransactionPolicy(new RdbmsScriptingTransactionPolicy()); + } + + /*************************************************************************** + ** Undersized or nullable audit identifiers refuse before worker execution. + ***************************************************************************/ + @Test + void correlationColumnMustBeWideEnoughAndNonNull() throws Exception + { + var built = build(draft(newProject(), "qualified-audit-width")); + var event = com.kingsrook.qbits.scripting.hooks.HookEvent.PRE_INSERT; + assertTrue(service.test(everything(), built.buildId(), event, "{}", null).succeeded()); + int before = worker.executeCalls; + int history = count(ScriptingTables.EXECUTION); + sql("ALTER TABLE scripting_execution ALTER COLUMN operation_id VARCHAR(36) NOT NULL"); + assertThrows(ScriptingException.class, () -> service.test(everything(), built.buildId(), event, "{}", null)); + assertEquals(before, worker.executeCalls); + assertEquals(history, count(ScriptingTables.EXECUTION)); + sql("ALTER TABLE scripting_execution ALTER COLUMN operation_id VARCHAR(64) NOT NULL"); + assertTrue(service.test(everything(), built.buildId(), event, "{}", null).succeeded()); + before = worker.executeCalls; + sql("ALTER TABLE scripting_execution ALTER COLUMN operation_id DROP NOT NULL"); + assertThrows(ScriptingException.class, () -> service.test(everything(), built.buildId(), event, "{}", null)); + assertEquals(before, worker.executeCalls); + sql("ALTER TABLE scripting_execution ALTER COLUMN operation_id SET NOT NULL"); + assertTrue(service.test(everything(), built.buildId(), event, "{}", null).succeeded()); + } + + + /*************************************************************************** + ** Release owned embedded file handles before the temporary directory closes. + ***************************************************************************/ + @AfterEach + void closeFileProviders() + { + ConnectionManager.closeConnectionProviders(); + } + + + /*************************************************************************** + ** Delayed flush, missing identity indexes and no-op transactions fail closed. + ***************************************************************************/ + @Test + void rejectsUnsafeDurabilityAndMissingSourceUniqueness() throws Exception + { + RDBMSBackendMetaData backend = (RDBMSBackendMetaData) instance.getBackendForTable("script"); + String safeUrl = backend.getJdbcUrl(); + ConnectionManager.closeConnectionProviders(); + backend.setJdbcUrl(safeUrl.replace("WRITE_DELAY=0", "WRITE_DELAY=100")); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + ConnectionManager.closeConnectionProviders(); + backend.setJdbcUrl(safeUrl); + assertTrue(newProject() > 0); + sql("DROP INDEX source_revision_identity"); + assertThrows(ScriptingException.class, this::newProject); + assertThrows(ScriptingException.class, () -> new RdbmsScriptingTransactionPolicy().validate(new QBackendTransaction(), java.util.List.of("script"))); + } +} diff --git a/rdbms/src/test/java/com/kingsrook/qbits/scripting/FixtureEndpoint.java b/rdbms/src/test/java/com/kingsrook/qbits/scripting/FixtureEndpoint.java new file mode 100644 index 0000000..aaa8a6f --- /dev/null +++ b/rdbms/src/test/java/com/kingsrook/qbits/scripting/FixtureEndpoint.java @@ -0,0 +1,214 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.github.dockerjava.api.command.InspectContainerResponse; +import com.github.dockerjava.api.model.ExposedPort; +import com.github.dockerjava.api.model.PortBinding; +import com.github.dockerjava.api.model.Ports; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; + + +/******************************************************************************* + ** Where a disposable test container is reachable FROM THIS JVM. + ** + ** The original behavior is Testcontainers' own: the host it reports (which honors + ** TESTCONTAINERS_HOST_OVERRIDE) and the published port. The bridge address of the + ** sibling is used in exactly one topology: a JVM inside a build container that the + ** SAME daemon reports on its default bridge (a local full verify through the host + ** socket). Being inside some container is not enough: on the native runner the job + ** container is not a container of the isolated test daemon at all, so a sibling's + ** bridge address (a private address of that other daemon) is unreachable and the + ** published endpoint at the configured host must be dialed. + ** + ** The choice is made from the daemon's own description of the build container's + ** networks (no connection probing). If the build container cannot be identified or + ** is not on the default bridge (a job network, host networking, a remote or + ** separate daemon), the published endpoint is used. Timeouts, scenarios and + ** assertions are unchanged. The address is re-read from the daemon on every call + ** (a restarted container may get another bridge IP or published port). + *******************************************************************************/ +final class FixtureEndpoint +{ + private static final String DEFAULT_BRIDGE = "bridge"; + + private static final Pattern MOUNTINFO_CONTAINER_ID = Pattern.compile("/docker/containers/([0-9a-f]{64})/"); + + private static volatile Boolean buildContainerOnDefaultBridge; + + + + /*************************************************************************** + ** Host and port to dial. + ***************************************************************************/ + record Address(String host, int port) + { + } + + + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private FixtureEndpoint() + { + } + + + + /*************************************************************************** + ** Resolve the endpoint of one exposed port of a running container, from a fresh + ** inspection (not Testcontainers' cached mapping). + ***************************************************************************/ + static Address of(GenericContainer container, int internalPort) + { + InspectContainerResponse inspected = container.getDockerClient().inspectContainerCmd(container.getContainerId()).exec(); + var networks = inspected.getNetworkSettings().getNetworks(); + String bridgeIp = networks == null || networks.get(DEFAULT_BRIDGE) == null ? null : networks.get(DEFAULT_BRIDGE).getIpAddress(); + Integer publishedPort = null; + var bindings = inspected.getNetworkSettings().getPorts().getBindings().get(new ExposedPort(internalPort)); + if(bindings != null && bindings.length > 0 && bindings[0].getHostPortSpec() != null) + { + publishedPort = Integer.valueOf(bindings[0].getHostPortSpec()); + } + return (choose(onBridgeWithBuildContainer(), bridgeIp, internalPort, container.getHost(), publishedPort)); + } + + + + /*************************************************************************** + ** Pure decision, separately tested. With the build container on the default + ** bridge: the sibling's bridge address and its own port. Otherwise the published + ** endpoint exactly as Testcontainers reports it. + ***************************************************************************/ + static Address choose(boolean buildContainerOnDefaultBridge, String bridgeIp, int internalPort, String publishedHost, Integer publishedPort) + { + if(buildContainerOnDefaultBridge) + { + if(bridgeIp == null || bridgeIp.isBlank()) + { + throw new IllegalStateException("FIXTURE_ENDPOINT_NO_BRIDGE_ADDRESS: the build container cannot reach a sibling without its bridge address"); + } + return (new Address(bridgeIp, internalPort)); + } + if(publishedHost == null || publishedHost.isBlank() || publishedPort == null) + { + throw new IllegalStateException("FIXTURE_ENDPOINT_NOT_PUBLISHED: no published endpoint to dial"); + } + return (new Address(publishedHost, publishedPort)); + } + + + + /*************************************************************************** + ** How the fixture publishes its port: the historical loopback-only binding + ** wherever a loopback dial can work, otherwise the daemon's own default binding. + ***************************************************************************/ + static PortBinding publishedBinding(int internalPort) + { + boolean loopback = loopbackPublication(onBridgeWithBuildContainer(), DockerClientFactory.instance().dockerHostIpAddress()); + return (new PortBinding(loopback ? Ports.Binding.bindIpAndPort("127.0.0.1", 0) : Ports.Binding.empty(), new ExposedPort(internalPort))); + } + + + + /*************************************************************************** + ** Pure decision, separately tested. A loopback-only publication is reachable + ** when the build container dials the bridge address (publication unused) or the + ** Docker host is this machine; if Testcontainers dials another host (a remote or + ** separate daemon, TESTCONTAINERS_HOST_OVERRIDE) a loopback binding could never + ** be reached, so the daemon's configured default binding is used instead. + ***************************************************************************/ + static boolean loopbackPublication(boolean buildContainerOnDefaultBridge, String dockerHost) + { + return (buildContainerOnDefaultBridge || dockerHost == null || dockerHost.isBlank() + || dockerHost.equals("localhost") || dockerHost.equals("127.0.0.1") || dockerHost.equals("::1")); + } + + + + /*************************************************************************** + ** Whether this JVM runs in a container that the daemon reports on the default + ** bridge. Decided once per JVM from the daemon's description of this container. + ***************************************************************************/ + private static boolean onBridgeWithBuildContainer() + { + Boolean known = buildContainerOnDefaultBridge; + if(known == null) + { + known = decideBuildContainerOnDefaultBridge(); + buildContainerOnDefaultBridge = known; + } + return (known); + } + + + + /*************************************************************************** + ** Inspect the build container; anything that cannot be established means the + ** original published endpoint (false). + ***************************************************************************/ + private static boolean decideBuildContainerOnDefaultBridge() + { + if(!Files.exists(Path.of("/.dockerenv"))) + { + return (false); + } + try + { + String id = ownContainerId(Files.readString(Path.of("/proc/self/mountinfo")), System.getenv("HOSTNAME")); + if(id == null) + { + return (false); + } + var networks = DockerClientFactory.instance().client().inspectContainerCmd(id).exec().getNetworkSettings().getNetworks(); + return (networks != null && networks.containsKey(DEFAULT_BRIDGE)); + } + catch(Exception e) + { + return (false); + } + } + + + + /*************************************************************************** + ** The id of the container this JVM runs in: from the mount table when it names + ** the container directory, otherwise the host name (the container id unless + ** overridden); null when neither exists. + ***************************************************************************/ + static String ownContainerId(String mountInfo, String hostName) + { + Matcher matcher = MOUNTINFO_CONTAINER_ID.matcher(mountInfo == null ? "" : mountInfo); + if(matcher.find()) + { + return (matcher.group(1)); + } + return (hostName == null || hostName.isBlank() ? null : hostName.trim()); + } + +} diff --git a/rdbms/src/test/java/com/kingsrook/qbits/scripting/FixtureEndpointTest.java b/rdbms/src/test/java/com/kingsrook/qbits/scripting/FixtureEndpointTest.java new file mode 100644 index 0000000..ecd24f6 --- /dev/null +++ b/rdbms/src/test/java/com/kingsrook/qbits/scripting/FixtureEndpointTest.java @@ -0,0 +1,114 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The topology decision of the sibling-container fixtures. + *******************************************************************************/ +class FixtureEndpointTest +{ + private static final String CONTAINER_ID = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + + + /*************************************************************************** + ** Build container on the default bridge: dial the sibling bridge IP and its own port, never the published port. + ***************************************************************************/ + @Test + void buildContainerOnTheBridgeUsesTheBridgeAddress() + { + assertEquals(new FixtureEndpoint.Address("172.17.0.9", 5432), FixtureEndpoint.choose(true, "172.17.0.9", 5432, "localhost", 41111)); + assertEquals(new FixtureEndpoint.Address("172.17.0.9", 5432), FixtureEndpoint.choose(true, "172.17.0.9", 5432, null, null)); + } + + + + /*************************************************************************** + ** Every other topology (JVM on the Docker host, a separate or remote daemon such as the native runner's isolated + ** test daemon, job network, host networking, unidentified build container) dials the published endpoint + ** at the host Testcontainers reports (including TESTCONTAINERS_HOST_OVERRIDE), never a private bridge address. + ***************************************************************************/ + @Test + void everyOtherTopologyDialsThePublishedEndpoint() + { + assertEquals(new FixtureEndpoint.Address("localhost", 41111), FixtureEndpoint.choose(false, "172.17.0.9", 5432, "localhost", 41111)); + assertEquals(new FixtureEndpoint.Address("172.17.0.1", 32768), FixtureEndpoint.choose(false, "172.17.0.3", 6379, "172.17.0.1", 32768)); + assertEquals(new FixtureEndpoint.Address("host.docker.internal", 40000), FixtureEndpoint.choose(false, null, 5432, "host.docker.internal", 40000)); + } + + + + /*************************************************************************** + ** An unknown endpoint is an error, never a guessed address. + ***************************************************************************/ + @Test + void missingEndpointFailsClosed() + { + assertThrows(IllegalStateException.class, () -> FixtureEndpoint.choose(true, null, 5432, "localhost", 41111)); + assertThrows(IllegalStateException.class, () -> FixtureEndpoint.choose(true, " ", 5432, "localhost", 41111)); + assertThrows(IllegalStateException.class, () -> FixtureEndpoint.choose(false, "172.17.0.9", 5432, "localhost", null)); + assertThrows(IllegalStateException.class, () -> FixtureEndpoint.choose(false, "172.17.0.9", 5432, " ", 41111)); + } + + + + /*************************************************************************** + ** The historical loopback-only publication stays wherever a loopback dial can work; when Testcontainers dials + ** another host (the native runner's separate daemon) the daemon's default binding is used so the dial can succeed. + ***************************************************************************/ + @Test + void loopbackPublicationOnlyWhereALoopbackDialCanWork() + { + assertTrue(FixtureEndpoint.loopbackPublication(false, "localhost")); + assertTrue(FixtureEndpoint.loopbackPublication(false, "127.0.0.1")); + assertTrue(FixtureEndpoint.loopbackPublication(false, "::1")); + assertTrue(FixtureEndpoint.loopbackPublication(false, null)); + assertTrue(FixtureEndpoint.loopbackPublication(true, "172.17.0.1")); + assertFalse(FixtureEndpoint.loopbackPublication(false, "172.17.0.1")); + assertFalse(FixtureEndpoint.loopbackPublication(false, "host.docker.internal")); + assertFalse(FixtureEndpoint.loopbackPublication(false, "docker.example.test")); + } + + + + /*************************************************************************** + ** The build container id comes from the mount table when present, else the host name. + ***************************************************************************/ + @Test + void ownContainerIdPrefersTheMountTableThenTheHostName() + { + String mountInfo = "700 699 0:50 /docker/containers/" + CONTAINER_ID + "/hostname /etc/hostname rw,relatime - ext4 /dev/sda1 rw"; + assertEquals(CONTAINER_ID, FixtureEndpoint.ownContainerId(mountInfo, "unrelated")); + assertEquals("abc123", FixtureEndpoint.ownContainerId("no container mounts here", " abc123 ")); + assertNull(FixtureEndpoint.ownContainerId("", null)); + assertNull(FixtureEndpoint.ownContainerId(null, " ")); + } + +} diff --git a/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlAfterHookTest.java b/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlAfterHookTest.java new file mode 100644 index 0000000..8d15b1f --- /dev/null +++ b/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlAfterHookTest.java @@ -0,0 +1,399 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qbits.scripting.hooks.HookEvent; +import com.kingsrook.qbits.scripting.rdbms.RdbmsScriptingTransactionPolicy; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.config.Configurator; +import org.json.JSONObject; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The unchanged native AFTER transaction/authority/audit scenarios on actual + ** MySQL, with both real owner qualifiers and shipped scripting schema/triggers. + *******************************************************************************/ +class MySqlAfterHookTest extends AfterHookJdbcTest +{ + private static final String ROOT_PASSWORD = UUID.randomUUID().toString(); + private static final String APP_PASSWORD = UUID.randomUUID().toString(); + private static final Map LOG_LEVELS = new HashMap<>(); + private static GenericContainer database; + private static String currentHost; + private static int currentPort; + private String schema; + private String jdbcUrl; + + /*************************************************************************** + ** Same pinned, disposable loopback provider; add mandatory native Jobs rules. + ***************************************************************************/ + @BeforeAll + static void startDatabase() + { + for(String logger : List.of("org.apache.hc", "org.apache.http", "org.testcontainers", "com.github.dockerjava")) + { + LOG_LEVELS.put(logger, LogManager.getLogger(logger).getLevel()); + Configurator.setLevel(logger, Level.WARN); + } + database = MySqlScriptingTest.newDatabase(ROOT_PASSWORD, APP_PASSWORD, true); + database.start(); + refreshEndpoint(); + } + + /*************************************************************************** + ** Remove only this owned container and restore test logging configuration. + ***************************************************************************/ + @AfterAll + static void stopDatabase() + { + try + { + if(database != null) + { + try + { + var health = database.getCurrentContainerInfo(); + LogManager.getLogger(MySqlFixtureDiagnostics.class).warn("MYSQL_AFTER_FIXTURE_HEALTH running={} oomKilled={} restarts={}", + health.getState().getRunning(), health.getState().getOOMKilled(), health.getRestartCount()); + } + finally + { + database.close(); + } + } + } + finally + { + LOG_LEVELS.forEach(Configurator::setLevel); + } + } + + /*************************************************************************** + ** Native source, scripting and Jobs writes use one schema and DML-only user. + ***************************************************************************/ + @Override + protected RDBMSBackendMetaData createBackend() throws Exception + { + ConnectionManager.closeConnectionProviders(); + schema = "scripting_after_" + UUID.randomUUID().toString().replace("-", ""); + String server = "jdbc:mysql://" + currentHost + ":" + currentPort + "/"; + jdbcUrl = server + schema + MySqlScriptingTest.connectionOptions(); + try(Connection connection = DriverManager.getConnection(server + "bootstrap" + MySqlScriptingTest.connectionOptions(), "root", ROOT_PASSWORD); + Statement statement = connection.createStatement()) + { + statement.execute("CREATE DATABASE " + schema + " CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"); + statement.execute("GRANT SELECT, INSERT, UPDATE, DELETE ON " + schema + ".* TO 'scripting_app'@'%'"); + } + runtime.withTransactionPolicy(new RdbmsScriptingTransactionPolicy()); + return (new RDBMSBackendMetaData().withName(schema).withVendor("mysql").withJdbcUrl(jdbcUrl).withDatabaseName(schema) + .withUsername("scripting_app").withPassword(APP_PASSWORD)); + } + + /*************************************************************************** + ** Migration and independent observations use only the owned fixture identity. + ***************************************************************************/ + @Override + protected Connection connection() throws Exception + { + return (DriverManager.getConnection(jdbcUrl, "root", ROOT_PASSWORD)); + } + + /*************************************************************************** + ** Native Jobs metadata sizes/nullability remain unchanged on MySQL types. + ***************************************************************************/ + @Override + protected String sqlType(QFieldMetaData field) + { + return (switch(field.getType()) + { + case DATE_TIME -> "DATETIME(6)"; + case TEXT, HTML -> "LONGTEXT"; + default -> super.sqlType(field); + }); + } + + /*************************************************************************** + ** Auto-increment is an actual provider identity, not a simulated key counter. + ***************************************************************************/ + @Override + protected String identityType() + { + return ("INTEGER AUTO_INCREMENT PRIMARY KEY"); + } + + /*************************************************************************** + ** Every test-owned table uses recovery-capable, binary-comparison storage. + ***************************************************************************/ + @Override + protected String tableSuffix() + { + return (" ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"); + } + + /*************************************************************************** + ** Owner tables come from the actual shipped DDL including immutability guards. + ***************************************************************************/ + @Override + protected boolean migratedTable(QTableMetaData table) + { + return (table.getName().startsWith("scripting_")); + } + + /*************************************************************************** + ** Both native source uniqueness and the production persistence policy apply. + ***************************************************************************/ + @Override + protected void finishSchema(Statement statement) throws Exception + { + statement.execute("CREATE UNIQUE INDEX source_revision_identity ON scriptRevision(scriptId, sequenceNo)"); + statement.execute("CREATE UNIQUE INDEX source_file_identity ON scriptRevisionFile(scriptRevisionId, fileName)"); + MySqlScriptingTest.applyOwnedSchema(statement); + } + + /*************************************************************************** + ** Preserve the same fault while using MySQL's CHECK-removal spelling. + ***************************************************************************/ + @Override + protected void dropCheck(Statement statement, String name) throws Exception + { + statement.execute("ALTER TABLE scripting_execution DROP CHECK " + name); + } + + /*************************************************************************** + ** A queued committed intent survives real provider restart and fresh pools. + ***************************************************************************/ + @Override + protected void restartPersistence() throws Exception + { + super.restartPersistence(); + String oldEndpoint = currentHost + ":" + currentPort; + database.getDockerClient().restartContainerCmd(database.getContainerId()).exec(); + refreshEndpoint(); + jdbcUrl = jdbcUrl.replace(oldEndpoint + "/", currentHost + ":" + currentPort + "/"); + ((RDBMSBackendMetaData) instance.getBackendForTable("script")).setJdbcUrl(jdbcUrl); + long deadline = System.nanoTime() + Duration.ofSeconds(60).toNanos(); + boolean ready = false; + while(System.nanoTime() < deadline) + { + try(Connection connection = connection(); Statement statement = connection.createStatement()) + { + statement.execute("SELECT 1"); + ready = true; + break; + } + catch(java.sql.SQLException unavailable) + { + Thread.sleep(250); + } + } + assertTrue(ready, "Owned MySQL did not recover within the existing restart bound"); + } + + /*************************************************************************** + ** The shipped original-to-current migration preserves all native event bytes. + ***************************************************************************/ + @Test + void shippedUpgradeRetainsLongCommittedCorrelation() throws Exception + { + publishNew(newProject(), "retained-pre-upgrade-history"); + try(Connection connection = connection(); Statement statement = connection.createStatement()) + { + assertOperationWidth(statement, 64); + List> originalHistory = auditRows(statement); + assertEquals(2, originalHistory.size()); + assertTrue(originalHistory.stream().allMatch(row -> row.get(1).length() == 36)); + statement.execute("DROP TABLE scripting_execution"); + statement.execute("DROP TABLE scripting_binding"); + try(var resource = getClass().getResourceAsStream("/db/mysql/001_scripting.sql")) + { + String ddl = new String(java.util.Objects.requireNonNull(resource).readAllBytes(), StandardCharsets.UTF_8) + .replaceAll("(?m)^DELIMITER.*$", "") + .replace("CONSTRAINT scripting_binding_event_check ", "") + .replace(", 'AFTER_INSERT', 'AFTER_UPDATE', 'AFTER_DELETE'", "") + .replace("operation_id VARCHAR(64)", "operation_id VARCHAR(36)"); + for(String command : ddl.split("\\$\\$")) + { + String sql = command.trim(); + if(sql.startsWith("CREATE TABLE scripting_binding ") || sql.startsWith("CREATE TABLE scripting_execution ") + || sql.startsWith("CREATE TRIGGER scripting_binding_") || sql.startsWith("CREATE TRIGGER scripting_execution_")) + { + statement.execute(sql); + } + } + } + assertOperationWidth(statement, 36); + restoreHistoricalAudit(connection, originalHistory); + assertEquals(originalHistory, auditRows(statement)); + int buildId = Integer.parseInt(originalHistory.getFirst().get(3)); + int before = worker.executeCalls; + assertThrows(ScriptingException.class, () -> service.test(everything(), buildId, HookEvent.PRE_INSERT, "{}", null)); + assertEquals(before, worker.executeCalls); + assertEquals(originalHistory, auditRows(statement)); + try(var resource = getClass().getResourceAsStream("/db/mysql/003_durable_after_hooks.sql")) + { + String ddl = new String(java.util.Objects.requireNonNull(resource).readAllBytes(), StandardCharsets.UTF_8).replaceAll("(?m)^--.*$", ""); + for(String command : ddl.split(";")) + { + if(!command.isBlank()) + { + statement.execute(command.trim()); + } + } + } + assertOperationWidth(statement, 64); + assertEquals(originalHistory, auditRows(statement), "Every existing audit column retains its value"); + int oldId = Integer.parseInt(originalHistory.getFirst().getFirst()); + assertThrows(java.sql.SQLException.class, () -> statement.executeUpdate("UPDATE scripting_execution SET status='changed' WHERE id=" + oldId)); + assertThrows(java.sql.SQLException.class, () -> statement.executeUpdate("DELETE FROM scripting_execution WHERE id=" + oldId)); + assertEquals(originalHistory, auditRows(statement)); + } + bind(HookEvent.AFTER_INSERT); + insert("upgraded"); + runJobs(); + var job = allJobs().getFirst(); + assertEquals("COMPLETED", job.getValueString("status")); + String correlation = new JSONObject(job.getValueString("paramsJson")).getString("eventId") + ":0"; + assertTrue(correlation.length() >= 38); + try(Connection connection = connection(); Statement statement = connection.createStatement(); + var rows = statement.executeQuery("SELECT operation_id,status FROM scripting_execution WHERE kind='DURABLE_HOOK' ORDER BY id")) + { + for(String status : List.of("STARTED", "OK")) + { + assertTrue(rows.next()); + assertEquals(correlation, rows.getString(1)); + assertEquals(status, rows.getString(2)); + } + assertFalse(rows.next()); + } + } + + /*************************************************************************** + ** Inspect the actual provider column, never just a matching metadata fixture. + ***************************************************************************/ + private static void assertOperationWidth(Statement statement, int expected) throws Exception + { + try(var rows = statement.executeQuery("SELECT operation_id FROM scripting_execution WHERE 1=0")) + { + assertEquals(expected, rows.getMetaData().getPrecision(1)); + } + } + + /*************************************************************************** + ** Snapshot every stored audit column without exposing it in driver logs. + ***************************************************************************/ + private static List> auditRows(Statement statement) throws Exception + { + List> result = new ArrayList<>(); + try(var rows = statement.executeQuery("SELECT * FROM scripting_execution ORDER BY id")) + { + while(rows.next()) + { + List cells = new ArrayList<>(); + for(int i = 1; i <= rows.getMetaData().getColumnCount(); i++) + { + cells.add(rows.getString(i)); + } + result.add(cells); + } + } + return (result); + } + + /*************************************************************************** + ** Reconstruct existing historical rows using only the fixture migration user. + ** The production qualifier remains enabled and refuses this original schema. + ***************************************************************************/ + private static void restoreHistoricalAudit(Connection connection, List> history) throws Exception + { + for(List row : history) + { + String placeholders = String.join(",", java.util.Collections.nCopies(row.size(), "?")); + try(var statement = connection.prepareStatement("INSERT INTO scripting_execution VALUES (" + placeholders + ")")) + { + for(int i = 0; i < row.size(); i++) + { + statement.setString(i + 1, row.get(i)); + } + assertEquals(1, statement.executeUpdate()); + } + } + } + + /*************************************************************************** + ** Stop native worker threads before dropping only this test-owned schema. + ***************************************************************************/ + @Override + @AfterEach + void stopAndClose() + { + super.stopAndClose(); + QContext.clear(); + if(schema != null && schema.matches("scripting_after_[a-f0-9]{32}")) + { + try(Connection connection = connection(); Statement statement = connection.createStatement()) + { + statement.execute("DROP DATABASE " + schema); + } + catch(Exception failure) + { + throw new AssertionError("Could not remove owned AFTER test schema", failure); + } + } + } + + /*************************************************************************** + ** Where the fixture is reachable from this JVM (host loopback, or the + ** sibling container's bridge address from a build container). + ***************************************************************************/ + private static void refreshEndpoint() + { + var address = FixtureEndpoint.of(database, 3306); + currentHost = address.host(); + currentPort = address.port(); + } + +} diff --git a/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlFixtureDiagnostics.java b/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlFixtureDiagnostics.java new file mode 100644 index 0000000..23ffe60 --- /dev/null +++ b/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlFixtureDiagnostics.java @@ -0,0 +1,89 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import com.mysql.cj.exceptions.ExceptionInterceptor; +import com.mysql.cj.log.Log; +import org.apache.logging.log4j.LogManager; + + +/******************************************************************************* + ** Test-only transport diagnostics never include exception messages, SQL, values, + ** connection properties, credentials or database/schema names. + *******************************************************************************/ +public final class MySqlFixtureDiagnostics implements ExceptionInterceptor +{ + /*************************************************************************** + ** Connector/J constructs this explicitly named test interceptor. + ***************************************************************************/ + public MySqlFixtureDiagnostics() {} + + /*************************************************************************** + ** Driver properties are deliberately neither retained nor inspected. + ***************************************************************************/ + @Override + public ExceptionInterceptor init(Properties properties, Log log) + { + return (this); + } + + /*************************************************************************** + ** There is no retained driver or caller state to release. + ***************************************************************************/ + @Override + public void destroy() {} + + /*************************************************************************** + ** Bounded type/state evidence preserves the original driver's exception. + ***************************************************************************/ + @Override + public Exception interceptException(Exception failure) + { + List classes = new ArrayList<>(); + Throwable current = failure; + for(int i = 0; current != null && i < 8; i++) + { + classes.add(current.getClass().getName()); + current = current.getCause(); + } + String state = failure instanceof SQLException sql ? sql.getSQLState() : null; + if(state == null || !state.matches("[A-Z0-9]{5}")) + { + state = "unknown"; + } + String policy = "outside-policy"; + for(StackTraceElement frame : Thread.currentThread().getStackTrace()) + { + if(frame.getClassName().equals("com.kingsrook.qbits.scripting.rdbms.RdbmsScriptingTransactionPolicy")) + { + policy = frame.getMethodName(); + break; + } + } + LogManager.getLogger(MySqlFixtureDiagnostics.class).warn("MYSQL_FIXTURE_DIAGNOSTIC state={} classes={} policyMethod={}", state, classes, policy); + return (failure); + } +} diff --git a/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlScriptingTest.java b/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlScriptingTest.java new file mode 100644 index 0000000..0976db7 --- /dev/null +++ b/rdbms/src/test/java/com/kingsrook/qbits/scripting/MySqlScriptingTest.java @@ -0,0 +1,486 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qbits.scripting.rdbms.RdbmsScriptingTransactionPolicy; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.config.Configurator; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The same native transaction/concurrency/hooks suite against actual MySQL8.4 + ** and the shipped migration. Worker protocol remains a trusted fixture; no + ** untrusted script, production database, external volume or host service runs. + *******************************************************************************/ +class MySqlScriptingTest extends JdbcScriptingTest +{ + private static final String IMAGE = "mysql@sha256:a0225d1ea6631c8bc2385dc4c2faba17bbf1834ecf369e1659a9a382bd8d5dd7"; + private static final String ROOT_PASSWORD = UUID.randomUUID().toString(); + private static final String APP_PASSWORD = UUID.randomUUID().toString(); + private static GenericContainer database; + private static String currentHost; + private static int currentPort; + private static final Map LOG_LEVELS = new HashMap<>(); + private String schema; + private String url; + + + + /*************************************************************************** + ** Only an owned disposable loopback container, with durable commit settings. + ***************************************************************************/ + @BeforeAll + static void startDatabase() + { + for(String logger : List.of("org.apache.hc", "org.apache.http", "org.testcontainers", "com.github.dockerjava")) + { + LOG_LEVELS.put(logger, LogManager.getLogger(logger).getLevel()); + Configurator.setLevel(logger, Level.WARN); + } + database = newDatabase(ROOT_PASSWORD, APP_PASSWORD, false); + database.start(); + refreshEndpoint(); + } + + + + /*************************************************************************** + ** Shared pinned provider factory; Jobs adds its mandatory isolation/DDL bound. + ***************************************************************************/ + static GenericContainer newDatabase(String rootPassword, String appPassword, boolean transactionalJobs) + { + List commands = new ArrayList<>(List.of("--innodb-flush-log-at-trx-commit=1", "--sync-binlog=1", "--innodb-lock-wait-timeout=5")); + if(transactionalJobs) + { + commands.add("--lock-wait-timeout=5"); + commands.add("--transaction-isolation=READ-COMMITTED"); + } + return (new GenericContainer<>(IMAGE) + .withEnv("MYSQL_ROOT_PASSWORD", rootPassword).withEnv("MYSQL_ROOT_HOST", "%") + .withEnv("MYSQL_DATABASE", "bootstrap").withEnv("MYSQL_USER", "scripting_app").withEnv("MYSQL_PASSWORD", appPassword) + .withCommand(commands.toArray(String[]::new)) + .withExposedPorts(3306) + .withCreateContainerCmdModifier(command -> command.getHostConfig() + .withMemory(768L * 1024 * 1024).withNanoCPUs(2_000_000_000L) + .withPortBindings(FixtureEndpoint.publishedBinding(3306))) + .waitingFor(Wait.forListeningPort().withStartupTimeout(java.time.Duration.ofMinutes(2)))); + } + + /*************************************************************************** + ** Keep identical bounded transport, UTC conversion and safe diagnostics. + ***************************************************************************/ + static String connectionOptions() + { + return ("?sslMode=DISABLED&allowPublicKeyRetrieval=true&connectionTimeZone=UTC&forceConnectionTimeZoneToSession=true&connectTimeout=1000&socketTimeout=10000&exceptionInterceptors=com.kingsrook.qbits.scripting.MySqlFixtureDiagnostics"); + } + + /*************************************************************************** + ** Actual owner migration retains immutable history/source protection triggers. + ***************************************************************************/ + static void applyOwnedSchema(Statement statement) throws Exception + { + try(var resource = MySqlScriptingTest.class.getResourceAsStream("/db/mysql/001_scripting.sql")) + { + String ddl = new String(java.util.Objects.requireNonNull(resource).readAllBytes(), StandardCharsets.UTF_8) + .replaceAll("(?m)^DELIMITER.*$", ""); + for(String command : ddl.split("\\$\\$")) + { + if(!command.isBlank()) + { + statement.execute(command.trim()); + } + } + } + } + + /*************************************************************************** + ** Exact owned container removal; no shared services or named volumes. + ***************************************************************************/ + @AfterAll + static void stopDatabase() + { + try + { + if(database != null) + { + try + { + var health = database.getCurrentContainerInfo(); + LogManager.getLogger(MySqlFixtureDiagnostics.class).warn("MYSQL_FIXTURE_HEALTH running={} oomKilled={} restarts={}", + health.getState().getRunning(), health.getState().getOOMKilled(), health.getRestartCount()); + } + finally + { + database.close(); + } + } + } + finally + { + LOG_LEVELS.forEach(Configurator::setLevel); + } + } + + + + /*************************************************************************** + ** Each inherited test has a new schema. Native QQQ gets a DML-only identity; + ** migration and fault injection use the separate test migration identity. + ***************************************************************************/ + @Override + @BeforeEach + void useJdbc() throws Exception + { + ConnectionManager.closeConnectionProviders(); + schema = "scripting_" + UUID.randomUUID().toString().replace("-", ""); + String server = "jdbc:mysql://" + currentHost + ":" + currentPort + "/"; + String options = connectionOptions(); + url = server + schema + options; + try(Connection root = DriverManager.getConnection(server + "bootstrap" + options, "root", ROOT_PASSWORD); var statement = root.createStatement()) + { + statement.execute("CREATE DATABASE " + schema + " CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin"); + statement.execute("GRANT SELECT, INSERT, UPDATE, DELETE ON " + schema + ".* TO 'scripting_app'@'%'"); + } + String backend = "mysql_" + schema; + instance.addBackend(new RDBMSBackendMetaData().withName(backend).withVendor("mysql").withJdbcUrl(url) + .withDatabaseName(schema).withUsername("scripting_app").withPassword(APP_PASSWORD)); + try(Connection root = jdbcConnection(); var statement = root.createStatement()) + { + for(var table : instance.getTables().values()) + { + table.setBackendName(backend); + table.setBackendDetails(new RDBMSTableBackendDetails().withTableName(table.getName())); + for(var field : table.getFields().values()) + { + field.setBackendName(field.getName()); + } + if(table.getName().startsWith("scripting_")) + { + continue; + } + List columns = new ArrayList<>(); + for(var field : table.getFields().values()) + { + String type = switch(field.getType()) + { + case INTEGER -> "INTEGER"; + case LONG -> "BIGINT"; + case BOOLEAN -> "BOOLEAN"; + case DECIMAL -> "DECIMAL(30,10)"; + case DATE_TIME -> "TIMESTAMP(6) NULL"; + case DATE -> "DATE"; + case TIME -> "TIME"; + case TEXT, HTML -> "LONGTEXT"; + case BLOB -> "LONGBLOB"; + default -> "VARCHAR(" + (field.getMaxLength() == null ? 2000 : field.getMaxLength()) + ")"; + }; + columns.add("`" + field.getName() + "` " + (field.getName().equals("id") ? "INTEGER AUTO_INCREMENT PRIMARY KEY" : type)); + } + if(table.getUniqueKeys() != null) + { + for(var key : table.getUniqueKeys()) + { + columns.add("UNIQUE (" + String.join(",", key.getFieldNames()) + ")"); + } + } + statement.execute("CREATE TABLE `" + table.getName() + "` (" + String.join(",", columns) + + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_bin"); + } + statement.execute("CREATE UNIQUE INDEX source_revision_identity ON scriptRevision(scriptId, sequenceNo)"); + statement.execute("CREATE UNIQUE INDEX source_file_identity ON scriptRevisionFile(scriptRevisionId, fileName)"); + applyOwnedSchema(statement); + } + runtime.withTransactionPolicy(new RdbmsScriptingTransactionPolicy()); + } + + + + /*************************************************************************** + ** The observer is independent of the application's pending transaction. + ***************************************************************************/ + @Override + Connection jdbcConnection() throws Exception + { + return (DriverManager.getConnection(url, "root", ROOT_PASSWORD)); + } + + + + /*************************************************************************** + ** Test-owned constraint injection uses the MySQL CHECK removal spelling. + ***************************************************************************/ + @Override + void sql(String command) throws Exception + { + if(command.startsWith("ALTER TABLE scripting_execution ADD CONSTRAINT reject_refusal ")) + { + // MySQL forbids AUTO_INCREMENT references in CHECK expressions. Reject + // subsequent inserts with the same real database fault, preserving history. + super.sql("CREATE TRIGGER reject_refusal BEFORE INSERT ON scripting_execution FOR EACH ROW BEGIN " + + "IF NEW.kind='HOOK_REFUSAL' THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='Injected audit outage'; END IF; END"); + return; + } + super.sql(command.replace(" DROP CONSTRAINT ", " DROP CHECK ")); + } + + + + /*************************************************************************** + ** Remove only this test's generated schema after native pools release it. + ***************************************************************************/ + @AfterEach + void closeSchema() throws Exception + { + QContext.clear(); + ConnectionManager.closeConnectionProviders(); + if(schema != null && schema.matches("scripting_[a-f0-9]{32}")) + { + try(Connection root = jdbcConnection(); var statement = root.createStatement()) + { + statement.execute("DROP DATABASE " + schema); + } + } + } + + + + /*************************************************************************** + ** The shipped schema, not application button visibility, protects provenance. + ***************************************************************************/ + @Test + void migrationProtectsHistoryAndDmlIdentity() throws Exception + { + var built = build(draft(newProject(), "immutable")); + try(Connection app = DriverManager.getConnection(url, "scripting_app", APP_PASSWORD); var statement = app.createStatement()) + { + assertThrows(java.sql.SQLException.class, () -> statement.executeUpdate("UPDATE scripting_build SET status='FAILED' WHERE id=" + built.buildId())); + assertThrows(java.sql.SQLException.class, () -> statement.executeUpdate("DELETE FROM scripting_build WHERE id=" + built.buildId())); + assertThrows(java.sql.SQLException.class, () -> statement.execute("DROP TRIGGER scripting_build_no_update")); + } + try(Connection root = jdbcConnection(); var statement = root.createStatement(); + var result = statement.executeQuery("SELECT @@innodb_flush_log_at_trx_commit, @@sync_binlog, @@log_bin")) + { + assertTrue(result.next()); + assertEquals(1, result.getInt(1)); + assertEquals(1, result.getInt(2)); + assertEquals(1, result.getInt(3)); + } + assertEquals(1, count(ScriptingTables.BUILD)); + } + + + /*************************************************************************** + ** Core source storage and commit configuration are checked in production, + ** not merely established by this test's startup command. + ***************************************************************************/ + @Test + void unsafeCoreEngineAndCommitSettingsAreRejectedBeforeWrites() throws Exception + { + sql("ALTER TABLE scriptRevisionFile ENGINE=MyISAM"); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + sql("ALTER TABLE scriptRevisionFile ENGINE=InnoDB"); + try + { + sql("SET GLOBAL innodb_flush_log_at_trx_commit=2"); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + } + finally + { + sql("SET GLOBAL innodb_flush_log_at_trx_commit=1"); + } + assertTrue(newProject() > 0); + } + + + /*************************************************************************** + ** Missing identity uniqueness or case-folding source columns deny admission. + ***************************************************************************/ + @Test + void sourceIdentityConstraintsAreRequired() throws Exception + { + sql("DROP INDEX source_revision_identity ON scriptRevision"); + assertThrows(ScriptingException.class, this::newProject); + sql("CREATE UNIQUE INDEX source_revision_identity ON scriptRevision(scriptId, sequenceNo)"); + sql("ALTER TABLE scriptRevisionFile MODIFY fileName VARCHAR(255) COLLATE utf8mb4_0900_ai_ci"); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + } + + + /*************************************************************************** + ** A completed publication and core activation survive a real provider restart. + ** Refresh the actual dynamic port; Testcontainers caches its initial mapping. + ***************************************************************************/ + @Test + void committedPublicationSurvivesProviderRestart() throws Exception + { + Integer project = newProject(); + Integer publication = publishNew(project, "restart"); + ConnectionManager.closeConnectionProviders(); + String oldEndpoint = currentHost + ":" + currentPort; + database.getDockerClient().restartContainerCmd(database.getContainerId()).exec(); + refreshEndpoint(); + url = url.replace(oldEndpoint + "/", currentHost + ":" + currentPort + "/"); + ((RDBMSBackendMetaData) instance.getBackendForTable("script")).setJdbcUrl(url); + long deadline = System.nanoTime() + java.time.Duration.ofSeconds(60).toNanos(); + boolean ready = false; + while(System.nanoTime() < deadline) + { + try(Connection observer = jdbcConnection(); var statement = observer.createStatement()) + { + statement.execute("SELECT 1"); + ready = true; + break; + } + catch(java.sql.SQLException unavailable) + { + Thread.sleep(250); + } + } + assertTrue(ready); + assertEquals(publication, rows(ScriptingTables.PUBLICATION).getFirst().getValueInteger("id")); + assertTrue(service.authoring().project(everything(), project).latestRevisionId() > 0); + assertTrue(draft(project, "after-restart") > 0); + } + + + + /*************************************************************************** + ** Preserve native source guards while two saves reach the database with the + ** same expected revision and sequence. Exactly one immutable draft commits. + ***************************************************************************/ + @Test + void sourceUniqueKeyArbitratesConcurrentDrafts() throws Exception + { + Integer project = newProject(); + var table = instance.getTable("scriptRevision"); + var role = com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_INSERT_RECORD; + var prior = table.getCustomizer(role.getRole()).orElseThrow(); + table.getCustomizers().put(role.getRole(), com.kingsrook.qqq.backend.core.actions.customizers.MultiCustomizer.of(prior, + new com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference(PublicationRaceGate.class))); + PublicationRaceGate.gate = new java.util.concurrent.CyclicBarrier(2); + var session = QContext.getQSession(); + try(var executor = java.util.concurrent.Executors.newFixedThreadPool(2)) + { + java.util.concurrent.Callable save = () -> + { + QContext.init(instance, session); + try + { + service.saveDraft(everything(), project, null, "Concurrent save", source("race")); + return (true); + } + catch(ScriptingException conflict) + { + return (false); + } + finally + { + QContext.clear(); + } + }; + var first = executor.submit(save); + var second = executor.submit(save); + int committed = (first.get(15, java.util.concurrent.TimeUnit.SECONDS) ? 1 : 0) + (second.get(15, java.util.concurrent.TimeUnit.SECONDS) ? 1 : 0); + assertEquals(1, committed); + } + assertEquals(1, count("scriptRevision")); + assertEquals(1, count("scriptRevisionFile")); + } + + + + /*************************************************************************** + ** Detection-only torn-page handling and session read-only transactions cannot + ** qualify durable scripting writes. Changes affect this disposable server only. + ***************************************************************************/ + @Test + void doublewriteAndSessionReadOnlyAreEnforced() throws Exception + { + try + { + sql("SET GLOBAL innodb_doublewrite=DETECT_ONLY"); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + } + finally + { + sql("SET GLOBAL innodb_doublewrite=ON"); + } + try(var transaction = com.kingsrook.qqq.backend.core.actions.QBackendTransaction.openFor( + new com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput("script"))) + { + Connection connection = ((com.kingsrook.qqq.backend.module.rdbms.actions.RDBMSTransaction) transaction).getConnection(); + try(var statement = connection.createStatement()) + { + try + { + statement.execute("SET SESSION transaction_read_only=1"); + assertThrows(ScriptingException.class, () -> new RdbmsScriptingTransactionPolicy().validate(transaction, List.of("script"))); + } + finally + { + statement.execute("SET SESSION transaction_read_only=0"); + } + } + } + assertTrue(newProject() > 0); + } + + /*************************************************************************** + ** Where the fixture is reachable from this JVM (host loopback, or the + ** sibling container's bridge address from a build container). + ***************************************************************************/ + private static void refreshEndpoint() + { + var address = FixtureEndpoint.of(database, 3306); + currentHost = address.host(); + currentPort = address.port(); + } + +} diff --git a/rdbms/src/test/java/com/kingsrook/qbits/scripting/PostgresScriptingTest.java b/rdbms/src/test/java/com/kingsrook/qbits/scripting/PostgresScriptingTest.java new file mode 100644 index 0000000..b13f755 --- /dev/null +++ b/rdbms/src/test/java/com/kingsrook/qbits/scripting/PostgresScriptingTest.java @@ -0,0 +1,387 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.scripting; + + +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import com.kingsrook.qbits.scripting.rdbms.RdbmsScriptingTransactionPolicy; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.module.postgres.model.metadata.PostgreSQLBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDetails; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The same native transaction/concurrency/hooks suite against an actual + ** PostgreSQL 17 server and the shipped PostgreSQL migrations. The worker + ** protocol remains a trusted fixture; no untrusted script, production database, + ** external volume or host service runs. Core script tables are mapped to + ** lower-case physical names, as a PostgreSQL host must (unquoted identifiers). + *******************************************************************************/ +class PostgresScriptingTest extends JdbcScriptingTest +{ + private static final String ROOT_PASSWORD = UUID.randomUUID().toString(); + private static final String APP_PASSWORD = UUID.randomUUID().toString(); + private static final String OPTIONS = "?connectTimeout=5&socketTimeout=20&options=-c%20TimeZone%3DUTC"; + private static GenericContainer database; + private static java.util.TimeZone previousZone; + private static String server; + private String schema; + private String url; + + /*************************************************************************** + ** Only an owned disposable loopback container, with durable commit settings. + ***************************************************************************/ + @BeforeAll + static void startDatabase() throws Exception + { + // pgjdbc announces the JVM zone as the session TimeZone; qualification requires UTC. + previousZone = java.util.TimeZone.getDefault(); + java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC")); + database = new GenericContainer<>("postgres:17") + .withEnv("POSTGRES_PASSWORD", ROOT_PASSWORD).withEnv("POSTGRES_DB", "bootstrap") + .withCommand("postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "max_connections=80") + .withExposedPorts(5432) + .withCreateContainerCmdModifier(command -> command.getHostConfig().withMemory(768L * 1024 * 1024).withNanoCPUs(2_000_000_000L) + .withPortBindings(FixtureEndpoint.publishedBinding(5432))) + .waitingFor(Wait.forLogMessage(".*database system is ready to accept connections.*\\s", 2).withStartupTimeout(java.time.Duration.ofMinutes(2))); + database.start(); + refreshEndpoint(); + try(Connection root = DriverManager.getConnection(server + "bootstrap" + OPTIONS, "postgres", ROOT_PASSWORD); var statement = root.createStatement()) + { + statement.execute("CREATE ROLE scripting_app LOGIN PASSWORD '" + APP_PASSWORD + "' NOSUPERUSER NOCREATEDB NOCREATEROLE"); + statement.execute("ALTER ROLE scripting_app SET lock_timeout='5s'"); + } + } + + /*************************************************************************** + ** Exact owned container removal; no shared services or named volumes. + ***************************************************************************/ + @AfterAll + static void stopDatabase() + { + if(database != null) + { + database.close(); + } + if(previousZone != null) + { + java.util.TimeZone.setDefault(previousZone); + } + } + + /*************************************************************************** + ** Each inherited test has a new database. Native QQQ gets a DML-only identity; + ** migration and fault injection use the separate fixture superuser. + ***************************************************************************/ + @Override + @BeforeEach + void useJdbc() throws Exception + { + ConnectionManager.closeConnectionProviders(); + schema = "scripting_" + UUID.randomUUID().toString().replace("-", ""); + url = server + schema + OPTIONS; + try(Connection root = DriverManager.getConnection(server + "bootstrap" + OPTIONS, "postgres", ROOT_PASSWORD); var statement = root.createStatement()) + { + statement.execute("CREATE DATABASE " + schema); + statement.execute("REVOKE ALL ON DATABASE " + schema + " FROM PUBLIC"); + statement.execute("GRANT CONNECT ON DATABASE " + schema + " TO scripting_app"); + } + String backend = "postgres_" + schema; + var postgresBackend = new PostgreSQLBackendMetaData(); + postgresBackend.setName(backend); + postgresBackend.setDatabaseName(schema); + postgresBackend.setUsername("scripting_app"); + postgresBackend.setPassword(APP_PASSWORD); + postgresBackend.setJdbcUrl(url); + instance.addBackend(postgresBackend); + try(Connection root = jdbcConnection(); var statement = root.createStatement()) + { + statement.execute("GRANT USAGE ON SCHEMA public TO scripting_app"); + statement.execute("ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO scripting_app"); + statement.execute("ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO scripting_app"); + for(var table : instance.getTables().values()) + { + boolean owned = table.getName().startsWith("scripting_"); + String physical = owned ? table.getName() : table.getName().toLowerCase(); + table.setBackendName(backend); + table.setBackendDetails(new RDBMSTableBackendDetails().withTableName(physical)); + List columns = new ArrayList<>(); + for(var field : table.getFields().values()) + { + field.setBackendName(owned ? field.getName() : field.getName().toLowerCase()); + String type = switch(field.getType()) + { + case INTEGER -> "INTEGER"; + case LONG -> "BIGINT"; + case BOOLEAN -> "BOOLEAN"; + case DECIMAL -> "NUMERIC(30,10)"; + case DATE_TIME -> "TIMESTAMPTZ"; + case DATE -> "DATE"; + case TIME -> "TIME"; + case TEXT, HTML -> "TEXT"; + case BLOB -> "BYTEA"; + default -> "VARCHAR(" + (field.getMaxLength() == null ? 2000 : field.getMaxLength()) + ")"; + }; + columns.add(field.getBackendName() + " " + (field.getName().equals("id") ? "INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY" : type)); + } + if(owned) + { + continue; + } + if(table.getUniqueKeys() != null) + { + for(var key : table.getUniqueKeys()) + { + columns.add("UNIQUE (" + String.join(",", key.getFieldNames().stream().map(String::toLowerCase).toList()) + ")"); + } + } + statement.execute("CREATE TABLE " + physical + " (" + String.join(",", columns) + ")"); + } + statement.execute("CREATE UNIQUE INDEX source_revision_identity ON scriptrevision(scriptid, sequenceno)"); + statement.execute("CREATE UNIQUE INDEX source_file_identity ON scriptrevisionfile(scriptrevisionid, filename)"); + statement.execute(resource("/db/postgresql/001_scripting.sql")); + statement.execute(resource("/db/postgresql/002_source_import.sql")); + } + runtime.withTransactionPolicy(new RdbmsScriptingTransactionPolicy()); + } + + /*************************************************************************** + ** The observer is independent of the application's pending transaction. + ***************************************************************************/ + @Override + Connection jdbcConnection() throws Exception + { + return (DriverManager.getConnection(url, "postgres", ROOT_PASSWORD)); + } + + /*************************************************************************** + ** Remove only this test's generated database after native pools release it. + ***************************************************************************/ + @AfterEach + void closeSchema() throws Exception + { + QContext.clear(); + ConnectionManager.closeConnectionProviders(); + if(schema != null && schema.matches("scripting_[a-f0-9]{32}")) + { + try(Connection root = DriverManager.getConnection(server + "bootstrap" + OPTIONS, "postgres", ROOT_PASSWORD); var statement = root.createStatement()) + { + statement.execute("DROP DATABASE " + schema + " WITH (FORCE)"); + } + } + } + + /*************************************************************************** + ** The shipped schema, not application button visibility, protects provenance. + ***************************************************************************/ + @Test + void migrationProtectsHistoryAndDmlIdentity() throws Exception + { + var built = build(draft(newProject(), "immutable")); + try(Connection app = DriverManager.getConnection(url, "scripting_app", APP_PASSWORD); var statement = app.createStatement()) + { + assertThrows(java.sql.SQLException.class, () -> statement.executeUpdate("UPDATE scripting_build SET status='FAILED' WHERE id=" + built.buildId())); + assertThrows(java.sql.SQLException.class, () -> statement.executeUpdate("DELETE FROM scripting_build WHERE id=" + built.buildId())); + assertThrows(java.sql.SQLException.class, () -> statement.execute("DROP TRIGGER scripting_build_immutable ON scripting_build")); + assertThrows(java.sql.SQLException.class, () -> statement.execute("ALTER TABLE scripting_build DISABLE TRIGGER scripting_build_immutable")); + } + try(Connection root = jdbcConnection(); var statement = root.createStatement(); + var result = statement.executeQuery("SELECT current_setting('fsync'), current_setting('synchronous_commit'), current_setting('full_page_writes')")) + { + assertTrue(result.next()); + assertEquals("on", result.getString(1)); + assertEquals("on", result.getString(2)); + assertEquals("on", result.getString(3)); + } + assertEquals(1, count(ScriptingTables.BUILD)); + } + + /*************************************************************************** + ** Core source storage and commit configuration are checked in production, + ** not merely established by this test's startup command. + ***************************************************************************/ + @Test + void unsafeCoreTablesAndCommitSettingsAreRejectedBeforeWrites() throws Exception + { + sql("ALTER TABLE scriptrevisionfile SET UNLOGGED"); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + sql("ALTER TABLE scriptrevisionfile SET LOGGED"); + try + { + sql("ALTER ROLE scripting_app SET synchronous_commit='off'"); + ConnectionManager.closeConnectionProviders(); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + sql("ALTER ROLE scripting_app SET synchronous_commit='on'"); + sql("ALTER ROLE scripting_app RESET lock_timeout"); + ConnectionManager.closeConnectionProviders(); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + sql("ALTER ROLE scripting_app SET lock_timeout='5s'"); + sql("ALTER ROLE scripting_app SET default_transaction_isolation='repeatable read'"); + ConnectionManager.closeConnectionProviders(); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + } + finally + { + sql("ALTER ROLE scripting_app SET synchronous_commit='on'"); + sql("ALTER ROLE scripting_app SET lock_timeout='5s'"); + sql("ALTER ROLE scripting_app RESET default_transaction_isolation"); + ConnectionManager.closeConnectionProviders(); + } + assertTrue(newProject() > 0); + } + + /*************************************************************************** + ** Missing, partial or case-folding source identity constraints deny admission. + ***************************************************************************/ + @Test + void sourceIdentityConstraintsAreRequired() throws Exception + { + sql("DROP INDEX source_revision_identity"); + assertThrows(ScriptingException.class, this::newProject); + sql("CREATE UNIQUE INDEX source_revision_partial ON scriptrevision(scriptid, sequenceno) WHERE sequenceno > 0"); + assertThrows(ScriptingException.class, this::newProject); + sql("DROP INDEX source_revision_partial"); + sql("CREATE UNIQUE INDEX source_revision_identity ON scriptrevision(scriptid, sequenceno)"); + sql("CREATE COLLATION fixture_ci (provider = icu, locale = 'und-u-ks-level2', deterministic = false)"); + sql("ALTER TABLE scriptrevisionfile ALTER COLUMN filename TYPE VARCHAR(255) COLLATE fixture_ci"); + assertThrows(ScriptingException.class, this::newProject); + assertEquals(0, count("script")); + } + + /*************************************************************************** + ** A vendor label on the generic rdbms module is not the native PostgreSQL backend. + ***************************************************************************/ + @Test + void genericRdbmsLabelIsNotQualified() throws Exception + { + ((RDBMSBackendMetaData) instance.getBackendForTable("script")).setBackendType("rdbms"); + assertThrows(Exception.class, this::newProject); + assertEquals(0, count("script")); + } + + /*************************************************************************** + ** An installation from the original 001 script upgrades in place, keeping + ** append-only history, and is refused until the upgrade is applied. + ***************************************************************************/ + @Test + void originalSchemaIsRefusedUntilUpgradeThenKeepsHistory() throws Exception + { + var built = build(draft(newProject(), "legacy")); + assertEquals(1, count(ScriptingTables.BUILD)); + sql("ALTER TABLE scripting_execution ALTER COLUMN operation_id TYPE VARCHAR(36)"); + sql("ALTER TABLE scripting_binding DROP CONSTRAINT scripting_binding_event_check"); + sql("ALTER TABLE scripting_binding ADD CONSTRAINT scripting_binding_event_check CHECK (event IN ('PRE_INSERT', 'PRE_UPDATE', 'PRE_DELETE', 'POST_INSERT', 'POST_UPDATE', 'POST_DELETE'))"); + assertThrows(ScriptingException.class, this::newProject); + int before = count(ScriptingTables.EXECUTION); + sql(resource("/db/postgresql/003_durable_after_hooks.sql")); + sql(resource("/db/postgresql/003_durable_after_hooks.sql")); + assertTrue(newProject() > 0); + assertEquals(1, count(ScriptingTables.BUILD)); + assertEquals(before, count(ScriptingTables.EXECUTION)); + try(Connection root = jdbcConnection(); var statement = root.createStatement(); + var result = statement.executeQuery("SELECT pg_get_constraintdef(oid), (SELECT character_maximum_length FROM information_schema.columns" + + " WHERE table_name='scripting_execution' AND column_name='operation_id') FROM pg_constraint WHERE conname='scripting_binding_event_check'")) + { + assertTrue(result.next()); + assertTrue(result.getString(1).contains("AFTER_INSERT")); + assertEquals(64, result.getInt(2)); + } + assertTrue(built.buildId() > 0); + } + + /*************************************************************************** + ** A completed publication and core activation survive a real server restart. + ** Refresh the actual dynamic port; Testcontainers caches its initial mapping. + ***************************************************************************/ + @Test + void committedPublicationSurvivesProviderRestart() throws Exception + { + Integer project = newProject(); + Integer publication = publishNew(project, "restart"); + ConnectionManager.closeConnectionProviders(); + String oldServer = server; + database.getDockerClient().stopContainerCmd(database.getContainerId()).withTimeout(10).exec(); + database.getDockerClient().startContainerCmd(database.getContainerId()).exec(); + refreshEndpoint(); + url = url.replace(oldServer, server); + ((RDBMSBackendMetaData) instance.getBackendForTable("script")).setJdbcUrl(url); + long deadline = System.nanoTime() + java.time.Duration.ofSeconds(60).toNanos(); + boolean ready = false; + while(System.nanoTime() < deadline) + { + try(Connection observer = jdbcConnection(); var statement = observer.createStatement()) + { + statement.execute("SELECT 1"); + ready = true; + break; + } + catch(java.sql.SQLException unavailable) + { + Thread.sleep(250); + } + } + assertTrue(ready); + assertEquals(publication, rows(ScriptingTables.PUBLICATION).getFirst().getValueInteger("id")); + assertTrue(service.authoring().project(everything(), project).latestRevisionId() > 0); + assertTrue(draft(project, "after-restart") > 0); + } + + /*************************************************************************** + ** Read a classpath migration exactly as shipped. + ***************************************************************************/ + private static String resource(String name) throws Exception + { + try(var input = PostgresScriptingTest.class.getResourceAsStream(name)) + { + return (new String(java.util.Objects.requireNonNull(input).readAllBytes(), StandardCharsets.UTF_8)); + } + } + + /*************************************************************************** + ** Native inspect avoids Testcontainers' stale mapped-port cache after restart. + ***************************************************************************/ + private static void refreshEndpoint() + { + var address = FixtureEndpoint.of(database, 5432); + server = "jdbc:postgresql://" + address.host() + ":" + address.port() + "/"; + } +} diff --git a/supervisor/AGENTS.md b/supervisor/AGENTS.md new file mode 100644 index 0000000..5aae89a --- /dev/null +++ b/supervisor/AGENTS.md @@ -0,0 +1,5 @@ +# External supervisor scope + +This is a separate process from the QQQ application JVM. Root explicitly authorized the fixed Javy/Wasm execution adapter here through the public Extism C ABI. Do not import QQQ, QContext, other QBits, Node execution, or application credentials. The sibling base QBit never evaluates author code. + +Run the repository wrapper from the parent `qbit-scripting` directory: `./mvnw.cmd -o -f supervisor/pom.xml verify`. The shared Checkstyle header path is relative to that working directory. Do not install into shared caches. Java21, three-space Allman, explicit sorted imports and flower-box Javadoc as parent style. Root owns native dependency downloads and exact isolated Linux execution. Trusted protocol fixtures are not runtime qualification. diff --git a/supervisor/COLD-COMPILER.md b/supervisor/COLD-COMPILER.md new file mode 100644 index 0000000..295f74d --- /dev/null +++ b/supervisor/COLD-COMPILER.md @@ -0,0 +1,84 @@ +# Separate cold TypeScript compiler endpoint + +`CompilerMain` is a separate optional process/listener from `SupervisorMain`. It implements only the existing authenticated COMPILE exchange; it has no Wasm engine, execution slots, ready cache, broker, QQQ dependency, application credentials or author-code execution path. The Wasm endpoint continues to refuse COMPILE. The caller configures `PersistentWorkerProvider.compileEndpoint` separately; cold compilation never consumes interactive execution admission. + +This is a staged transport/compiler candidate. Trusted Java child and TLS fixtures prove protocol/cleanup behavior, not Node isolation, Kubernetes qualification or latency. A matching operator evidence string is configuration, not self-certification. + +## Fixed compiler image + +Reuse the existing worker's pinned Node **24.22.2**, TypeScript **6.0.3**, `compiler.ts`, `protocol.ts` and `frames.ts`. `compiler-entry.ts` imports only that compiler path; it never imports `execute.ts`, evaluates the emitted bundle, loads author packages/plugins or accepts compiler flags. The compiler retains its virtual source filesystem, explicit SDK, strict/noEmitOnError policy, approved relative/type-only imports and standard-library allowlist. + +Root/operator build recipe (no dependency installation or author script invocation in this recipe): + +```text + /node_modules/typescript/bin/tsc -p /tsconfig.json + /fixtures/build-compiler-image.mjs +``` + +The generator refuses an existing output directory. It packages four emitted compiler JS files, the worker package metadata, TypeScript's package metadata and standalone typescript.js, plus shipped `lib.*.d.ts`. It records exact SHA256 hashes in `compiler/compiler-manifest.json`; adjacent `compiler-image.json` records the manifest hash and the platform-specific Node executable hash. Adjacent notices preserve the worker Apache notice/license and TypeScript license. Ship Node's official distribution license with the separately pinned Node binary. There are no executor, coordinator, test or arbitrary package files in the compiler tree. The generator does not import or run the compiler. + +`CompilerImage` independently checks the executable hash, manifest hash, exact file inventory, every file hash, allowed directories and no symlink files/root. Limits:256 manifest files,16MiB per file,64MiB aggregate compiler files. Mount the binary/compiler tree readonly from an operator-owned immutable image: startup hash checks do not prevent a privileged operator replacing files afterwards. Record the build source, target architecture, Node distribution provenance, manifest and final image digest in external qualification evidence. + +## Endpoint configuration + +Launch separately with ordinary supervisor Java dependencies; no QQQ application JAR is needed: + +```text +java -cp '/opt/qqq-compiler/lib/*' com.kingsrook.scripting.supervisor.CompilerMain /run/qqq-compiler/config.json +``` + +The absolute configuration file is at most64KiB with exactly these fields. Placeholders are not accepted qualification values: + +```json +{ + "bind": "127.0.0.1", "port": 18444, + "keyStore": "/run/qqq-compiler/server.p12", + "trustStore": "/run/qqq-compiler/clients.p12", + "certificateNamespaces": {"": ""}, + "profile": "", + "toolchain": "typescript-6.0.3-node-24", + "evidence": "", "requiredEvidence": "", + "nodePath": "/opt/node/bin/node", "nodeSha256": "", + "compilerRoot": "/opt/qqq-compiler/compiler", "manifestSha256": "", + "heapMiB": 256, "slots": 1, "connections": 2, + "cpuMillis": 10000, "wallMillis": 20000 +} +``` + +PKCS12 passwords use the existing `QQQ_SUPERVISOR_KEYSTORE_PASSWORD` and `QQQ_SUPERVISOR_TRUSTSTORE_PASSWORD` process secrets. They never enter the child environment. TLS1.3 requires a trusted client certificate, then exact leaf-DER digest to namespace mapping. No wildcard subjects or client-provided namespace authority. All HELLO profile/toolchain/evidence fields must match. Use a literal bind address: Java's cold DNS resolution is outside an interruptible socket deadline. + +The connections setting N (1..16) creates exactly N session workers and at most N waiting socket handoff positions: at most2N admitted sockets. A waiting socket has no parsed source or capability; its existing5s handshake deadline begins at acceptance and is never restarted when a worker becomes available. Excess sockets immediately close. This bounded handoff allows a sequential call to wait through the prior completed task tail without client retry. Child slots are separately bounded1..8; occupied child capacity still closes/refuses immediately, with no queued compiler work. Each accepted connection handles exactly one COMPILE and then closes. A client must not replay an uncertain sent request automatically. Already verified execution artifacts/readiness are unchanged by compiler availability. + +## Closed wire and actual retirement + +The unchanged outer `qbit-scripting-supervisor/1` uses four-byte length and strict UTF8 JSON (4MiB frame,8MiB transcript). HELLO/ACCEPT is the same certificate-authorized session handshake. COMPILE has exactly `protocol,type,session,sequence,requestId,request`, sequence1. `request` is the unchanged `qbit-scripting-worker/1` compile object: `protocol,type,requestId,toolchainId,entryPoint,sourceSha256,strict,files`. Strict must be Boolean true; source manifest/file counts/lengths are bounded before child creation and the compiler verifies canonical content identity/import/type policy. No capability, context or record values are accepted as extra envelope fields. + +The fixed child uses newline JSON, separate private protocol `qqq-typescript-child/1`: + +1. READY includes exact `protocol,type,nodeVersion,typescriptVersion,toolchainId`. Parent checks the pinned versions before sending any source. +2. Parent writes exactly one existing compile request. Child runs compile only and returns compileResult, remaining alive. +3. Parent samples actual child CPU, sends exact `{protocol,type:"RETIRE"}`, closes child stdin. +4. Child writes exact `{protocol,type:"EXITING"}` and exits. Parent requires actual exit0, stdout EOF, no stderr, and successful cleanup before returning. +5. Server validates exact result identity, strict/toolchain, bounded diagnostics, success bundle hash/map or failure-without-artifact. Only then does it send outer RESULT with `message`, followed by RETIRED with `retired:true`. + +No RESULT or RETIRED is issued when cleanup/accounting/identity is uncertain. Stderr is bounded8KiB and discarded without logging content; any nonempty stderr rejects. Oversized/partial/extra output rejects. Each admitted call owns a new child; there is no Node process reuse or author-code execution. Unsuccessful normal type checking may return a validated compileResult with `success:false` after ordinary retirement. + +## Limits and remaining platform qualification + +Fixed argv contains only the verified executable, heap flag, Node permission mode with the compiler root read allowance, `--no-addons`, `--no-warnings`, `--disable-proto=throw`, and the fixed entrypoint. No shell, PATH lookup, author argv/config, NODE_OPTIONS, application tokens or inherited environment. Child environment is LANG/TZ only, plus SystemRoot on Windows. Node permissions are defense in depth; **they are not a hostile-code containment claim**. The child never executes submitted JS, but parser/runtime vulnerabilities and resource exhaustion still need the qualified process/pod boundary. + +CPU uses the actual child `ProcessHandle.Info.totalCpuDuration`, sampled every2ms and at protocol boundaries. Missing, negative or decreasing live samples fail closed. Only after retirement has begun, a valid prior live sample exists and a fresh OS ProcessHandle check confirms actual death may missing/decreased post-exit accounting retain that prior sample; it never becomes a fabricated final usage value. Higher observed CPU remains subject to the same limit, and clean exit/EXITING/EOF are still mandatory. CPU settings1..30000ms are distinct from wall100..60000ms. No parent-thread CPU or synthetic fallback is substituted. The child intentionally stays alive after compilation so actual compile CPU can be sampled before retirement. OS accounting granularity, scheduled sampling/kill latency and the small post-sample EXITING/process teardown tail mean this is not exact final OS rusage. No compiler usage object or zero-CPU claim is manufactured. Measure/enforce an accepted overshoot bound on the target Linux image under contention before admission as a qualified compiler profile. + +Wall accounting begins before process creation and never resets for partial reads/writes. Once a Process exists, watchdog expiry forcibly terminates that exact owned child; cleanup waits up to2s and retains/refuses the slot if death is unknown. Java cannot interrupt an OS ProcessBuilder.start call that itself hangs before returning a Process; require a supervisor/pod startup/liveness deadline. Confirmed normal process exit is mandatory for RETIRED; a trusted child handshake alone is insufficient. + +**Node's heap flag is not a process RSS limit.** Deploy the separate compiler pod/process with a measured hard cgroup memory/pid/CPU allocation, finite connections/slots, readonly filesystem, no application credentials/preopens, and egress denied except explicitly needed operational infrastructure. Account for parent JSON buffers, pipes, Node native memory, compiler libraries and concurrent children; do not infer a safe aggregate RSS from heapMiB × slots. No compiler child should be able to escape the qualified cgroup. Hard memory enforcement, Node permission behavior, actual pinned compiler success/failure, child CPU support/overshoot, clean drain and the real main-provider cold endpoint still require target-platform tests owned by root. No Kubernetes deployment was performed. +## Root-run fixed actual compiler corpus + +After the exact Node worker build and packaging, compile the Java test driver without running it: + +```text +./mvnw -o -f supervisor/pom.xml -Dmaven.repo.local= -DskipTests test-compile +java -cp '/target/classes:/target/test-classes:' com.kingsrook.scripting.supervisor.NativeCompilerQualification +``` + +Use the platform's classpath separator. Only the compiled `NativeCompilerQualification.class` and `TlsIdentity.class` test helpers are needed besides production classes and org.json. The current JDK's keytool creates an ephemeral loopback certificate; the driver removes its own temporary credentials and listener afterwards. It does not need Extism/JNA, Wasm, QQQ, a shared port or an application credential. Fixed normal sources cover relative modules, SDK typing/TLA, strict type and forbidden ambient import diagnostics, then a later success. This proves actual compilation only after root runs it on the pinned image; it does not execute generated JavaScript, benchmark saving records, qualify a production namespace or claim cgroup isolation. The driver prints fixed counts and public artifact hashes only. diff --git a/supervisor/LICENSE b/supervisor/LICENSE new file mode 100644 index 0000000..4bb040e --- /dev/null +++ b/supervisor/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, source + code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 Kingsrook, LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/supervisor/NATIVE-QUALIFICATION.md b/supervisor/NATIVE-QUALIFICATION.md new file mode 100644 index 0000000..1961573 --- /dev/null +++ b/supervisor/NATIVE-QUALIFICATION.md @@ -0,0 +1,33 @@ +# Explicit trusted native qualification recipe + +This driver is excluded from Surefire discovery. It is not executed by `verify`, does not accept author submissions and cannot create a production evidence ID. Root owns its actual isolated Linux execution. Nothing below authorizes a host-runtime installation or deployment. + +The source driver is `src/test/java/com/kingsrook/scripting/supervisor/NativeSupervisorQualification.java`. Its only helper is `TlsIdentity.java`; all nested Counted/Client/anonymous lifecycle classes generated from the driver must be present. It uses runtime JNA/json dependencies only, not JUnit. `test-compile` with Checkstyle passed at2026-10-31T10:29:30-05:00; this proves compilation, not native execution. + +From the qbit-scripting root, with the already pinned Node24.22.2 binary and installed worker dependencies: + +```powershell +& node worker/node_modules/typescript/bin/tsc -p worker/tsconfig.json # exact Node 24.22.2 required +& node supervisor/fixtures/compile-trusted.mjs # exact Node 24.22.2 required +.\mvnw.cmd -o -f supervisor/pom.xml test-compile +``` + +The first command refreshes only the existing trusted compiler's JS output. The generator calls the actual worker strict compiler on four fixed synthetic TypeScript projects. It does not run emitted code. `target/native-fixtures` receives exact ESM bundles, source requests and a hash manifest. The Unicode fixture spans three bounded source files and intentionally emits over1MiB UTF8 while remaining under1Mi UTF16/2MiB UTF8. Do not replace these outputs with handwritten ESM while claiming compiler-path proof. + +Freeze and hash the module's main/test classes, JNA5.18.1 and json20251224 JARs, fixture directory, exact Extism1.41.0 native library and fixed sibling Wasm. Run only those snapshots in the bounded root-owned Linux container. Example paths inside that container: + +```text +java -cp '/qualification/classes:/qualification/test-classes:/qualification/lib/*' \ + com.kingsrook.scripting.supervisor.NativeSupervisorQualification \ + /qualification/native/libextism.so ACTUAL_NATIVE_SHA256 \ + /qualification/runtime/qqq_javy_runtime.wasm ACTUAL_WASM_SHA256 \ + /qualification/fixtures +``` + +Java21 plus keytool are required. The driver binds only an ephemeral loopback port and generates a short-lived test certificate in its private temporary directory; it never receives production keys or tokens. Permit that owned loopback communication while denying external egress. Provide bounded writable temporary space for ephemeral identity/JNA extraction, readonly snapshots for the exact libraries/runtime/fixtures, no host environment secrets, no guest preopens and finite outer CPU/memory/pids/time/output limits. Containerization itself is not qualification. + +Assertions cover actual TLS PREPARE→READY→RETIRED, original.ts/emitted.js identity, static relative modules/live binding/top-level await, awaited SDK broker, four fresh counter/time/random/context calls, Unicode boundary, generic script-error handling followed by a normal call, actual CPU/wall usage and counted native cleanup before every retirement. A mismatched memory profile must refuse before constructing a new instance. Separate direct calls prove that the host binding refuses a second call on one instance and that the same small normal preparation fails under fuel1; no infinite loop, allocation bomb or escape payload is used. + +The printed JSON contains only fixed scope, public binary hashes, counts, measured usage and `productionQualified:false`. It is a correctness corpus, **not a latency benchmark**. Random-value variation is evidence of this fixed repeated-state fixture, not cryptographic randomness qualification. Low-fuel refusal is not proof of complete CPU or native-callback accounting. Full containment/import inventory, quantitative cancellation/free overshoot, memory-pressure behavior, real application PersistentWorkerProvider interoperability, strict cold compiler service and Kubernetes deployment remain separate gates. + +On failure preserve the exact snapshot, full private test log and sanitized failure code; do not silently rerun with larger budgets or alter expected values. Successful test output must name the actual binary hashes and source snapshot. diff --git a/supervisor/NOTICE b/supervisor/NOTICE new file mode 100644 index 0000000..6fbe46b --- /dev/null +++ b/supervisor/NOTICE @@ -0,0 +1,6 @@ +QBit Scripting +Copyright 2026 Kingsrook, LLC + +This product includes software developed at Kingsrook, LLC (https://kingsrook.com/). + +Licensed under the Apache License, Version 2.0. diff --git a/supervisor/README.md b/supervisor/README.md new file mode 100644 index 0000000..b685755 --- /dev/null +++ b/supervisor/README.md @@ -0,0 +1,83 @@ +# External script supervisor + +This optional Java21 process implements `qbit-scripting-supervisor/1` for the existing persistent TLS WorkerProvider. It has no QQQ, QContext, storage, record permission or other QBit dependencies. Application authorization remains in the QQQ caller and its per-invocation broker. It is a transport/runtime candidate, **not a qualified Kubernetes deployment or a latency result**. + +The execution entrypoint `SupervisorMain` only constructs the pinned Extism/Javy binding. `GuestRuntime` permits trusted lifecycle fixtures; it is not selectable from operator JSON. That execution entrypoint has no Node process, arbitrary guest Wasm, runtime download or permissive fallback. The separate [cold compiler endpoint](COLD-COMPILER.md) uses a disposable pinned Node process solely for strict TypeScript compilation; emitted author JavaScript is never executed by it. + +## Build and run + +From the qbit-scripting root, using the existing wrapper and isolated dependency cache: + +```powershell +.\mvnw.cmd -o -f supervisor/pom.xml verify +``` + +Runtime Java dependencies are exactly JNA5.18.1 and org.json20251224. Keep their ordinary separate JARs with the supervisor JAR; do not replace the application QBit JAR or install this service into QQQ. With those three files in an operator-owned `lib` directory, the Linux launch is: + +```text +java -cp '/opt/qqq-supervisor/lib/*' com.kingsrook.scripting.supervisor.SupervisorMain /run/qqq-supervisor/config.json +``` + +For a nonroot process with a read-only root filesystem, explicitly set `HOME=/tmp` and `XDG_CACHE_HOME=/tmp/cache` to directories within the process's bounded, writable private temporary mount. Keep the runtime, native library, configuration and TLS stores read-only. These are supervisor startup directories, not guest filesystem grants; the guest manifest still has no allowed paths. Do not mount an operator's real home or cache. + +The October 31 local demo refused startup with `RUNTIME_COMPILE` and then started after those two environment settings were supplied. That observation establishes the working launch configuration; it does not identify a particular internal Extism cache operation or qualify Kubernetes containment. No guest execution or successful demo journey is inferred from startup alone. + +The configuration file must be absolute, at most64KiB, and contain exactly these fields (the angle-bracket strings are placeholders, not runnable qualification): + +```json +{ + "bind": "127.0.0.1", "port": 18443, + "keyStore": "/run/qqq-supervisor/server.p12", + "trustStore": "/run/qqq-supervisor/clients.p12", + "certificateNamespaces": {"": ""}, + "profile": "", "toolchain": "", + "evidence": "", "requiredEvidence": "", + "runtimePath": "/opt/qqq-supervisor/runtime/qqq_javy_runtime.wasm", + "runtimeSha256": "", + "nativePath": "/opt/qqq-supervisor/native/libextism.so", + "nativeSha256": "", + "slots": 1, "connections": 2, "coldOperations": 1, + "entries": 8, "codeBytes": 8388608, "memoryBytes": 67108864, + "idleMillis": 30000, "fuel": 100000000 +} +``` + +PKCS12 passwords are required process secrets `QQQ_SUPERVISOR_KEYSTORE_PASSWORD` and `QQQ_SUPERVISOR_TRUSTSTORE_PASSWORD`; neither belongs in configuration, logs or guest input. TLS1.3 requires a client certificate, then an exact leaf SHA256 mapping authorizes its one namespace. The client independently verifies server trust and hostname. Certificate rotation needs explicit updated fingerprints and reconnection/reprepare; there are no subject-name wildcards. Configuration, trust stores, binaries and signing/evidence policy remain trusted operator inputs. Equal evidence labels do not independently prove containment. + +## Wire, cache and lifecycle + +The closed outer wire is the existing [persistent transport contract](../docs/PERSISTENT-WORKER-TRANSPORT.md): four-byte big-endian length plus strict UTF8 JSON, at most4MiB per frame,8MiB across both directions per operation. Duplicate keys, coercion, unsupported fields, wrong session/sequence/request and mismatched capability tokens refuse the connection. Broker frames are separately limited to64KiB and50 calls. The server never retries a sent operation. + +### Additive cold heartbeat + +Failed or expired sessions abort socket input before closing TLS. A graceful TLS 1.3 close can otherwise drain unread peer input for the full idle timeout, retaining the sole clean engine while the client waits for a heartbeat. Teardown also uses zero linger; it does not wait for a peer close-notify before releasing an already-destroyed guest's compiled slot. This changes neither guest retirement proof nor quarantine rules. Startup failure attempts cleanup of every acquired engine, listener and executor, preserving the primary failure and suppressing distinct cleanup failures. + +The unpublished supervisor/1 contract also accepts `PING` on an authenticated idle single-flight connection. Its exact keys are `protocol,type,session,sequence,requestId`; all existing identity/order/deadline checks apply. It returns `PONG` with exactly those five envelope keys, then the ordinary `RETIRED` envelope with `retired:true`. No artifact identifier, inventory, source, record, token or other payload is accepted or returned. The handler creates no guest, compiles no code and neither reads nor modifies prepared entries. + +PONG plus RETIRED confirms that this same connection/session cache epoch remains intact. The implementation never silently clears/replaces a session cache; any future cache-epoch loss must close the session, not return PONG. This is liveness of that authenticated session, not a new authorization or proof that a requested publication is prepared. The caller maintains its existing per-session ready identities and sends heartbeat only when it has exclusively acquired an idle connection. Server processing is serial, so no heartbeat is processed concurrently with a guest. Normal successful PREPARE/INVOKE retirement may also refresh the caller's confirmed-session age. This cold maintenance feature does not add preparation, reconnection or retries to hot execute. + +After certificate/HELLO admission, one warm TLS session owns one serial compiled-runtime slot. The slot's common Wasm is compiled at cold service startup, never in INVOKE. A bounded session cache holds only prepared bytecode and immutable digest/entry/profile identity. PREPARE accepts the existing `qqq-esm/1` bundle (at most1Mi UTF16 code units AND2MiB UTF8), verifies its digest and invokes cold Javy bytecode preparation. This is **not strict TypeScript compilation**. Outer COMPILE is intentionally refused; a separately qualified compiler endpoint must supply actual strict-TypeScript builds. Compilation must not consume interactive execution slots. + +PREPARE/READY retain the source `.ts` entrypoint. The private prepared result must contain its exact `.js` conversion, sorted unique portable module paths, at most200 modules/2MiB decoded bytecode, canonical base64 and the peer runtime's domain-separated digest. Only the compiler-produced cached object can enter RUN; client-supplied bytecode is never accepted. Entry-count and serialized-byte caps refuse additional preparation without evicting existing code. Cache fullness closes the session; subsequent cold rewarming is explicit. + +INVOKE requires an already prepared readyId and matching artifact/entrypoint. It neither connects, compiles, uploads the bundle nor fetches missing code. Each prepare/run creates a fresh Extism Plugin/Store, runs the fixed `run` export once, and actually frees that instance before READY/RESULT and RETIRED. Prepared bytes survive in trusted Java memory; guest globals, context, tokens and pending jobs do not. Any failed operation discards its connection and prepared-code cache; it is never replayed. A recoverable I/O or malformed-JSON failure may return only the common compiled engine to admission after the fresh guest was actually destroyed (or before any guest construction began). This includes bounded invocation refusal and failure to deliver a response after cleanup. A new session must authenticate and prepare its own cache again; initialized guests are never reused. Cleanup failure, fatal Error, unexpected runtime failure or uncertain construction quarantines that engine until service restart. Suppressed cleanup failures also prevent reuse. When every configured engine is quarantined, the listener closes and the service loop fails with ENGINE_CAPACITY_EXHAUSTED; SupervisorMain exits nonzero through its existing failure path instead of remaining live with zero capacity. No reset, replacement engine allocation or speculative reuse occurs. Current QQQ authority is rechecked on each caller invocation; readiness grants no permission. + +The sole application import is `extism:host/user::qqq_broker(i64)->i64`. Java validates the current invocation thread and Extism allocation length, relays a query through the current TLS session, and returns a fresh allocation. The fixed Wasm plugin frees both request and response blocks. Java never returns the borrowed request block or retains broker authority in compiled code. Callback failures cannot be caught by author code and transformed into a successful supervisor result. + +## Budgets and retirement limits + +Admission has no unbounded queue:1–64 engine slots, at most128 connection threads,1–8 concurrent cold preparations. Each session cache has at most256 entries/64MiB serialized code. These are ceilings, not deployment recommendations; native engine/Java heap and concurrent frame copies require host capacity measurements. Invocation memory must exactly match the configured64–256MiB profile. Arbitrary lower per-request memory is refused rather than silently rounded up. + +CPU usage is actual supported/enabled `ThreadMXBean` time for the calling invocation thread, including native Wasm execution on that thread, host callback CPU and construction/free. Unsupported/disabled/unavailable accounting refuses startup/execution; there is no fabricated zero or wall-as-CPU substitution. CPU and wall are sampled every1ms, checked at broker/return boundaries and after actual cleanup. Guest wall hints are ignored. Broker waiting counts toward wall, not CPU. A finite Extism fuel budget and10s native timeout supplement these checks. + +**One-millisecond scheduling is not a proven one-millisecond enforcement bound.** OS scheduling, JNA/native cancellation and cleanup can overshoot. Native destruction runs under the cancellation lock to prevent use-after-free; a hung native free cannot be safely interrupted by Java. Exact Linux tests must establish cancellation/free behavior, target-thread native CPU accounting, fuel and overshoot under contention. A containing pod/process budget and termination policy remain required; this implementation does not attest hostile-runtime safety from a successful timer callback. + +TLS handshake/HELLO has a5s whole-exchange deadline. Each complete session operation, including partial input and final writes, has an idleMillis+15s outer socket deadline; requested guest wall remains at most10s. Per-operation deadline objects serialize expiry with retirement, so a dequeued old timer cannot close a subsequent operation. Startup bind-address DNS resolution and native common-runtime compilation are cold startup work without a Java hard deadline; use a literal bind address and an operator startup timeout. + +Shutdown stops new admission, closes owned sockets, and waits up to15s for invocation cleanup. It frees common engines only after session threads have terminated. Incomplete drain refuses rather than issuing retirement or freeing active native state; the orchestrator must then terminate the process. This is bounded shutdown waiting, not a promise of graceful completion of every in-flight call. + +## Evidence and remaining qualification + +`trusted-lifecycle-full.log` records the initial10-test standalone verify: real ephemeral mutual TLS, exact namespace refusal, sequential fresh-instance calls, bounded cache/slot admission, broker authority refusal, measured thread-CPU cancellation, cleanup/error retirement, strict framing and stale timer isolation. The runtime fixture never evaluates source; these are transport/lifecycle tests, not Javy or Kubernetes qualification. Historical evidence files were not carried into this standalone source tree. + +Required next evidence: exact fixed-Wasm+Extism native execution through this binding; actual prepared-module/Promise/broker semantics; one-call/fresh-state proof; bounded host callback allocation/free; CPU/fuel/memory/timeout/drain and overshoot; complete WASI import/environment/filesystem/network inventory; actual production PersistentWorkerProvider interoperability; qualified cold strict-TypeScript endpoint; deployment certificate/namespace/NetworkPolicy/cgroup capacity and full record-save latency. No production image, cluster installation, author-code run or latency benchmark is implied by this source package. diff --git a/supervisor/SOURCES.md b/supervisor/SOURCES.md new file mode 100644 index 0000000..4d3bc2d --- /dev/null +++ b/supervisor/SOURCES.md @@ -0,0 +1,15 @@ +# Public native ABI provenance + +The adapter uses the public C API of Extism1.41.0, not a copied SDK implementation, reflection or private symbol access. Root retrieved the following official versioned files; the local copies were read to check allocation/function ownership: + +| Source | SHA256 | +|---|---| +| [Extism header](https://github.com/extism/extism/blob/v1.41.0/runtime/extism.h) | `69A790998BC21A3BBDCBB02DC7D1C4862B62EDAB60B98139831EACEA21E5CA1E` | +| [runtime/src/sdk.rs](https://github.com/extism/extism/blob/v1.41.0/runtime/src/sdk.rs) | `C6AF172AE8F7DE438EB98453E578FEF1894B90D3E3697F254A2D140D4679EE2A` | +| [runtime/src/function.rs](https://github.com/extism/extism/blob/v1.41.0/runtime/src/function.rs) | `AC8DA752796E41D3C0283D169054C17868DB4299F951C0825955620425E3B40A` | + +Extism's compiled-plugin constructor takes the Rust Function value from each passed C wrapper; the wrapper still requires `extism_function_free`. The Java slot retains its JNA callback strongly, frees the common compiled plugin after its active instance is gone, then frees the wrapper exactly once. Constructor error strings are freed without copying/logging their text. This source reasoning needs actual native qualification; it is not evidence that a different binary honors the same ABI. + +The pinned fixed guest is the sibling [wasm-runtime](../wasm-runtime/README.md), with its independently maintained Cargo.lock, source/licensing provenance and private JSON ABI. The supervisor checks actual native/Wasm file hashes and Extism's reported version. Operator hash selection and secure packaging remain trust prerequisites. + +Java dependency pins: JNA5.18.1 (Apache2.0/LGPL2.1 dual licensed upstream), org.json20251224 (upstream public-domain project declaration), and test-only JUnit6.0.1. Preserve upstream JAR metadata/notices when distributing their separate binaries. This source module's own Apache2.0 LICENSE/NOTICE are packaged under META-INF. No dependency was downloaded or globally installed as part of this tranche. diff --git a/supervisor/fixtures/build-compiler-image.mjs b/supervisor/fixtures/build-compiler-image.mjs new file mode 100644 index 0000000..ad50c67 --- /dev/null +++ b/supervisor/fixtures/build-compiler-image.mjs @@ -0,0 +1,54 @@ +// Copyright (C) 2026 Kingsrook, LLC. +// SPDX-License-Identifier: Apache-2.0 +import { createHash } from "node:crypto"; +import { copyFile, lstat, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +// Packaging only. Never imports the compiler, executes source or installs packages. +// Root runs this with the approved exact Node executable after worker npm run build. +if (process.versions.node !== "24.22.2" || process.argv.length !== 4) { + throw new Error("Use Node 24.22.2: build-compiler-image.mjs "); +} +const worker = process.argv[2]; +const output = process.argv[3]; +if (!path.isAbsolute(worker) || !path.isAbsolute(output) || output.includes("*") || output.includes(",")) { + throw new Error("Absolute literal paths required"); +} +const tsPackage = JSON.parse(await readFile(path.join(worker, "node_modules/typescript/package.json"), "utf8")); +if (tsPackage.version !== "6.0.3") throw new Error("TypeScript 6.0.3 required"); +const libs = (await readdir(path.join(worker, "node_modules/typescript/lib"))) + .filter(name => /^lib\.[a-z0-9.]+\.d\.ts$/.test(name)) + .map(name => "node_modules/typescript/lib/" + name); +const files = ["package.json", "dist/compiler-entry.js", "dist/compiler.js", "dist/frames.js", "dist/protocol.js", + "node_modules/typescript/package.json", "node_modules/typescript/lib/typescript.js", ...libs].sort(); +if (files.length > 256 || libs.length === 0) throw new Error("Compiler closure size"); +await mkdir(output); // Deliberately refuses any existing output; never overwrites a sealed image. +const root = path.join(output, "compiler"); +await mkdir(root); +let total = 0; +const hashes = {}; +for (const name of files) { + const source = path.join(worker, name); + const stat = await lstat(source); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size < 1 || stat.size > 16 * 1024 * 1024) throw new Error("Invalid compiler source file"); + total += stat.size; + if (total > 64 * 1024 * 1024) throw new Error("Compiler closure bytes"); + const content = await readFile(source); + hashes[name] = createHash("sha256").update(content).digest("hex"); + const target = path.join(root, name); + await mkdir(path.dirname(target), { recursive: true }); + await writeFile(target, content, { flag: "wx" }); +} +const manifest = JSON.stringify({ protocol: "qqq-typescript-image/1", nodeVersion: "24.22.2", typescriptVersion: "6.0.3", files: hashes }, null, 2) + "\n"; +await writeFile(path.join(root, "compiler-manifest.json"), manifest, { flag: "wx" }); +// Notices are adjacent to, never extra executable files inside, the closed tree. +const notices = path.join(output, "notices"); +await mkdir(notices); +await copyFile(path.join(worker, "node_modules/typescript/LICENSE.txt"), path.join(notices, "typescript-LICENSE.txt")); +await copyFile(path.join(worker, "LICENSE"), path.join(notices, "worker-LICENSE")); +await copyFile(path.join(worker, "NOTICE"), path.join(notices, "worker-NOTICE")); +const nodeHash = createHash("sha256").update(await readFile(process.execPath)).digest("hex"); +const evidence = { nodePath: process.execPath, nodeSha256: nodeHash, compilerRoot: root, + manifestSha256: createHash("sha256").update(manifest).digest("hex"), files: files.length, bytes: total }; +await writeFile(path.join(output, "compiler-image.json"), JSON.stringify(evidence, null, 2) + "\n", { flag: "wx" }); +console.log(JSON.stringify(evidence)); \ No newline at end of file diff --git a/supervisor/fixtures/compile-trusted.mjs b/supervisor/fixtures/compile-trusted.mjs new file mode 100644 index 0000000..5c686b5 --- /dev/null +++ b/supervisor/fixtures/compile-trusted.mjs @@ -0,0 +1,58 @@ +// Copyright (C) 2026 Kingsrook, LLC. +// SPDX-License-Identifier: Apache-2.0 +// Fixed trusted qualification corpus only. This is not a production compiler route. +import { mkdir, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { compile } from "../../worker/dist/compiler.js"; +import { PROTOCOL, TOOLCHAIN, sourceHash } from "../../worker/dist/protocol.js"; + +if(process.versions.node !== "24.22.2") +{ + throw new Error("QUALIFICATION_NODE_VERSION"); +} + +const destination = new URL("../target/native-fixtures/", import.meta.url); +await mkdir(destination, { recursive: true }); +const sdk = 'import type { Context, Qqq } from "@qqq/sdk";\n'; +const cases = [ + { name: "state", files: [ + { path: "main.ts", content: sdk + `import { bump, stamp, random } from './state.js'; +await Promise.resolve(); +export async function main(context: Context, qqq: Qqq) { + const rows = await qqq.query('purchaseOrder', ['id'], [], 1); + return { count: bump(), stamp, random, id: rows[0]?.id, actor: context.payload.actor }; +}` }, + { path: "state.ts", content: "let count = 0; export function bump() { return ++count; } export const stamp = Date.now(); export const random = Math.random();" }, + ] }, + { name: "unicode", files: [ + { path: "main.ts", content: "import { a } from './a.js'; import { b } from './b.js'; import { c } from './c.js'; export async function main() { return { label: a + b + c }; }" }, + ...["a", "b", "c"].map((name, index) => ({ path: `${name}.ts`, content: `/*${"é".repeat(180000)}*/\nexport const ${name} = ${JSON.stringify(["café ", "漢字 ", "Привет"][index])};` })), + ] }, + { name: "plain", files: [{ path: "main.ts", content: "export async function main() { return { label: 'plain' }; }" }] }, + { name: "script-error", files: [{ path: "main.ts", content: "export async function main() { throw new Error('fixed trusted fixture'); }" }] }, +]; + +const hashes = []; +for(const fixture of cases) +{ + const request = { protocol: PROTOCOL, type: "compile", requestId: `trusted-${fixture.name}`, strict: true, + toolchainId: TOOLCHAIN, entryPoint: "main.ts", files: fixture.files, sourceSha256: sourceHash(fixture.files) }; + const result = compile(request); + if(result.success !== true || typeof result.bundle !== "string") + { + throw new Error(`TRUSTED_COMPILE_REFUSED:${fixture.name}`); + } + const record = { name: fixture.name, entryPoint: "main.ts", sourceSha256: request.sourceSha256, + toolchainId: TOOLCHAIN, bundle: result.bundle, sha256: createHash("sha256").update(result.bundle).digest("hex") }; + if(fixture.name === "unicode" && Buffer.byteLength(record.bundle) <= 1024 * 1024) + { + throw new Error("UNICODE_BOUNDARY_NOT_EXERCISED"); + } + const serialized = JSON.stringify(record); + await writeFile(new URL(`${fixture.name}.json`, destination), serialized, "utf8"); + await writeFile(new URL(`${fixture.name}.sources.json`, destination), JSON.stringify(request), "utf8"); + hashes.push({ name: fixture.name, sha256: createHash("sha256").update(serialized).digest("hex"), artifactSha256: record.sha256 }); +} +await writeFile(new URL("manifest.json", destination), JSON.stringify({ scope: "fixed trusted TypeScript compiler fixtures", node: process.versions.node, toolchain: TOOLCHAIN, files: hashes }, null, 2), "utf8"); +process.stdout.write(`Compiled ${cases.length} fixed trusted fixtures into ${fileURLToPath(destination)}\n`); diff --git a/supervisor/pom.xml b/supervisor/pom.xml new file mode 100644 index 0000000..96e1fe0 --- /dev/null +++ b/supervisor/pom.xml @@ -0,0 +1,26 @@ + + 4.0.0 + com.kingsrook.scriptingqqq-script-supervisor0.1.0-SNAPSHOT + External QQQ script supervisor + 21UTF-8 + + net.java.dev.jnajna5.18.1 + org.jsonjson20251224 + org.junit.jupiterjunit-jupiter6.0.1test + + + org.apache.maven.pluginsmaven-resources-plugin3.3.1 + own-license-noticeprocess-resourcescopy-resources + ${project.build.outputDirectory}/META-INF${project.basedir}falseLICENSENOTICE + + + org.apache.maven.pluginsmaven-compiler-plugin3.14.1 + org.apache.maven.pluginsmaven-surefire-plugin3.5.4 + org.apache.maven.pluginsmaven-jar-plugin3.4.2com.kingsrook.scripting.supervisor.SupervisorMain + org.apache.maven.pluginsmaven-checkstyle-plugin3.3.1 + com.puppycrawl.toolscheckstyle10.12.6 + ${maven.multiModuleProjectDirectory}/checkstyle/config.xmlcheckstyle.reactor.root=${maven.multiModuleProjectDirectory}truetruewarningtrue + validatecheck + + + diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/ColdCompiler.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/ColdCompiler.java new file mode 100644 index 0000000..5e1a79b --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/ColdCompiler.java @@ -0,0 +1,41 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import org.json.JSONObject; + +/******************************************************************************* + ** Separate cold compiler boundary. Return only after actual child termination; + ** this seam permits trusted protocol tests, not a production fallback. + *******************************************************************************/ +public interface ColdCompiler extends AutoCloseable +{ + /*************************************************************************** + ** Compile once without a broker or any execution of emitted author code. + ***************************************************************************/ + JSONObject compile(JSONObject request) throws Exception; + + /*************************************************************************** + ** Stop admission and terminate owned children before releasing the service. + ***************************************************************************/ + @Override + void close() throws Exception; +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerImage.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerImage.java new file mode 100644 index 0000000..0ab5c39 --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerImage.java @@ -0,0 +1,183 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; +import org.json.JSONObject; + +/******************************************************************************* + ** Hash-bound minimal compiler tree and exact executable, owned by the operator. + ** Readonly deployment mounts remain necessary to prevent post-check replacement. + *******************************************************************************/ +public record CompilerImage(Path node, String nodeSha256, Path root, String manifestSha256, int heapMiB) +{ + private static final Set REQUIRED = Set.of("package.json", "dist/compiler-entry.js", "dist/compiler.js", "dist/frames.js", "dist/protocol.js", + "node_modules/typescript/package.json", "node_modules/typescript/lib/typescript.js"); + + /*************************************************************************** + ** No executable lookup, user-selected flags, ambient config or source plugins. + ***************************************************************************/ + public CompilerImage + { + if(node == null || root == null || !node.isAbsolute() || !root.isAbsolute() || nodeSha256 == null || manifestSha256 == null + || !nodeSha256.matches("[a-f0-9]{64}") || !manifestSha256.matches("[a-f0-9]{64}") || heapMiB < 64 || heapMiB > 512 + || root.toString().contains("*") || root.toString().contains(",")) + { + throw new IllegalArgumentException("COMPILER_IMAGE"); + } + node = node.normalize(); + root = root.normalize(); + } + + /*************************************************************************** + ** Verify the exact small dependency closure and reject extra files/symlinks. + ***************************************************************************/ + public void verify() throws Exception + { + if(Files.isSymbolicLink(node) || Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS) + || !hashFile(node, 256L * 1024 * 1024).equals(nodeSha256)) + { + throw new IOException("COMPILER_IMAGE_IDENTITY"); + } + Path manifest = root.resolve("compiler-manifest.json"); + if(!hashFile(manifest, 128 * 1024).equals(manifestSha256)) + { + throw new IOException("COMPILER_MANIFEST_IDENTITY"); + } + JSONObject value = Protocol.parse(Files.readAllBytes(manifest)); + Protocol.keys(value, "protocol", "nodeVersion", "typescriptVersion", "files"); + Protocol.equal(value, "protocol", "qqq-typescript-image/1"); + Protocol.equal(value, "nodeVersion", "24.22.2"); + Protocol.equal(value, "typescriptVersion", "6.0.3"); + JSONObject files = value.getJSONObject("files"); + if(files.length() > 256 || !files.keySet().containsAll(REQUIRED)) + { + throw new IOException("COMPILER_IMAGE_FILES"); + } + long total = 0; + for(String name : files.keySet()) + { + if(!REQUIRED.contains(name) && !name.matches("node_modules/typescript/lib/lib\\.[a-z0-9.]+\\.d\\.ts")) + { + throw new IOException("COMPILER_IMAGE_PATH"); + } + Path file = root.resolve(name); + if(!file.normalize().startsWith(root) || !hashFile(file, 16L * 1024 * 1024).equals(files.getString(name))) + { + throw new IOException("COMPILER_IMAGE_FILE_IDENTITY"); + } + total += Files.size(file); + if(total > 64L * 1024 * 1024) + { + throw new IOException("COMPILER_IMAGE_BYTES"); + } + } + Set found = new HashSet<>(); + try(var paths = Files.walk(root, 5)) + { + List inventory = paths.limit(513).toList(); + if(inventory.size() > 512) + { + throw new IOException("COMPILER_IMAGE_FILES"); + } + for(Path file : inventory) + { + if(Files.isSymbolicLink(file)) + { + throw new IOException("COMPILER_IMAGE_SYMLINK"); + } + if(Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) + { + found.add(root.relativize(file).toString().replace('\\', '/')); + } + else if(!Files.isDirectory(file, LinkOption.NOFOLLOW_LINKS)) + { + throw new IOException("COMPILER_IMAGE_FILE_TYPE"); + } + else if(!Set.of("", "dist", "node_modules", "node_modules/typescript", "node_modules/typescript/lib") + .contains(root.relativize(file).toString().replace('\\', '/'))) + { + throw new IOException("COMPILER_IMAGE_DIRECTORY"); + } + } + } + Set expected = new HashSet<>(files.keySet()); + expected.add("compiler-manifest.json"); + if(!found.equals(expected)) + { + throw new IOException("COMPILER_IMAGE_EXTRA_FILES"); + } + } + + /*************************************************************************** + ** Closed fixed argv: heap is supplemental, never a process-RSS guarantee. + ***************************************************************************/ + List command() + { + List command = new ArrayList<>(); + command.add(node.toString()); + command.add("--max-old-space-size=" + heapMiB); + command.add("--permission"); + command.add("--allow-fs-read=" + root); + command.add("--no-addons"); + command.add("--no-warnings"); + command.add("--disable-proto=throw"); + command.add(root.resolve("dist/compiler-entry.js").toString()); + return (List.copyOf(command)); + } + + /*************************************************************************** + ** Stream bounded operator files instead of allocating executable-sized arrays. + ***************************************************************************/ + static String hashFile(Path file, long maximum) throws Exception + { + if(!Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) || Files.size(file) < 1 || Files.size(file) > maximum) + { + throw new IOException("COMPILER_FILE_SIZE"); + } + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + long count = 0; + try(var input = Files.newInputStream(file)) + { + byte[] buffer = new byte[8192]; + int length; + while((length = input.read(buffer)) != -1) + { + count += length; + if(count > maximum) + { + throw new IOException("COMPILER_FILE_SIZE"); + } + digest.update(buffer, 0, length); + } + } + return (HexFormat.of().formatHex(digest.digest())); + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerMain.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerMain.java new file mode 100644 index 0000000..ebd6aed --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerMain.java @@ -0,0 +1,111 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.json.JSONObject; + +/******************************************************************************* + ** Explicit separate cold endpoint. There is no compiler fallback in Wasm execute. + *******************************************************************************/ +public final class CompilerMain +{ + /*************************************************************************** + ** Utility entrypoint. + ***************************************************************************/ + private CompilerMain() {} + + /*************************************************************************** + ** One bounded operator configuration; no generic commands or author arguments. + ***************************************************************************/ + public static void main(String[] args) + { + try + { + if(args.length != 1) + { + throw new IOException("COMPILER_CONFIG_REQUIRED"); + } + Path path = Path.of(args[0]); + if(!path.isAbsolute() || !Files.isRegularFile(path) || Files.size(path) > 64 * 1024) + { + throw new IOException("COMPILER_CONFIG_FILE"); + } + JSONObject config = Protocol.parse(Files.readAllBytes(path)); + Protocol.keys(config, "bind", "port", "keyStore", "trustStore", "certificateNamespaces", "profile", "toolchain", "evidence", "requiredEvidence", + "nodePath", "nodeSha256", "compilerRoot", "manifestSha256", "heapMiB", "slots", "connections", "cpuMillis", "wallMillis"); + Map certificates = new HashMap<>(); + JSONObject mappings = config.getJSONObject("certificateNamespaces"); + for(String fingerprint : mappings.keySet()) + { + certificates.put(fingerprint, mappings.getString(fingerprint)); + } + int wall = (int) Protocol.number(config, "wallMillis", 100, 60_000); + var settings = new CompilerServer.Configuration(SupervisorMain.tls(Path.of(config.getString("keyStore")), Path.of(config.getString("trustStore"))), + new InetSocketAddress(config.getString("bind"), (int) Protocol.number(config, "port", 1, 65535)), certificates, + config.getString("profile"), config.getString("toolchain"), config.getString("evidence"), config.getString("requiredEvidence"), + (int) Protocol.number(config, "connections", 1, 16), wall); + var image = new CompilerImage(Path.of(config.getString("nodePath")), config.getString("nodeSha256"), Path.of(config.getString("compilerRoot")), + config.getString("manifestSha256"), (int) Protocol.number(config, "heapMiB", 64, 512)); + try(var compiler = new NodeColdCompiler(image, (int) Protocol.number(config, "slots", 1, 8), + (int) Protocol.number(config, "cpuMillis", 1, 30_000), wall); var server = new CompilerServer(settings, compiler)) + { + Thread shutdown = new Thread(() -> + { + try + { + server.close(); + } + catch(Exception ignored) + { + /* Operator process termination owns unresolved cleanup. */ + } + }, "script-cold-compiler-shutdown"); + Runtime.getRuntime().addShutdownHook(shutdown); + try + { + server.serve(); + } + finally + { + try + { + Runtime.getRuntime().removeShutdownHook(shutdown); + } + catch(IllegalStateException ignored) + { + /* Shutdown is already underway. */ + } + } + } + } + catch(Exception failure) + { + System.err.println("COLD_COMPILER_UNAVAILABLE"); + System.exit(1); + } + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerServer.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerServer.java new file mode 100644 index 0000000..86c19f5 --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/CompilerServer.java @@ -0,0 +1,412 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLSocket; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.json.JSONObject; + +/******************************************************************************* + ** Separate compile-only authenticated endpoint; it owns no Wasm execution slots, + ** ready cache or broker. One child-confirmed result per short-lived connection. + *******************************************************************************/ +public final class CompilerServer implements AutoCloseable +{ + private final Configuration config; + private final ColdCompiler compiler; + private final Runnable sessionTail; + private final SSLServerSocket listener; + private final ThreadPoolExecutor clients; + private final ScheduledThreadPoolExecutor deadlines = new ScheduledThreadPoolExecutor(1); + private final Set sockets = ConcurrentHashMap.newKeySet(); + private final AtomicBoolean closed = new AtomicBoolean(); + + /*************************************************************************** + ** Exact identity/profile and finite capacity; qualification is operator-owned. + ***************************************************************************/ + public record Configuration(SSLContext tls, InetSocketAddress address, Map certificateNamespaces, + String profile, String toolchain, String evidence, String requiredEvidence, int connections, int wallMillis) + { + /*********************************************************************** + ** Refuse missing identity or unbounded admission before binding a listener. + ***********************************************************************/ + public Configuration + { + certificateNamespaces = Map.copyOf(certificateNamespaces); + if(tls == null || address == null || certificateNamespaces.isEmpty() || certificateNamespaces.size() > 256 + || !id(profile) || !"typescript-6.0.3-node-24".equals(toolchain) || !id(evidence) || !evidence.equals(requiredEvidence) + || connections < 1 || connections > 16 || wallMillis < 100 || wallMillis > 60_000) + { + throw new IllegalArgumentException("COMPILER_SERVER_CONFIGURATION"); + } + for(var entry : certificateNamespaces.entrySet()) + { + if(!entry.getKey().matches("[a-f0-9]{64}") || !id(entry.getValue())) + { + throw new IllegalArgumentException("COMPILER_CERTIFICATE_NAMESPACE"); + } + } + } + } + + /*************************************************************************** + ** Fixed workers and bounded socket handoff, without queued parsed source data. + ***************************************************************************/ + public CompilerServer(Configuration config, ColdCompiler compiler) throws IOException + { + this(config, compiler, () -> {}); + } + + /*************************************************************************** + ** Package-only deterministic lifecycle fixture; never operator configurable. + ***************************************************************************/ + CompilerServer(Configuration config, ColdCompiler compiler, Runnable sessionTail) throws IOException + { + this.config = config; + this.compiler = compiler; + this.sessionTail = sessionTail; + clients = new ThreadPoolExecutor(config.connections(), config.connections(), 30, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(config.connections()), runnable -> + { + Thread thread = new Thread(runnable, "script-cold-compiler-session"); + thread.setDaemon(true); + return (thread); + }); + deadlines.setRemoveOnCancelPolicy(true); + deadlines.setThreadFactory(runnable -> Thread.ofPlatform().daemon().name("script-cold-compiler-deadline").unstarted(runnable)); + listener = (SSLServerSocket) config.tls().getServerSocketFactory().createServerSocket(); + listener.setNeedClientAuth(true); + listener.setEnabledProtocols(new String[] {"TLSv1.3"}); + listener.bind(config.address()); + } + + /*************************************************************************** + ** Bound endpoint for trusted host orchestration only. + ***************************************************************************/ + public int port() + { + return (listener.getLocalPort()); + } + + /*************************************************************************** + ** At most N working and N waiting sockets; waiting consumes handshake time. + ***************************************************************************/ + public void serve() throws IOException + { + while(!closed.get()) + { + try + { + SSLSocket socket = (SSLSocket) listener.accept(); + admit(socket); + } + catch(IOException failure) + { + if(!closed.get()) + { + throw failure; + } + } + } + } + + /*************************************************************************** + ** Serialize accepting ownership with shutdown; rejected sockets are closed now. + ***************************************************************************/ + private synchronized void admit(SSLSocket socket) + { + if(closed.get()) + { + disconnect(socket); + return; + } + sockets.add(socket); + SocketDeadline handshake = null; + try + { + handshake = new SocketDeadline(deadlines, 5000, () -> disconnect(socket)); + SocketDeadline accepted = handshake; + clients.execute(() -> session(socket, accepted)); + } + catch(RuntimeException full) + { + if(handshake != null) + { + handshake.close(); + } + sockets.remove(socket); + disconnect(socket); + } + } + + /*************************************************************************** + ** Same outer contract, COMPILE only; no broker or execution/data capability. + ***************************************************************************/ + private void session(SSLSocket socket, SocketDeadline acceptedHandshake) + { + try(socket; acceptedHandshake) + { + socket.setTcpNoDelay(true); + socket.setSoTimeout(5000); + Protocol wire = new Protocol(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + String session; + try(var handshake = acceptedHandshake) + { + socket.startHandshake(); + String namespace = config.certificateNamespaces().get(Protocol.hash(socket.getSession().getPeerCertificates()[0].getEncoded())); + if(namespace == null) + { + throw new IOException("COMPILER_CERTIFICATE_NAMESPACE"); + } + JSONObject hello = wire.read(input); + Protocol.keys(hello, "protocol", "type", "namespace", "profile", "toolchain", "evidence", "nonce"); + Protocol.equal(hello, "protocol", Protocol.VERSION); + Protocol.equal(hello, "type", "HELLO"); + Protocol.equal(hello, "namespace", namespace); + Protocol.equal(hello, "profile", config.profile()); + Protocol.equal(hello, "toolchain", config.toolchain()); + Protocol.equal(hello, "evidence", config.evidence()); + Protocol.id(hello, "nonce"); + session = UUID.randomUUID().toString(); + wire.write(output, new JSONObject(hello.toString()).put("type", "ACCEPT").put("session", session)); + } + try(var exchange = new SocketDeadline(deadlines, config.wallMillis() + 10_000, () -> disconnect(socket))) + { + wire.reset(); + JSONObject frame = wire.read(input); + Protocol.keys(frame, "protocol", "type", "session", "sequence", "requestId", "request"); + Protocol.equal(frame, "protocol", Protocol.VERSION); + Protocol.equal(frame, "type", "COMPILE"); + Protocol.equal(frame, "session", session); + Protocol.number(frame, "sequence", 1, 1); + String requestId = Protocol.id(frame, "requestId"); + JSONObject request = frame.getJSONObject("request"); + validateRequest(request, requestId, config.toolchain()); + JSONObject result = compiler.compile(request); + validateResult(result, requestId, config.toolchain()); + JSONObject envelope = new JSONObject().put("protocol", Protocol.VERSION).put("type", "RESULT").put("session", session) + .put("sequence", 1).put("requestId", requestId).put("message", result); + wire.write(output, envelope); + envelope.remove("message"); + wire.write(output, envelope.put("type", "RETIRED").put("retired", true)); + } + } + catch(Throwable failure) + { + // No compiler/source/token/native exception text is logged or forwarded. + } + finally + { + sockets.remove(socket); + sessionTail.run(); + } + } + + /*************************************************************************** + ** Cheap closed admission before child creation; compiler owns TS semantics. + ***************************************************************************/ + static void validateRequest(JSONObject request, String id, String toolchain) throws Exception + { + Protocol.keys(request, "protocol", "type", "requestId", "toolchainId", "entryPoint", "sourceSha256", "strict", "files"); + Protocol.equal(request, "protocol", Protocol.WORKER); + Protocol.equal(request, "type", "compile"); + Protocol.equal(request, "requestId", id); + Protocol.equal(request, "toolchainId", toolchain); + if(!Boolean.TRUE.equals(request.opt("strict")) || !request.getString("sourceSha256").matches("[a-f0-9]{64}") + || request.getJSONArray("files").isEmpty() || request.getJSONArray("files").length() > 200) + { + throw new IOException("COMPILER_REQUEST"); + } + long bytes = 0; + for(Object item : request.getJSONArray("files")) + { + if(!(item instanceof JSONObject file)) + { + throw new IOException("COMPILER_SOURCE"); + } + Protocol.keys(file, "path", "content"); + String content = file.getString("content"); + if(content.length() > 262_144 || file.getString("path").length() > 200) + { + throw new IOException("COMPILER_SOURCE"); + } + bytes += content.getBytes(StandardCharsets.UTF_8).length; + } + if(bytes > 2 * 1024 * 1024) + { + throw new IOException("COMPILER_SOURCE_BYTES"); + } + } + + /*************************************************************************** + ** A successful result requires strict/pinned identity and exact bundle hash. + ***************************************************************************/ + static void validateResult(JSONObject result, String id, String toolchain) throws Exception + { + Set allowed = Set.of("protocol", "type", "requestId", "strict", "toolchainId", "success", "diagnostics", "bundle", "bundleSha256", "sourceMap"); + if(!allowed.containsAll(result.keySet()) || !(result.opt("success") instanceof Boolean) || !Boolean.TRUE.equals(result.opt("strict"))) + { + throw new IOException("COMPILER_RESULT"); + } + Protocol.equal(result, "protocol", Protocol.WORKER); + Protocol.equal(result, "type", "compileResult"); + Protocol.equal(result, "requestId", id); + Protocol.equal(result, "toolchainId", toolchain); + if(result.getJSONArray("diagnostics").length() > 100) + { + throw new IOException("COMPILER_DIAGNOSTICS"); + } + for(Object item : result.getJSONArray("diagnostics")) + { + if(!(item instanceof JSONObject diagnostic)) + { + throw new IOException("COMPILER_DIAGNOSTIC"); + } + Protocol.keys(diagnostic, "file", "line", "column", "code", "message"); + if(diagnostic.getString("file").length() > 200 || diagnostic.getString("code").length() > 40 || diagnostic.getString("message").length() > 500) + { + throw new IOException("COMPILER_DIAGNOSTIC"); + } + Protocol.number(diagnostic, "line", 0, Integer.MAX_VALUE); + Protocol.number(diagnostic, "column", 0, Integer.MAX_VALUE); + } + if(result.getBoolean("success")) + { + String bundle = result.getString("bundle"); + if(bundle.isEmpty() || bundle.length() > 1024 * 1024 || !Protocol.hash(bundle.getBytes(StandardCharsets.UTF_8)).equals(result.getString("bundleSha256")) + || result.getString("sourceMap").length() > 1024 * 1024) + { + throw new IOException("COMPILER_ARTIFACT"); + } + } + else if(result.has("bundle") || result.has("bundleSha256") || result.has("sourceMap")) + { + throw new IOException("FAILED_COMPILER_ARTIFACT"); + } + } + + /*************************************************************************** + ** Bounded nonsecret operator identifiers. + ***************************************************************************/ + private static boolean id(String value) + { + return (value != null && value.matches("[A-Za-z0-9._:-]{1,128}")); + } + + /*************************************************************************** + ** Close only this owned socket, never exposing TLS diagnostics. + ***************************************************************************/ + private static void disconnect(SSLSocket socket) + { + try + { + socket.close(); + } + catch(IOException ignored) + { + /* Already closed. */ + } + } + + /*************************************************************************** + ** Stop cold admission, terminate compiler children and await owned threads. + ***************************************************************************/ + @Override + public void close() throws Exception + { + Exception failure = null; + synchronized(this) + { + closed.set(true); + try + { + listener.close(); + } + catch(IOException exception) + { + failure = exception; + } + sockets.forEach(CompilerServer::disconnect); + clients.shutdown(); + } + try + { + compiler.close(); + } + catch(Exception exception) + { + failure = combined(failure, exception); + } + finally + { + deadlines.shutdownNow(); + } + try + { + if(!clients.awaitTermination(5000, TimeUnit.MILLISECONDS) || !deadlines.awaitTermination(1000, TimeUnit.MILLISECONDS)) + { + failure = combined(failure, new IOException("COMPILER_SERVER_DRAIN_UNKNOWN")); + } + } + catch(InterruptedException interrupted) + { + Thread.currentThread().interrupt(); + failure = combined(failure, interrupted); + } + if(failure != null) + { + throw failure; + } + } + + /*************************************************************************** + ** Preserve the primary shutdown failure while reporting incomplete cleanup. + ***************************************************************************/ + private static Exception combined(Exception primary, Exception additional) + { + if(primary == null) + { + return (additional); + } + if(primary != additional) + { + primary.addSuppressed(additional); + } + return (primary); + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/ExtismApi.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/ExtismApi.java new file mode 100644 index 0000000..6a6ea57 --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/ExtismApi.java @@ -0,0 +1,166 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import com.sun.jna.Callback; +import com.sun.jna.Library; +import com.sun.jna.Pointer; +import com.sun.jna.Structure; +import com.sun.jna.Union; +import com.sun.jna.ptr.PointerByReference; + +/******************************************************************************* + ** Exact public Extism1.41.0 C ABI subset; loaded only by the standalone service. + *******************************************************************************/ +interface ExtismApi extends Library +{ + /*************************************************************************** + ** Exact library version identity. + ***************************************************************************/ + String extism_version(); + + /*************************************************************************** + ** Allocate the single narrow broker host function. + ***************************************************************************/ + Pointer extism_function_new(String name, int[] inputs, long inputCount, int[] outputs, long outputCount, HostFunction callback, Pointer data, Pointer freeData); + + /*************************************************************************** + ** Bind only the explicit application namespace. + ***************************************************************************/ + void extism_function_set_namespace(Pointer function, String namespace); + + /*************************************************************************** + ** Release the host function owner after compiled engine destruction. + ***************************************************************************/ + void extism_function_free(Pointer function); + + /*************************************************************************** + ** Compile trusted fixed Wasm once with finite fuel/manifest limits. + ***************************************************************************/ + Pointer extism_compiled_plugin_new_with_fuel_limit(byte[] manifest, long length, Pointer[] functions, long count, byte wasi, long fuel, PointerByReference error); + + /*************************************************************************** + ** Free common engine only when its serial slot has no active instance. + ***************************************************************************/ + void extism_compiled_plugin_free(Pointer compiled); + + /*************************************************************************** + ** Fresh plugin/store for every invocation. + ***************************************************************************/ + Pointer extism_plugin_new_from_compiled(Pointer compiled, PointerByReference error); + + /*************************************************************************** + ** Free constructor error without copying its potentially private text. + ***************************************************************************/ + void extism_plugin_new_error_free(Pointer error); + + /*************************************************************************** + ** Actual instance destruction, not reset or pooling of guest state. + ***************************************************************************/ + void extism_plugin_free(Pointer plugin); + + /*************************************************************************** + ** Fixed zero-argument run export with private bounded JSON input. + ***************************************************************************/ + int extism_plugin_call(Pointer plugin, String function, byte[] input, long length); + + /*************************************************************************** + ** Bound native output before copying into Java heap. + ***************************************************************************/ + long extism_plugin_output_length(Pointer plugin); + + /*************************************************************************** + ** Borrow output only until this instance is freed. + ***************************************************************************/ + Pointer extism_plugin_output_data(Pointer plugin); + + /*************************************************************************** + ** Cancellation handle remains valid only for its owned instance. + ***************************************************************************/ + Pointer extism_plugin_cancel_handle(Pointer plugin); + + /*************************************************************************** + ** Engine-epoch cancellation; slots must remain serial. + ***************************************************************************/ + byte extism_plugin_cancel(Pointer handle); + + /*************************************************************************** + ** Current callback's Wasm linear-memory base. + ***************************************************************************/ + Pointer extism_current_plugin_memory(Pointer plugin); + + /*************************************************************************** + ** Validate an Extism allocation handle before reading its bytes. + ***************************************************************************/ + long extism_current_plugin_memory_length(Pointer plugin, long offset); + + /*************************************************************************** + ** Allocate a fresh response block, never aliasing request storage. + ***************************************************************************/ + long extism_current_plugin_memory_alloc(Pointer plugin, long bytes); + + /*************************************************************************** + ** Release an unreturned allocation if callback preparation fails. + ***************************************************************************/ + void extism_current_plugin_memory_free(Pointer plugin, long offset); + + /*************************************************************************** + ** JNA callback retained strongly for the entire compiled slot lifetime. + ***************************************************************************/ + interface HostFunction extends Callback + { + /*********************************************************************** + ** C ABI callback; all failures are translated to zero, never thrown to C. + ***********************************************************************/ + void invoke(Pointer plugin, Pointer inputs, long inputCount, Pointer outputs, long outputCount, Pointer data); + } + + /*************************************************************************** + ** Exact eight-byte C union, selecting i64 for the sole broker argument. + ***************************************************************************/ + class ValueUnion extends Union + { + public int i32; + public long i64; + public float f32; + public double f64; + } + + /*************************************************************************** + ** Public struct layout; JNA computes native alignment instead of magic offsets. + ***************************************************************************/ + @Structure.FieldOrder({"type", "value"}) + class Value extends Structure + { + public int type; + public ValueUnion value = new ValueUnion(); + + /*********************************************************************** + ** Borrow one fixed ABI value from the current native callback. + ***********************************************************************/ + Value(Pointer pointer) + { + super(pointer); + value.setType(long.class); + read(); + } + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/GuestRuntime.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/GuestRuntime.java new file mode 100644 index 0000000..e1ccd7b --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/GuestRuntime.java @@ -0,0 +1,93 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +/******************************************************************************* + ** Small private external-runtime boundary. Implementations never enter QQQ. + *******************************************************************************/ +public interface GuestRuntime extends AutoCloseable +{ + /*************************************************************************** + ** Actual fixed runtime digest, verified before service construction. + ***************************************************************************/ + String sha256(); + + /*************************************************************************** + ** Cold compile one common engine per serial slot, never one per publication. + ***************************************************************************/ + Slot createSlot() throws Exception; + + /*************************************************************************** + ** No application sessions or objects cross this narrow JSON callback. + ***************************************************************************/ + interface Broker + { + /*********************************************************************** + ** Synchronous callback on the invocation thread, independently bounded. + ***********************************************************************/ + byte[] exchange(byte[] request) throws Exception; + } + + /*************************************************************************** + ** A single compiled engine; supervisor serializes every use and cancellation. + ***************************************************************************/ + interface Slot extends AutoCloseable + { + /*********************************************************************** + ** Always create a fresh guest Store/Plugin; never reuse initialized state. + ***********************************************************************/ + Invocation open(Broker broker) throws Exception; + + /*********************************************************************** + ** Release compiled engine only after all guest instances have closed. + ***********************************************************************/ + @Override + void close() throws Exception; + } + + /*************************************************************************** + ** One fresh invocation. Cancellation is thread-safe and close proves cleanup. + ***************************************************************************/ + interface Invocation extends AutoCloseable + { + /*********************************************************************** + ** Private bounded prepare/run ABI; no method accepts arbitrary native code. + ***********************************************************************/ + byte[] call(byte[] input) throws Exception; + + /*********************************************************************** + ** Request bounded native cancellation from the supervisor watchdog. + ***********************************************************************/ + void cancel(); + + /*********************************************************************** + ** Free actual Store/Plugin, clearing all callback context before return. + ***********************************************************************/ + @Override + void close() throws Exception; + } + + /*************************************************************************** + ** Service shutdown releases its runtime ownership after slot teardown. + ***************************************************************************/ + @Override + void close() throws Exception; +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/InvocationBudget.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/InvocationBudget.java new file mode 100644 index 0000000..fb0f148 --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/InvocationBudget.java @@ -0,0 +1,227 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadMXBean; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/******************************************************************************* + ** Authoritative invocation-thread CPU plus independent wall deadline. Sampling + ** overshoot and native cancellation latency remain deployment qualification gates. + *******************************************************************************/ +final class InvocationBudget implements AutoCloseable +{ + private final ThreadMXBean cpu; + private final long thread; + private final long startedCpu; + private final long startedWall = System.nanoTime(); + private final long maxCpu; + private final long maxWall; + private final Runnable abortTransport; + private final ScheduledFuture watchdog; + private GuestRuntime.Invocation invocation; + private boolean expired; + private boolean closed; + private boolean instanceDestroyed; + private boolean cancellationFailed; + private long endedCpu; + private long endedWall; + + /*************************************************************************** + ** Missing/disabled thread CPU metering is a refusal, never a zero fallback. + ***************************************************************************/ + static ThreadMXBean requireCpu() throws IOException + { + ThreadMXBean bean = ManagementFactory.getThreadMXBean(); + if(!bean.isThreadCpuTimeSupported() || !bean.isThreadCpuTimeEnabled() || bean.getCurrentThreadCpuTime() < 0) + { + throw new IOException("CPU_METER_UNAVAILABLE"); + } + return (bean); + } + + /*************************************************************************** + ** Includes guest construction, host callbacks and destruction on this thread. + ***************************************************************************/ + InvocationBudget(ScheduledExecutorService scheduler, int cpuMillis, int wallMillis, Runnable abortTransport) throws IOException + { + cpu = requireCpu(); + thread = Thread.currentThread().threadId(); + startedCpu = cpu.getThreadCpuTime(thread); + maxCpu = TimeUnit.MILLISECONDS.toNanos(cpuMillis); + maxWall = TimeUnit.MILLISECONDS.toNanos(wallMillis); + this.abortTransport = abortTransport; + watchdog = scheduler.scheduleAtFixedRate(this::sample, 1, 1, TimeUnit.MILLISECONDS); + } + + /*************************************************************************** + ** Register only the current instance; a late construction is canceled too. + ***************************************************************************/ + synchronized void attach(GuestRuntime.Invocation value) throws IOException + { + if(closed || invocation != null) + { + throw new IOException("INVOCATION_STATE"); + } + invocation = value; + if(expired) + { + value.cancel(); + throw new IOException("INVOCATION_LIMIT"); + } + } + + /*************************************************************************** + ** Synchronized retirement prevents stale cancellation targeting another call. + ***************************************************************************/ + private synchronized void sample() + { + if(closed || expired) + { + return; + } + long used; + try + { + used = cpu.getThreadCpuTime(thread); + } + catch(RuntimeException unavailable) + { + used = -1; + } + if(used < startedCpu || used - startedCpu > maxCpu || System.nanoTime() - startedWall > maxWall) + { + expired = true; + try + { + if(invocation != null) + { + invocation.cancel(); + } + } + catch(RuntimeException | Error failure) + { + cancellationFailed = true; + throw failure; + } + finally + { + abortTransport.run(); + } + } + } + + /*************************************************************************** + ** Caller checks at broker boundaries as well as on normal completion. + ***************************************************************************/ + synchronized void check() throws IOException + { + sample(); + if(expired || Thread.currentThread().isInterrupted()) + { + throw new IOException("INVOCATION_LIMIT"); + } + } + + /*************************************************************************** + ** Actual measured CPU, rounded up only after successful guest destruction. + ***************************************************************************/ + int cpuMillis() throws IOException + { + if(!closed || endedCpu < startedCpu || expired) + { + throw new IOException("CPU_MEASUREMENT"); + } + return ((int) ((endedCpu - startedCpu + 999_999) / 1_000_000)); + } + + /*************************************************************************** + ** Wall time includes blocking broker waits; it is never substituted for CPU. + ***************************************************************************/ + int wallMillis() throws IOException + { + if(!closed || expired) + { + throw new IOException("WALL_MEASUREMENT"); + } + return ((int) ((endedWall - startedWall + 999_999) / 1_000_000)); + } + + /*************************************************************************** + ** Cleanup proof is distinct from watchdog retirement or a close attempt. + ***************************************************************************/ + synchronized boolean instanceDestroyed() + { + return (instanceDestroyed && !cancellationFailed); + } + + /*************************************************************************** + ** Free under the same cancel lock; never report retirement after close fails. + ***************************************************************************/ + @Override + public synchronized void close() throws Exception + { + if(closed) + { + return; + } + try + { + if(invocation != null) + { + invocation.close(); + instanceDestroyed = true; + } + endedCpu = cpu.getThreadCpuTime(thread); + endedWall = System.nanoTime(); + if(endedCpu < startedCpu || endedCpu - startedCpu > maxCpu || endedWall - startedWall > maxWall) + { + expired = true; + } + } + catch(RuntimeException | Error failure) + { + cancellationFailed = true; + throw failure; + } + finally + { + closed = true; + try + { + watchdog.cancel(false); + } + catch(RuntimeException | Error failure) + { + cancellationFailed = true; + throw failure; + } + } + if(expired) + { + throw new IOException("INVOCATION_LIMIT"); + } + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/JavyExtismRuntime.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/JavyExtismRuntime.java new file mode 100644 index 0000000..55edc3b --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/JavyExtismRuntime.java @@ -0,0 +1,347 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import com.sun.jna.Native; +import com.sun.jna.Pointer; +import com.sun.jna.ptr.PointerByReference; +import org.json.JSONObject; + +/******************************************************************************* + ** Native binding in the external service only. Exact fixed Wasm/library hashes, + ** finite fuel/memory, one common engine per slot and fresh Store per invocation. + *******************************************************************************/ +public final class JavyExtismRuntime implements GuestRuntime +{ + private final ExtismApi api; + private final byte[] manifest; + private final String digest; + private final long fuel; + + /*************************************************************************** + ** Operator pins actual files; no downloads, search-path loading or user Wasm. + ***************************************************************************/ + public JavyExtismRuntime(Path library, String libraryHash, Path wasm, String wasmHash, long memoryBytes, long fuel) throws Exception + { + if(!library.isAbsolute() || !wasm.isAbsolute() || !Files.isRegularFile(library) || !Files.isRegularFile(wasm) + || Files.size(library) > 256L * 1024 * 1024 || Files.size(wasm) > 32L * 1024 * 1024 || Native.POINTER_SIZE != 8 + || memoryBytes < 64L * 1024 * 1024 || memoryBytes > 256L * 1024 * 1024 || memoryBytes % 65536 != 0 + || fuel < 1 || fuel > 1_000_000_000L || System.getenv("EXTISM_ENABLE_WASI_OUTPUT") != null) + { + throw new IOException("NATIVE_CONFIGURATION"); + } + byte[] code = Files.readAllBytes(wasm); + if(!Protocol.hash(code).equals(wasmHash) || !Protocol.hash(Files.readAllBytes(library)).equals(libraryHash)) + { + throw new IOException("NATIVE_IDENTITY"); + } + api = Native.load(library.toString(), ExtismApi.class); + if(!"1.41.0".equals(api.extism_version())) + { + throw new IOException("NATIVE_VERSION"); + } + digest = wasmHash; + this.fuel = fuel; + manifest = new JSONObject().put("wasm", List.of(Map.of("data", Base64.getEncoder().encodeToString(code)))) + .put("memory", Map.of("max_pages", memoryBytes / 65536, "max_var_bytes", 0)) + .put("allowed_hosts", List.of()).put("allowed_paths", Map.of()).put("timeout_ms", 10_000).toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + + /*************************************************************************** + ** This is content identity, not deployment qualification or authorization. + ***************************************************************************/ + @Override + public String sha256() + { + return (digest); + } + + /*************************************************************************** + ** One common compiled runtime and broker callback per serial engine slot. + ***************************************************************************/ + @Override + public Slot createSlot() throws Exception + { + return (new NativeSlot()); + } + + /*************************************************************************** + ** Slot ownership frees native state; no global engine or guest state exists. + ***************************************************************************/ + @Override + public void close() {} + + /*************************************************************************** + ** Strong callback reference and exactly one active invocation per engine. + ***************************************************************************/ + private final class NativeSlot implements Slot + { + private final ExtismApi.HostFunction callback = this::broker; + private final Pointer function; + private final Pointer compiled; + private volatile NativeInvocation active; + private boolean closed; + + /*********************************************************************** + ** Fixed host ABI only; WASI has no paths/hosts or inherited output enabled. + ***********************************************************************/ + private NativeSlot() throws Exception + { + function = api.extism_function_new("qqq_broker", new int[] {1}, 1, new int[] {1}, 1, callback, Pointer.NULL, Pointer.NULL); + if(function == null) + { + throw new IOException("HOST_FUNCTION"); + } + api.extism_function_set_namespace(function, "extism:host/user"); + PointerByReference error = new PointerByReference(); + Pointer candidate = api.extism_compiled_plugin_new_with_fuel_limit(manifest, manifest.length, new Pointer[] {function}, 1, (byte) 1, fuel, error); + freeError(error); + if(candidate == null) + { + api.extism_function_free(function); + throw new IOException("RUNTIME_COMPILE"); + } + compiled = candidate; + } + + /*********************************************************************** + ** Plugin construction is fresh; active callback context is invocation-owned. + ***********************************************************************/ + @Override + public synchronized Invocation open(Broker broker) throws Exception + { + if(closed || active != null) + { + throw new IOException("SLOT_BUSY"); + } + PointerByReference error = new PointerByReference(); + Pointer plugin = api.extism_plugin_new_from_compiled(compiled, error); + freeError(error); + if(plugin == null) + { + throw new IOException("INSTANCE_CREATE"); + } + try + { + NativeInvocation invocation = new NativeInvocation(plugin, broker); + active = invocation; + return (invocation); + } + catch(Exception failure) + { + api.extism_plugin_free(plugin); + throw failure; + } + } + + /*********************************************************************** + ** Never let Java exceptions escape a C callback or expose raw guest data. + ***********************************************************************/ + private void broker(Pointer plugin, Pointer inputs, long inputCount, Pointer outputs, long outputCount, Pointer data) + { + long allocated = 0; + long borrowed = 0; + try + { + NativeInvocation current = active; + if(current == null || current.owner != Thread.currentThread() || current.closed || inputCount != 1 || outputCount != 1) + { + throw new IOException("BROKER_CONTEXT"); + } + ExtismApi.Value input = new ExtismApi.Value(inputs); + if(input.type != 1 || input.value.i64 <= 0) + { + throw new IOException("BROKER_POINTER"); + } + borrowed = input.value.i64; + long size = api.extism_current_plugin_memory_length(plugin, input.value.i64); + if(size < 1 || size > 64 * 1024) + { + throw new IOException("BROKER_INPUT"); + } + Pointer memory = api.extism_current_plugin_memory(plugin); + if(memory == null) + { + throw new IOException("BROKER_MEMORY"); + } + byte[] response = current.broker.exchange(memory.getByteArray(input.value.i64, (int) size)); + if(response == null || response.length == 0 || response.length > 64 * 1024) + { + throw new IOException("BROKER_OUTPUT"); + } + allocated = api.extism_current_plugin_memory_alloc(plugin, response.length); + if(allocated <= 0 || allocated == input.value.i64) + { + throw new IOException("BROKER_ALLOC"); + } + memory = api.extism_current_plugin_memory(plugin); + memory.write(allocated, response, 0, response.length); + ExtismApi.Value output = new ExtismApi.Value(outputs); + output.type = 1; + output.value.i64 = allocated; + output.write(); + allocated = 0; + } + catch(Throwable failure) + { + NativeInvocation current = active; + if(current != null) + { + current.brokerFailed = true; + } + try + { + if(allocated > 0 && allocated != borrowed) + { + api.extism_current_plugin_memory_free(plugin, allocated); + } + if(outputCount == 1) + { + ExtismApi.Value output = new ExtismApi.Value(outputs); + output.type = 1; + output.value.i64 = 0; + output.write(); + } + } + catch(Throwable ignored) + { + /* Native trap/call failure remains fatal. */ + } + } + } + + /*********************************************************************** + ** Retire engine only after actual instance close, never while it runs. + ***********************************************************************/ + @Override + public synchronized void close() throws Exception + { + if(active != null) + { + throw new IOException("ACTIVE_ENGINE"); + } + if(!closed) + { + closed = true; + api.extism_compiled_plugin_free(compiled); + api.extism_function_free(function); + } + } + + /*********************************************************************** + ** One plugin, one call, one broker context; no reset/reuse path exists. + ***********************************************************************/ + private final class NativeInvocation implements Invocation + { + private final Pointer plugin; + private final Pointer cancel; + private final Broker broker; + private final Thread owner = Thread.currentThread(); + private final AtomicBoolean called = new AtomicBoolean(); + private boolean brokerFailed; + private volatile boolean closed; + + /******************************************************************** + ** Cancellation handle belongs only to this freshly created instance. + ********************************************************************/ + private NativeInvocation(Pointer plugin, Broker broker) throws IOException + { + this.plugin = plugin; + this.broker = broker; + cancel = api.extism_plugin_cancel_handle(plugin); + if(cancel == null) + { + throw new IOException("CANCEL_HANDLE"); + } + } + + /******************************************************************** + ** Bound input/output; generic failure only, never native error text. + ********************************************************************/ + @Override + public byte[] call(byte[] input) throws Exception + { + if(Thread.currentThread() != owner || closed || !called.compareAndSet(false, true) || input.length > Protocol.MAX_FRAME) + { + throw new IOException("INVOCATION_STATE"); + } + if(api.extism_plugin_call(plugin, "run", input, input.length) != 0 || brokerFailed) + { + throw new IOException("GUEST_FAILED"); + } + long length = api.extism_plugin_output_length(plugin); + Pointer output = api.extism_plugin_output_data(plugin); + if(length < 2 || length > Protocol.MAX_FRAME || output == null) + { + throw new IOException("GUEST_OUTPUT"); + } + return (output.getByteArray(0, (int) length)); + } + + /******************************************************************** + ** Budget lock serializes cancellation with free and slot retirement. + ********************************************************************/ + @Override + public void cancel() + { + if(!closed) + { + api.extism_plugin_cancel(cancel); + } + } + + /******************************************************************** + ** Free Store before clearing the slot; throw rather than fake retirement. + ********************************************************************/ + @Override + public void close() + { + synchronized(NativeSlot.this) + { + if(!closed) + { + api.extism_plugin_free(plugin); + closed = true; + active = null; + } + } + } + } + } + + /*************************************************************************** + ** Constructor errors are owned native allocations, not user-facing messages. + ***************************************************************************/ + private void freeError(PointerByReference error) + { + if(error.getValue() != null) + { + api.extism_plugin_new_error_free(error.getValue()); + } + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/NodeColdCompiler.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/NodeColdCompiler.java new file mode 100644 index 0000000..f651bb7 --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/NodeColdCompiler.java @@ -0,0 +1,520 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.json.JSONObject; + +/******************************************************************************* + ** Disposable compile-only child with actual child CPU and bounded stream/wall + ** monitoring. Return only after clean process exit; no emitted code is executed. + *******************************************************************************/ +public final class NodeColdCompiler implements ColdCompiler +{ + private final List command; + private final Path directory; + private final Semaphore capacity; + private final ExecutorService pipes; + private final ScheduledThreadPoolExecutor watchdog = new ScheduledThreadPoolExecutor(1); + private final Set active = ConcurrentHashMap.newKeySet(); + private final AtomicBoolean closed = new AtomicBoolean(); + private final int cpuMillis; + private final int wallMillis; + + /*************************************************************************** + ** Production construction admits only an exactly verified fixed compiler image. + ***************************************************************************/ + public NodeColdCompiler(CompilerImage image, int slots, int cpuMillis, int wallMillis) throws Exception + { + this(verified(image), image.root(), slots, cpuMillis, wallMillis); + } + + /*************************************************************************** + ** Package-only trusted child fixture seam; operator configuration cannot set it. + ***************************************************************************/ + NodeColdCompiler(List command, Path directory, int slots, int cpuMillis, int wallMillis) + { + if(slots < 1 || slots > 8 || cpuMillis < 1 || cpuMillis > 30_000 || wallMillis < 100 || wallMillis > 60_000) + { + throw new IllegalArgumentException("COMPILER_LIMITS"); + } + this.command = List.copyOf(command); + this.directory = directory; + this.cpuMillis = cpuMillis; + this.wallMillis = wallMillis; + capacity = new Semaphore(slots); + pipes = Executors.newFixedThreadPool(slots * 3); + watchdog.setRemoveOnCancelPolicy(true); + } + + /*************************************************************************** + ** Verify before creating any child or returning a usable compiler service. + ***************************************************************************/ + private static List verified(CompilerImage image) throws Exception + { + image.verify(); + return (image.command()); + } + + /*************************************************************************** + ** Missing or invalid actual OS accounting is never replaced with parent CPU. + ***************************************************************************/ + static long requireCpuNanos(Optional observed) throws IOException + { + try + { + long nanos = observed.orElseThrow().toNanos(); + if(nanos < 0) + { + throw new IOException("COMPILER_CPU_UNAVAILABLE"); + } + return (nanos); + } + catch(RuntimeException invalid) + { + throw new IOException("COMPILER_CPU_UNAVAILABLE"); + } + } + + /*************************************************************************** + ** OS accounting may disappear between isAlive and info during confirmed exit. + ** Retain only an actual prior live sample after retirement and observed death; + ** live/initial missing or decreasing samples still refuse without a fallback. + ***************************************************************************/ + static long accountCpu(Optional observed, long previous, boolean retiredAndExited) throws IOException + { + if(observed.isEmpty() && retiredAndExited && previous >= 0) + { + return (previous); + } + long actual = requireCpuNanos(observed); + if(actual < previous) + { + if(retiredAndExited && previous >= 0) + { + return (previous); + } + throw new IOException("COMPILER_CPU_UNAVAILABLE"); + } + return (actual); + } + + /*************************************************************************** + ** One child and at most three pipe tasks per admitted compilation, no replay. + ***************************************************************************/ + @Override + public JSONObject compile(JSONObject request) throws Exception + { + if(closed.get() || !capacity.tryAcquire()) + { + throw new IOException("COMPILER_CAPACITY"); + } + Process process = null; + ChildBudget budget = null; + boolean retired = true; + try + { + byte[] input = line(request); + ProcessBuilder builder = new ProcessBuilder(command).directory(directory.toFile()); + builder.environment().clear(); + builder.environment().put("LANG", "C.UTF-8"); + builder.environment().put("TZ", "UTC"); + if(System.getProperty("os.name").startsWith("Windows")) + { + String systemRoot = System.getenv("SystemRoot"); + if(systemRoot != null) + { + builder.environment().put("SystemRoot", systemRoot); + } + } + long started = System.nanoTime(); + process = startOwned(builder); + retired = false; + budget = new ChildBudget(process, started); + Process child = process; + ChildBudget limits = budget; + BufferedInputStream stdout = new BufferedInputStream(child.getInputStream()); + Future stderr = pipes.submit(() -> drain(child.getErrorStream(), 8192, limits)); + JSONObject ready = await(pipes.submit(() -> Protocol.parse(readLine(stdout, 4096))), limits); + Protocol.keys(ready, "protocol", "type", "nodeVersion", "typescriptVersion", "toolchainId"); + Protocol.equal(ready, "protocol", "qqq-typescript-child/1"); + Protocol.equal(ready, "type", "READY"); + Protocol.equal(ready, "nodeVersion", "24.22.2"); + Protocol.equal(ready, "typescriptVersion", "6.0.3"); + Protocol.equal(ready, "toolchainId", "typescript-6.0.3-node-24"); + limits.checkAlive(); + await(pipes.submit(() -> + { + child.getOutputStream().write(input); + child.getOutputStream().flush(); + return (true); + }), limits); + JSONObject result = await(pipes.submit(() -> Protocol.parse(readLine(stdout, Protocol.MAX_FRAME))), limits); + limits.checkAlive(); + // The child deliberately remains alive here for actual final CPU sampling. + limits.retiring = true; + byte[] retirement = line(new JSONObject().put("protocol", "qqq-typescript-child/1").put("type", "RETIRE")); + await(pipes.submit(() -> + { + child.getOutputStream().write(retirement); + child.getOutputStream().close(); + return (true); + }), limits); + JSONObject exiting = await(pipes.submit(() -> Protocol.parse(readLine(stdout, 4096))), limits); + Protocol.keys(exiting, "protocol", "type"); + Protocol.equal(exiting, "protocol", "qqq-typescript-child/1"); + Protocol.equal(exiting, "type", "EXITING"); + if(!child.waitFor(limits.remaining(), TimeUnit.MILLISECONDS)) + { + throw new IOException("COMPILER_EXIT_TIMEOUT"); + } + limits.check(); + if(child.exitValue() != 0 || await(stderr, limits) != 0 || await(pipes.submit((java.util.concurrent.Callable) stdout::read), limits) != -1) + { + throw new IOException("COMPILER_EXIT_OR_TRANSCRIPT"); + } + retired = true; + return (result); + } + finally + { + if(process != null) + { + if(process.isAlive()) + { + process.destroyForcibly(); + retired = waitOwned(process); + } + else + { + retired = true; + } + if(retired) + { + closePipes(process); + active.remove(process); + } + } + if(budget != null) + { + budget.close(); + } + if(retired) + { + capacity.release(); + } + else + { + closed.set(true); + throw new IOException("COMPILER_RETIREMENT_UNKNOWN"); + } + } + } + + /*************************************************************************** + ** Serialize child creation with service shutdown so no child escapes its set. + ***************************************************************************/ + private synchronized Process startOwned(ProcessBuilder builder) throws IOException + { + if(closed.get()) + { + throw new IOException("COMPILER_CLOSED"); + } + Process child = builder.start(); + active.add(child); + return (child); + } + + /*************************************************************************** + ** Cleanup preserves caller interruption but still waits for owned termination. + ***************************************************************************/ + private static boolean waitOwned(Process child) + { + long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + boolean interrupted = false; + try + { + while(child.isAlive() && System.nanoTime() < end) + { + try + { + child.waitFor(Math.max(1, TimeUnit.NANOSECONDS.toMillis(end - System.nanoTime())), TimeUnit.MILLISECONDS); + } + catch(InterruptedException canceled) + { + interrupted = true; + } + } + return (!child.isAlive()); + } + finally + { + if(interrupted) + { + Thread.currentThread().interrupt(); + } + } + } + + /*************************************************************************** + ** After confirmed death, release pipe handles without masking the primary error. + ***************************************************************************/ + private static void closePipes(Process child) + { + for(java.io.Closeable pipe : List.of(child.getOutputStream(), child.getInputStream(), child.getErrorStream())) + { + try + { + pipe.close(); + } + catch(IOException ignored) + { + /* The process is already dead; no successful retirement depends on it. */ + } + } + } + + /*************************************************************************** + ** Bounded line framing matches the existing compile worker protocol. + ***************************************************************************/ + private static byte[] line(JSONObject value) throws IOException + { + byte[] bytes = (value + "\n").getBytes(StandardCharsets.UTF_8); + if(bytes.length > Protocol.MAX_FRAME + 1) + { + throw new IOException("COMPILER_FRAME"); + } + return (bytes); + } + + /*************************************************************************** + ** Stop before oversized or incomplete child output can reach JSON allocation. + ***************************************************************************/ + private static byte[] readLine(InputStream input, int maximum) throws IOException + { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + int next; + while((next = input.read()) != -1) + { + if(next == '\n') + { + return (output.toByteArray()); + } + if(output.size() >= maximum) + { + throw new IOException("COMPILER_FRAME"); + } + output.write(next); + } + throw new IOException("COMPILER_EOF"); + } + + /*************************************************************************** + ** Discard bounded stderr bytes without retaining or logging their contents. + ***************************************************************************/ + private static int drain(InputStream input, int maximum, ChildBudget budget) throws IOException + { + int count = 0; + byte[] bytes = new byte[1024]; + int length; + while((length = input.read(bytes)) != -1) + { + count += length; + if(count > maximum) + { + budget.fail(); + throw new IOException("COMPILER_STDERR_LIMIT"); + } + } + return (count); + } + + /*************************************************************************** + ** The same operation deadline bounds reads and writes, including blocked pipes. + ***************************************************************************/ + private static T await(Future future, ChildBudget budget) throws Exception + { + try + { + return (future.get(budget.remaining(), TimeUnit.MILLISECONDS)); + } + finally + { + if(!future.isDone()) + { + future.cancel(true); + } + } + } + + /*************************************************************************** + ** Stop all owned compiler children and wait before returning shutdown success. + ***************************************************************************/ + @Override + public void close() throws Exception + { + synchronized(this) + { + closed.set(true); + for(Process child : active) + { + child.destroyForcibly(); + } + } + boolean terminated = true; + for(Process child : active) + { + terminated &= waitOwned(child); + } + watchdog.shutdownNow(); + pipes.shutdownNow(); + if(!terminated || !pipes.awaitTermination(2500, TimeUnit.MILLISECONDS)) + { + throw new IOException("COMPILER_DRAIN_UNKNOWN"); + } + } + + /*************************************************************************** + ** Actual child process CPU, never parent-thread timing or a synthetic zero. + ***************************************************************************/ + private final class ChildBudget implements AutoCloseable + { + private final Process child; + private final long started; + private final java.util.concurrent.ScheduledFuture timer; + private volatile boolean failed; + private volatile boolean retiring; + private volatile long lastCpu = -1; + + /*********************************************************************** + ** Require OS child accounting immediately; unsupported providers refuse. + ***********************************************************************/ + private ChildBudget(Process child, long started) throws IOException + { + this.child = child; + this.started = started; + sample(); + check(); + timer = watchdog.scheduleAtFixedRate(this::sample, 2, 2, TimeUnit.MILLISECONDS); + } + + /*********************************************************************** + ** Observe child CPU and wall until exit; no stale invocation handle reuse. + ***********************************************************************/ + private synchronized void sample() + { + try + { + if(child.isAlive()) + { + Optional observed = child.info().totalCpuDuration(); + long nanos = accountCpu(observed, lastCpu, retiring && !child.toHandle().isAlive()); + if(nanos < lastCpu || nanos > TimeUnit.MILLISECONDS.toNanos(cpuMillis)) + { + fail(); + } + lastCpu = nanos; + } + else if(!retiring) + { + fail(); + } + if(System.nanoTime() - started > TimeUnit.MILLISECONDS.toNanos(wallMillis)) + { + fail(); + } + } + catch(Exception unavailable) + { + fail(); + } + } + + /*********************************************************************** + ** One owned child only; no process-name searches or broad termination. + ***********************************************************************/ + private void fail() + { + failed = true; + child.destroyForcibly(); + } + + /*********************************************************************** + ** Completed compiler result must remain alive for final CPU observation. + ***********************************************************************/ + private void checkAlive() throws IOException + { + sample(); + check(); + if(!child.isAlive()) + { + throw new IOException("COMPILER_EARLY_EXIT"); + } + } + + /*********************************************************************** + ** Refuse any absent/accounting failure or expired operation budget. + ***********************************************************************/ + private void check() throws IOException + { + if(failed || lastCpu < 0 || Thread.currentThread().isInterrupted()) + { + throw new IOException("COMPILER_CPU_OR_WALL_LIMIT"); + } + } + + /*********************************************************************** + ** Remaining timeout is never extended by a partial frame or child output. + ***********************************************************************/ + private long remaining() throws IOException + { + sample(); + check(); + return (Math.max(1, wallMillis - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started))); + } + + /*********************************************************************** + ** Cancel monitoring only after caller has reaped or quarantined this child. + ***********************************************************************/ + @Override + public void close() + { + timer.cancel(false); + } + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/Protocol.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/Protocol.java new file mode 100644 index 0000000..2634f02 --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/Protocol.java @@ -0,0 +1,214 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.Set; +import org.json.JSONObject; +import org.json.JSONParserConfiguration; +import org.json.JSONTokener; + +/******************************************************************************* + ** Standalone closed supervisor framing. It has no dependency on a QBit codec. + *******************************************************************************/ +final class Protocol +{ + static final String VERSION = "qbit-scripting-supervisor/1"; + static final String WORKER = "qbit-scripting-worker/1"; + static final int MAX_FRAME = 4 * 1024 * 1024; + private int transcript; + + /*************************************************************************** + ** Reset only at an acknowledged operation boundary. + ***************************************************************************/ + void reset() + { + transcript = 0; + } + + /*************************************************************************** + ** Bound length before allocation and decode strictly before JSON parsing. + ***************************************************************************/ + JSONObject read(DataInputStream input) throws Exception + { + int length = input.readInt(); + if(length < 2 || length > MAX_FRAME) + { + throw new IOException("FRAME"); + } + account(length); + byte[] bytes = new byte[length]; + input.readFully(bytes); + return (parse(bytes)); + } + + /*************************************************************************** + ** Output is also bounded; no diagnostic contains message values. + ***************************************************************************/ + void write(DataOutputStream output, JSONObject object) throws Exception + { + byte[] bytes = object.toString().getBytes(StandardCharsets.UTF_8); + if(bytes.length > MAX_FRAME) + { + throw new IOException("FRAME"); + } + account(bytes.length); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + } + + /*************************************************************************** + ** Both directions count toward a finite per-operation transcript. + ***************************************************************************/ + private void account(int count) throws IOException + { + transcript += count + 4; + if(transcript > 8 * 1024 * 1024) + { + throw new IOException("TRANSCRIPT"); + } + } + + /*************************************************************************** + ** Reject depth, nonstandard JSON, duplicate keys and trailing data. + ***************************************************************************/ + static JSONObject parse(byte[] bytes) throws Exception + { + if(bytes.length > MAX_FRAME) + { + throw new IOException("FRAME"); + } + String text = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(ByteBuffer.wrap(bytes)).toString(); + int depth = 0; + boolean quoted = false; + boolean escaped = false; + for(char c : text.toCharArray()) + { + if(quoted) + { + if(escaped) + { + escaped = false; + } + else if(c == '\\') + { + escaped = true; + } + else if(c == '"') + { + quoted = false; + } + } + else if(c == '"') + { + quoted = true; + } + else if(c == '{' || c == '[') + { + if(++depth > 16) + { + throw new IOException("DEPTH"); + } + } + else if(c == '}' || c == ']') + { + depth--; + } + } + var tokenizer = new JSONTokener(text, new JSONParserConfiguration().withStrictMode(true).withOverwriteDuplicateKey(false)); + JSONObject result = new JSONObject(tokenizer); + if(tokenizer.nextClean() != 0) + { + throw new IOException("JSON"); + } + return (result); + } + + /*************************************************************************** + ** Exact schemas and types, never permissive string/numeric coercion. + ***************************************************************************/ + static void keys(JSONObject object, String... keys) throws IOException + { + if(!object.keySet().equals(Set.of(keys))) + { + throw new IOException("KEYS"); + } + } + + /*************************************************************************** + ** Bounded opaque identity. + ***************************************************************************/ + static String id(JSONObject object, String key) throws IOException + { + Object value = object.opt(key); + if(!(value instanceof String text) || !text.matches("[A-Za-z0-9._:-]{1,128}")) + { + throw new IOException("IDENTITY"); + } + return (text); + } + + /*************************************************************************** + ** Exact string match without coercion. + ***************************************************************************/ + static void equal(JSONObject object, String key, String expected) throws IOException + { + if(!expected.equals(object.opt(key))) + { + throw new IOException("IDENTITY"); + } + } + + /*************************************************************************** + ** Whole bounded number only. + ***************************************************************************/ + static long number(JSONObject object, String key, long minimum, long maximum) throws IOException + { + Object value = object.opt(key); + if(!(value instanceof Integer || value instanceof Long)) + { + throw new IOException("NUMBER"); + } + long number = ((Number) value).longValue(); + if(number < minimum || number > maximum) + { + throw new IOException("NUMBER"); + } + return (number); + } + + /*************************************************************************** + ** Immutable content identity using the standard JDK implementation. + ***************************************************************************/ + static String hash(byte[] bytes) throws Exception + { + return (HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes))); + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SocketDeadline.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SocketDeadline.java new file mode 100644 index 0000000..7e5eacb --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SocketDeadline.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/******************************************************************************* + ** Finite socket exchange deadline, including writes and slow partial frames. + ** A retired callback cannot disconnect the next operation on the same socket. + *******************************************************************************/ +final class SocketDeadline implements AutoCloseable +{ + private final Runnable abort; + private final ScheduledFuture timer; + private boolean active = true; + + /*************************************************************************** + ** One distinct deadline object per handshake or complete session operation. + ***************************************************************************/ + SocketDeadline(ScheduledExecutorService scheduler, int milliseconds, Runnable abort) + { + this.abort = abort; + timer = scheduler.schedule(this::expire, milliseconds, TimeUnit.MILLISECONDS); + } + + /*************************************************************************** + ** Serialize the actual socket close against retirement, not only cancellation. + ***************************************************************************/ + synchronized void expire() + { + if(active) + { + active = false; + abort.run(); + } + } + + /*************************************************************************** + ** A dequeued timer observes inactive even when Future.cancel cannot stop it. + ***************************************************************************/ + @Override + public synchronized void close() + { + active = false; + timer.cancel(false); + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SupervisorMain.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SupervisorMain.java new file mode 100644 index 0000000..7a35c5e --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SupervisorMain.java @@ -0,0 +1,171 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.util.HashMap; +import java.util.Map; +import org.json.JSONObject; + +/******************************************************************************* + ** Standalone external service entrypoint. No default evidence, trust-all TLS, + ** downloaded runtime, application session or permissive execution fallback. + *******************************************************************************/ +public final class SupervisorMain +{ + /*************************************************************************** + ** Utility entrypoint. + ***************************************************************************/ + private SupervisorMain() {} + + /*************************************************************************** + ** One operator-owned bounded JSON file; diagnostics never echo its contents. + ***************************************************************************/ + public static void main(String[] arguments) + { + try + { + if(arguments.length != 1) + { + throw new IOException("CONFIG_REQUIRED"); + } + Path file = Path.of(arguments[0]); + if(!file.isAbsolute() || !Files.isRegularFile(file) || Files.size(file) > 64 * 1024) + { + throw new IOException("CONFIG_FILE"); + } + JSONObject config = Protocol.parse(Files.readAllBytes(file)); + Protocol.keys(config, "bind", "port", "keyStore", "trustStore", "certificateNamespaces", "profile", "toolchain", "evidence", "requiredEvidence", + "runtimePath", "runtimeSha256", "nativePath", "nativeSha256", "slots", "connections", "coldOperations", "entries", "codeBytes", "memoryBytes", "idleMillis", "fuel"); + SSLContext tls = tls(Path.of(config.getString("keyStore")), Path.of(config.getString("trustStore"))); + Map certificates = new HashMap<>(); + JSONObject mappings = config.getJSONObject("certificateNamespaces"); + for(String fingerprint : mappings.keySet()) + { + certificates.put(fingerprint, mappings.getString(fingerprint)); + } + long memory = Protocol.number(config, "memoryBytes", 64L * 1024 * 1024, 256L * 1024 * 1024); + var settings = new SupervisorServer.Configuration(tls, new InetSocketAddress(config.getString("bind"), (int) Protocol.number(config, "port", 1, 65535)), + certificates, config.getString("profile"), config.getString("toolchain"), config.getString("evidence"), config.getString("requiredEvidence"), config.getString("runtimeSha256"), + (int) Protocol.number(config, "slots", 1, 64), (int) Protocol.number(config, "connections", 1, 128), (int) Protocol.number(config, "coldOperations", 1, 8), + (int) Protocol.number(config, "entries", 1, 256), Protocol.number(config, "codeBytes", 1, 64L * 1024 * 1024), memory, (int) Protocol.number(config, "idleMillis", 100, 60_000)); + try(var runtime = new JavyExtismRuntime(Path.of(config.getString("nativePath")), config.getString("nativeSha256"), + Path.of(config.getString("runtimePath")), config.getString("runtimeSha256"), memory, Protocol.number(config, "fuel", 1, 1_000_000_000L)); + var server = new SupervisorServer(settings, runtime)) + { + Thread shutdown = new Thread(() -> + { + try + { + server.close(); + } + catch(Exception ignored) + { + /* Process/container termination owns final cleanup. */ + } + }, "script-supervisor-shutdown"); + Runtime.getRuntime().addShutdownHook(shutdown); + try + { + server.serve(); + } + finally + { + try + { + Runtime.getRuntime().removeShutdownHook(shutdown); + } + catch(IllegalStateException ignored) + { + /* Shutdown hook is already running. */ + } + } + } + } + catch(Exception failure) + { + System.err.println("SUPERVISOR_UNAVAILABLE"); + System.exit(1); + } + } + + /*************************************************************************** + ** Load only bounded operator key stores; passwords never enter guest/config. + ***************************************************************************/ + static SSLContext tls(Path keys, Path trust) throws Exception + { + char[] keyPassword = password("QQQ_SUPERVISOR_KEYSTORE_PASSWORD"); + char[] trustPassword = password("QQQ_SUPERVISOR_TRUSTSTORE_PASSWORD"); + try + { + KeyStore keyStore = store(keys, keyPassword); + KeyManagerFactory keyFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keyFactory.init(keyStore, keyPassword); + TrustManagerFactory trustFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trustFactory.init(store(trust, trustPassword)); + SSLContext context = SSLContext.getInstance("TLSv1.3"); + context.init(keyFactory.getKeyManagers(), trustFactory.getTrustManagers(), null); + return (context); + } + finally + { + java.util.Arrays.fill(keyPassword, '\0'); + java.util.Arrays.fill(trustPassword, '\0'); + } + } + + /*************************************************************************** + ** Required process secret; no default password or logging fallback. + ***************************************************************************/ + private static char[] password(String name) throws IOException + { + String value = System.getenv(name); + if(value == null || value.isEmpty() || value.length() > 4096) + { + throw new IOException("TLS_SECRET_REQUIRED"); + } + return (value.toCharArray()); + } + + /*************************************************************************** + ** Key material is host-only, outside guest filesystem or environment. + ***************************************************************************/ + private static KeyStore store(Path file, char[] password) throws Exception + { + if(!file.isAbsolute() || !Files.isRegularFile(file) || Files.size(file) > 1024 * 1024) + { + throw new IOException("TLS_STORE"); + } + KeyStore store = KeyStore.getInstance("PKCS12"); + try(var input = Files.newInputStream(file)) + { + store.load(input, password); + } + return (store); + } +} diff --git a/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SupervisorServer.java b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SupervisorServer.java new file mode 100644 index 0000000..693b72a --- /dev/null +++ b/supervisor/src/main/java/com/kingsrook/scripting/supervisor/SupervisorServer.java @@ -0,0 +1,689 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLSocket; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.json.JSONException; +import org.json.JSONObject; + +/******************************************************************************* + ** External mTLS supervisor. All engine/code/session state is locally bounded; + ** source authorization remains with the QQQ caller and its narrow broker. + *******************************************************************************/ +public final class SupervisorServer implements AutoCloseable +{ + private final Configuration config; + private final GuestRuntime runtime; + private final SSLServerSocket listener; + private final ArrayBlockingQueue available; + private final List engines = new ArrayList<>(); + private final Set sockets = ConcurrentHashMap.newKeySet(); + private final Semaphore cold; + private final ScheduledThreadPoolExecutor deadlines = new ScheduledThreadPoolExecutor(2); + private final ThreadPoolExecutor clients; + private final AtomicBoolean draining = new AtomicBoolean(); + private final AtomicInteger quarantined = new AtomicInteger(); + private final AtomicBoolean exhausted = new AtomicBoolean(); + + /*************************************************************************** + ** Exact certificate fingerprints map to one configured namespace. No wildcard + ** identity, dynamic runtime or script-controlled endpoint is accepted. + ***************************************************************************/ + public record Configuration(SSLContext tls, InetSocketAddress address, Map certificateNamespaces, + String profile, String toolchain, String evidence, String requiredEvidence, String runtimeSha256, + int slots, int connections, int coldOperations, int entries, long codeBytes, long memoryBytes, int idleMillis) + { + /*********************************************************************** + ** All limits are operator configuration with hard finite upper bounds. + ***********************************************************************/ + public Configuration + { + certificateNamespaces = Map.copyOf(certificateNamespaces); + if(tls == null || address == null || certificateNamespaces.isEmpty() || certificateNamespaces.size() > 256 + || !identifier(profile) || !identifier(toolchain) || !identifier(evidence) || !evidence.equals(requiredEvidence) + || !digest(runtimeSha256) || slots < 1 || slots > 64 || connections < slots || connections > 128 + || coldOperations < 1 || coldOperations > 8 || entries < 1 || entries > 256 || codeBytes < 1 || codeBytes > 64L * 1024 * 1024 + || memoryBytes < 64L * 1024 * 1024 || memoryBytes > 256L * 1024 * 1024 || idleMillis < 100 || idleMillis > 60_000) + { + throw new IllegalArgumentException("SUPERVISOR_CONFIGURATION"); + } + for(var entry : certificateNamespaces.entrySet()) + { + if(!digest(entry.getKey()) || !identifier(entry.getValue())) + { + throw new IllegalArgumentException("CERTIFICATE_NAMESPACE"); + } + } + } + } + + /*************************************************************************** + ** Compile only the fixed runtime at cold startup, with no author publication. + ***************************************************************************/ + public SupervisorServer(Configuration config, GuestRuntime runtime) throws Exception + { + this.config = config; + this.runtime = runtime; + InvocationBudget.requireCpu(); + if(!config.runtimeSha256().equals(runtime.sha256())) + { + throw new IOException("RUNTIME_IDENTITY"); + } + available = new ArrayBlockingQueue<>(config.slots()); + cold = new Semaphore(config.coldOperations()); + clients = new ThreadPoolExecutor(0, config.connections(), 30, TimeUnit.SECONDS, new SynchronousQueue<>(), runnable -> + { + Thread thread = new Thread(runnable, "script-supervisor-session"); + thread.setDaemon(true); + return (thread); + }); + deadlines.setRemoveOnCancelPolicy(true); + SSLServerSocket bound = null; + try + { + for(int i = 0; i < config.slots(); i++) + { + GuestRuntime.Slot slot = runtime.createSlot(); + engines.add(slot); + available.add(slot); + } + bound = (SSLServerSocket) config.tls().getServerSocketFactory().createServerSocket(); + bound.setNeedClientAuth(true); + bound.setEnabledProtocols(new String[] {"TLSv1.3"}); + bound.bind(config.address()); + listener = bound; + } + catch(Exception failure) + { + if(bound != null) + { + closeAfterStartupFailure(bound, failure); + } + for(GuestRuntime.Slot slot : engines) + { + closeAfterStartupFailure(slot, failure); + } + closeAfterStartupFailure(() -> clients.shutdownNow(), failure); + closeAfterStartupFailure(() -> deadlines.shutdownNow(), failure); + throw failure; + } + } + + /*************************************************************************** + ** Preserve startup failure while attempting every independently owned cleanup. + ***************************************************************************/ + private static void closeAfterStartupFailure(AutoCloseable resource, Exception primary) + { + try + { + resource.close(); + } + catch(Throwable cleanup) + { + if(cleanup != primary) + { + primary.addSuppressed(cleanup); + } + } + } + + /*************************************************************************** + ** Expose only the bound local port for host readiness/test orchestration. + ***************************************************************************/ + public int port() + { + return (listener.getLocalPort()); + } + + /*************************************************************************** + ** Fixed bounded threads, no queued sessions and no guest-selected endpoints. + ***************************************************************************/ + public void serve() throws IOException + { + while(!draining.get()) + { + try + { + SSLSocket socket = (SSLSocket) listener.accept(); + sockets.add(socket); + try + { + clients.execute(() -> session(socket)); + } + catch(RuntimeException rejected) + { + sockets.remove(socket); + socket.close(); + } + } + catch(IOException failure) + { + if(exhausted.get()) + { + throw new IOException("ENGINE_CAPACITY_EXHAUSTED"); + } + if(!draining.get()) + { + throw failure; + } + } + } + } + + /*************************************************************************** + ** Certificate-authorized session owns one serial engine and bounded code map. + ***************************************************************************/ + private void session(SSLSocket socket) + { + GuestRuntime.Slot engine = null; + boolean reusable = true; + Retirement retirement = new Retirement(); + try + { + socket.setTcpNoDelay(true); + socket.setSoTimeout(5000); + String session; + DataInputStream input; + DataOutputStream output; + Protocol wire; + try(var handshakeDeadline = new SocketDeadline(deadlines, 5000, () -> closeSocket(socket))) + { + socket.startHandshake(); + String fingerprint = Protocol.hash(socket.getSession().getPeerCertificates()[0].getEncoded()); + String namespace = config.certificateNamespaces().get(fingerprint); + if(namespace == null) + { + throw new IOException("CERTIFICATE_NAMESPACE"); + } + input = new DataInputStream(socket.getInputStream()); + output = new DataOutputStream(socket.getOutputStream()); + wire = new Protocol(); + JSONObject hello = wire.read(input); + Protocol.keys(hello, "protocol", "type", "namespace", "profile", "toolchain", "evidence", "nonce"); + Protocol.equal(hello, "protocol", Protocol.VERSION); + Protocol.equal(hello, "type", "HELLO"); + Protocol.equal(hello, "namespace", namespace); + Protocol.equal(hello, "profile", config.profile()); + Protocol.equal(hello, "toolchain", config.toolchain()); + Protocol.equal(hello, "evidence", config.evidence()); + Protocol.id(hello, "nonce"); + if(draining.get() || (engine = available.poll()) == null) + { + throw new IOException("CAPACITY"); + } + session = UUID.randomUUID().toString(); + wire.write(output, new JSONObject(hello.toString()).put("type", "ACCEPT").put("session", session)); + } + socket.setSoTimeout(config.idleMillis()); + Map cache = new HashMap<>(); + long codeBytes = 0; + long sequence = 0; + while(!draining.get()) + { + try(var exchangeDeadline = new SocketDeadline(deadlines, config.idleMillis() + 15_000, () -> closeSocket(socket))) + { + wire.reset(); + JSONObject frame = wire.read(input); + String requestId = Protocol.id(frame, "requestId"); + Protocol.equal(frame, "protocol", Protocol.VERSION); + Protocol.equal(frame, "session", session); + if(Protocol.number(frame, "sequence", 1, Long.MAX_VALUE) != ++sequence) + { + throw new IOException("SEQUENCE"); + } + String type = Protocol.id(frame, "type"); + if("PING".equals(type)) + { + Protocol.keys(frame, "protocol", "type", "session", "sequence", "requestId"); + // This serial session still owns the same untouched cache epoch. + wire.write(output, envelope(frame, "PONG")); + wire.write(output, envelope(frame, "RETIRED").put("retired", true)); + } + else if("PREPARE".equals(type)) + { + Protocol.keys(frame, "protocol", "type", "session", "sequence", "requestId", "sha256", "entryPoint", "bundle"); + if(cache.size() >= config.entries() || !cold.tryAcquire()) + { + throw new IOException("COLD_CAPACITY"); + } + try + { + Publication publication = prepare(engine, socket, frame, retirement); + if(codeBytes + publication.bytes() > config.codeBytes()) + { + throw new IOException("CACHE_BYTES"); + } + String readyId = UUID.randomUUID().toString(); + cache.put(readyId, publication); + codeBytes += publication.bytes(); + wire.write(output, envelope(frame, "READY").put("sha256", publication.artifact()).put("entryPoint", publication.entryPoint()).put("readyId", readyId)); + wire.write(output, envelope(frame, "RETIRED").put("retired", true)); + } + finally + { + cold.release(); + } + } + else if("INVOKE".equals(type)) + { + Protocol.keys(frame, "protocol", "type", "session", "sequence", "requestId", "readyId", "request"); + Publication publication = cache.get(Protocol.id(frame, "readyId")); + if(publication == null) + { + throw new IOException("NOT_PREPARED"); + } + JSONObject result = invoke(engine, socket, wire, input, output, frame, publication, retirement); + wire.write(output, envelope(frame, "RESULT").put("message", result)); + wire.write(output, envelope(frame, "RETIRED").put("retired", true)); + } + else + { + // Strict TypeScript compilation is a separately qualified cold service. + throw new IOException("UNSUPPORTED_OPERATION"); + } + } + } + } + catch(Throwable failure) + { + // Drop uncertain sessions without logging source, tokens or TLS identities. + reusable = (failure instanceof IOException || failure instanceof JSONException) && failure.getSuppressed().length == 0 && retirement.clean; + } + finally + { + closeSocket(socket); + sockets.remove(socket); + if(engine != null && reusable && retirement.clean && !draining.get()) + { + available.offer(engine); + } + else if(engine != null && (!reusable || !retirement.clean)) + { + quarantine(); + } + } + } + + /*************************************************************************** + ** Session-local proof survives protocol/write errors but never caches a guest. + ***************************************************************************/ + private static final class Retirement + { + private boolean clean = true; + } + + /*************************************************************************** + ** All engines lost is a service failure, not an indefinitely live empty pool. + ***************************************************************************/ + private void quarantine() + { + if(quarantined.incrementAndGet() == config.slots()) + { + exhausted.set(true); + try + { + listener.close(); + } + catch(IOException ignored) + { + /* The listener is already closed; serve observes the exhaustion flag. */ + } + } + } + + /*************************************************************************** + ** Private cache retains generated bytecode only, never record/token/context. + ***************************************************************************/ + private record Publication(String artifact, String entryPoint, String preparedJson, String preparedDigest, long bytes) {} + + /*************************************************************************** + ** Cold Javy bytecode preparation is distinct from strict TypeScript compile. + ***************************************************************************/ + private Publication prepare(GuestRuntime.Slot engine, SSLSocket socket, JSONObject frame, Retirement retirement) throws Exception + { + String bundle = frame.getString("bundle"); + String digest = frame.getString("sha256"); + String entry = frame.getString("entryPoint"); + byte[] source = bundle.getBytes(StandardCharsets.UTF_8); + if(bundle.length() > 1024 * 1024 || source.length > 2 * 1024 * 1024 || !digest(digest) || !digest.equals(Protocol.hash(source)) || !path(entry, ".ts")) + { + throw new IOException("ARTIFACT"); + } + JSONObject request = new JSONObject().put("operation", "prepare").put("runtimeSha256", runtime.sha256()) + .put("artifactSha256", digest).put("bundle", bundle); + JSONObject prepared = call(engine, socket, request, bytes -> + { + throw new IOException("PREPARE_BROKER"); + }, 5000, 10_000, retirement).result(); + Protocol.keys(prepared, "profile", "runtimeSha256", "artifactSha256", "entryPoint", "modules", "preparedSha256"); + Protocol.equal(prepared, "profile", "qqq-javy8.1-rquickjs0.12-esm/1"); + Protocol.equal(prepared, "runtimeSha256", runtime.sha256()); + Protocol.equal(prepared, "artifactSha256", digest); + Protocol.equal(prepared, "entryPoint", entry.substring(0, entry.length() - 3) + ".js"); + String preparedDigest = prepared.getString("preparedSha256"); + if(!digest(preparedDigest)) + { + throw new IOException("PREPARED_IDENTITY"); + } + var modules = prepared.getJSONArray("modules"); + if(modules.isEmpty() || modules.length() > 200) + { + throw new IOException("MODULES"); + } + long decoded = 0; + Set names = new HashSet<>(); + boolean entryPresent = false; + String previous = ""; + java.security.MessageDigest preparedHash = java.security.MessageDigest.getInstance("SHA-256"); + preparedHash.update("qqq-javy-prepared-v1\0".getBytes(StandardCharsets.UTF_8)); + for(String key : new String[] {"profile", "runtimeSha256", "artifactSha256", "entryPoint"}) + { + hashField(preparedHash, prepared.getString(key)); + } + for(int i = 0; i < modules.length(); i++) + { + JSONObject module = modules.getJSONObject(i); + Protocol.keys(module, "path", "bytecode", "sha256"); + String name = module.getString("path"); + String encoded = module.getString("bytecode"); + if(!path(name, ".js") || name.compareTo(previous) <= 0 || !names.add(name.toLowerCase(java.util.Locale.ROOT)) || encoded.length() > 2_796_204) + { + throw new IOException("MODULE"); + } + previous = name; + entryPresent |= name.equals(prepared.getString("entryPoint")); + byte[] bytes; + try + { + bytes = Base64.getDecoder().decode(encoded); + } + catch(IllegalArgumentException malformed) + { + throw new IOException("BYTECODE"); + } + decoded += bytes.length; + if(bytes.length == 0 || decoded > 2 * 1024 * 1024 || !Base64.getEncoder().encodeToString(bytes).equals(encoded) + || !Protocol.hash(bytes).equals(module.getString("sha256"))) + { + throw new IOException("BYTECODE"); + } + for(String key : new String[] {"path", "sha256", "bytecode"}) + { + hashField(preparedHash, module.getString(key)); + } + } + if(!entryPresent + || !java.util.HexFormat.of().formatHex(preparedHash.digest()).equals(preparedDigest)) + { + throw new IOException("PREPARED_DIGEST"); + } + String json = prepared.toString(); + return (new Publication(digest, entry, json, preparedDigest, json.getBytes(StandardCharsets.UTF_8).length)); + } + + /*************************************************************************** + ** Validate current invocation and relay its only broker capability to caller. + ***************************************************************************/ + private JSONObject invoke(GuestRuntime.Slot engine, SSLSocket socket, Protocol wire, DataInputStream input, DataOutputStream output, + JSONObject outer, Publication publication, Retirement retirement) throws Exception + { + JSONObject request = outer.getJSONObject("request"); + Protocol.keys(request, "protocol", "type", "requestId", "artifactId", "artifactSha256", "entryPoint", "event", "payload", "capabilityToken", "limits"); + Protocol.equal(request, "protocol", Protocol.WORKER); + Protocol.equal(request, "type", "execute"); + Protocol.equal(request, "requestId", outer.getString("requestId")); + Protocol.equal(request, "artifactSha256", publication.artifact()); + Protocol.equal(request, "entryPoint", publication.entryPoint()); + Protocol.id(request, "artifactId"); + Protocol.id(request, "event"); + String token = Protocol.id(request, "capabilityToken"); + if(request.getJSONObject("payload").toString().getBytes(StandardCharsets.UTF_8).length > 256 * 1024) + { + throw new IOException("PAYLOAD"); + } + JSONObject limits = request.getJSONObject("limits"); + Protocol.keys(limits, "cpuMillis", "wallMillis", "memoryBytes", "outputBytes", "brokerCalls"); + int cpu = (int) Protocol.number(limits, "cpuMillis", 1, 5000); + int wall = (int) Protocol.number(limits, "wallMillis", 1, 10_000); + int outputLimit = (int) Protocol.number(limits, "outputBytes", 1, 128 * 1024); + int calls = (int) Protocol.number(limits, "brokerCalls", 0, 50); + if(Protocol.number(limits, "memoryBytes", 1, 256L * 1024 * 1024) != config.memoryBytes()) + { + throw new IOException("MEMORY_PROFILE"); + } + Thread caller = Thread.currentThread(); + int[] count = {0}; + GuestRuntime.Broker broker = bytes -> + { + if(Thread.currentThread() != caller || bytes.length > 64 * 1024 || ++count[0] > calls) + { + throw new IOException("BROKER_LIMIT"); + } + JSONObject message = Protocol.parse(bytes); + Protocol.keys(message, "protocol", "type", "requestId", "operation", "token", "table", "fields", "criteria", "limit"); + Protocol.equal(message, "protocol", Protocol.WORKER); + Protocol.equal(message, "type", "broker"); + Protocol.equal(message, "requestId", outer.getString("requestId")); + Protocol.equal(message, "token", token); + Protocol.equal(message, "operation", "query"); + wire.write(output, envelope(outer, "BROKER").put("message", message)); + JSONObject answer = wire.read(input); + Protocol.keys(answer, "protocol", "type", "session", "sequence", "requestId", "message"); + Protocol.equal(answer, "protocol", Protocol.VERSION); + Protocol.equal(answer, "type", "BROKER_REPLY"); + Protocol.equal(answer, "session", outer.getString("session")); + Protocol.equal(answer, "requestId", outer.getString("requestId")); + if(Protocol.number(answer, "sequence", 1, Long.MAX_VALUE) != outer.getLong("sequence")) + { + throw new IOException("SEQUENCE"); + } + JSONObject response = answer.getJSONObject("message"); + Protocol.equal(response, "protocol", Protocol.WORKER); + Protocol.equal(response, "type", "brokerResult"); + byte[] encoded = response.toString().getBytes(StandardCharsets.UTF_8); + if(encoded.length > 64 * 1024) + { + throw new IOException("BROKER_OUTPUT"); + } + return (encoded); + }; + JSONObject operation = new JSONObject().put("operation", "run").put("runtimeSha256", runtime.sha256()) + .put("preparedSha256", publication.preparedDigest()).put("prepared", new JSONObject(publication.preparedJson())).put("request", request); + Call result = call(engine, socket, operation, broker, cpu, wall, retirement); + Protocol.keys(result.result(), "message", "guestWallMillis"); + JSONObject message = result.result().getJSONObject("message"); + Protocol.equal(message, "protocol", Protocol.WORKER); + Protocol.equal(message, "type", "executeResult"); + Protocol.equal(message, "requestId", outer.getString("requestId")); + if(message.has("usage")) + { + throw new IOException("GUEST_USAGE"); + } + message.put("usage", new JSONObject().put("cpuMillis", result.cpu()).put("wallMillis", result.wall())); + if(message.toString().getBytes(StandardCharsets.UTF_8).length > outputLimit) + { + throw new IOException("OUTPUT"); + } + return (message); + } + + /*************************************************************************** + ** Only successful instance destruction allows any result or retirement frame. + ***************************************************************************/ + private Call call(GuestRuntime.Slot engine, SSLSocket socket, JSONObject request, GuestRuntime.Broker broker, int cpu, int wall, Retirement retirement) throws Exception + { + byte[] input = request.toString().getBytes(StandardCharsets.UTF_8); + if(input.length > Protocol.MAX_FRAME) + { + throw new IOException("GUEST_INPUT"); + } + InvocationBudget budget = new InvocationBudget(deadlines, cpu, wall, () -> closeSocket(socket)); + byte[] output; + retirement.clean = false; + try(budget) + { + GuestRuntime.Invocation invocation = engine.open(bytes -> + { + budget.check(); + byte[] result = broker.exchange(bytes); + budget.check(); + return (result); + }); + budget.attach(invocation); + output = invocation.call(input); + budget.check(); + } + finally + { + retirement.clean = budget.instanceDestroyed(); + } + return (new Call(Protocol.parse(output), budget.cpuMillis(), budget.wallMillis())); + } + + /*************************************************************************** + ** Host measurements, not a guest-controlled usage claim. + ***************************************************************************/ + private record Call(JSONObject result, int cpu, int wall) {} + + /*************************************************************************** + ** Match the fixed Rust ABI's domain-separated u64 length-prefix digest. + ***************************************************************************/ + private static void hashField(java.security.MessageDigest digest, String field) + { + byte[] bytes = field.getBytes(StandardCharsets.UTF_8); + digest.update(java.nio.ByteBuffer.allocate(8).putLong(bytes.length).array()); + digest.update(bytes); + } + + /*************************************************************************** + ** Bind every outbound control message to the exact session operation. + ***************************************************************************/ + private static JSONObject envelope(JSONObject input, String type) + { + return (new JSONObject().put("protocol", Protocol.VERSION).put("type", type).put("session", input.getString("session")) + .put("sequence", input.getLong("sequence")).put("requestId", input.getString("requestId"))); + } + + /*************************************************************************** + ** Portable closed module names, never filesystem resolution or network paths. + ***************************************************************************/ + private static boolean path(String value, String extension) + { + return (value != null && value.length() <= 200 && value.endsWith(extension) && !value.startsWith("/") && !value.contains("..") + && value.matches("[A-Za-z0-9_./-]+")); + } + + /*************************************************************************** + ** Fixed opaque configuration identifiers. + ***************************************************************************/ + private static boolean identifier(String value) + { + return (value != null && value.matches("[A-Za-z0-9._:-]{1,128}")); + } + + /*************************************************************************** + ** Exact lowercase SHA-256. + ***************************************************************************/ + private static boolean digest(String value) + { + return (value != null && value.matches("[a-f0-9]{64}")); + } + + /*************************************************************************** + ** Abort failed sessions without waiting for a peer TLS close-notify response. + ** A TLS 1.3 graceful close can drain input for the entire idle timeout. + ***************************************************************************/ + private static void closeSocket(SSLSocket socket) + { + try + { + socket.setSoLinger(true, 0); + socket.setSoTimeout(250); + } + catch(IOException ignored) + { + /* A disconnected socket may no longer accept option changes. */ + } + try + { + socket.shutdownInput(); + } + catch(IOException ignored) + { + /* JSSE still shuts input down when the peer omitted close_notify. */ + } + try + { + socket.close(); + } + catch(IOException ignored) + { + /* Already disconnected. */ + } + } + + /*************************************************************************** + ** Stop admission, disconnect clients and await cleanup; never free an engine + ** while a native invocation remains active or claim that it was retired. + ***************************************************************************/ + @Override + public synchronized void close() throws Exception + { + draining.set(true); + listener.close(); + clients.shutdown(); + sockets.forEach(SupervisorServer::closeSocket); + if(!clients.awaitTermination(15, TimeUnit.SECONDS)) + { + throw new IOException("DRAIN_INCOMPLETE"); + } + deadlines.shutdownNow(); + for(GuestRuntime.Slot engine : engines) + { + engine.close(); + } + runtime.close(); + } +} diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompileChildFixture.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompileChildFixture.java new file mode 100644 index 0000000..da91c4e --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompileChildFixture.java @@ -0,0 +1,105 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import org.json.JSONArray; +import org.json.JSONObject; + +/******************************************************************************* + ** Fixed trusted protocol-only child. No source compilation or evaluation occurs. + *******************************************************************************/ +public final class CompileChildFixture +{ + /*************************************************************************** + ** Utility child entrypoint. + ***************************************************************************/ + private CompileChildFixture() {} + + /*************************************************************************** + ** A few bounded transcript faults exercise cleanup, never hostile author code. + ***************************************************************************/ + public static void main(String[] args) throws Exception + { + String mode = args[0]; + Path pid = Path.of(args[1]); + Path pending = pid.resolveSibling("pid.pending"); + Files.writeString(pending, Long.toString(ProcessHandle.current().pid())); + Files.move(pending, pid, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + if("stderr".equals(mode)) + { + System.err.print("x".repeat(8193)); + Thread.sleep(1000); + return; + } + if("partial".equals(mode)) + { + System.out.print("{"); + return; + } + send(new JSONObject().put("protocol", "qqq-typescript-child/1").put("type", "READY").put("nodeVersion", "24.22.2") + .put("typescriptVersion", "6.0.3").put("toolchainId", "typescript-6.0.3-node-24")); + try(var input = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8))) + { + JSONObject request = new JSONObject(input.readLine()); + if("wait".equals(mode)) + { + Thread.sleep(1500); + } + if(!java.util.Set.of("LANG", "TZ", "SystemRoot").containsAll(System.getenv().keySet())) + { + throw new IllegalStateException("ENVIRONMENT_NOT_CLEAN"); + } + String bundle = "fixed trusted fixture only"; + JSONObject result = new JSONObject().put("protocol", Protocol.WORKER).put("type", "compileResult").put("requestId", request.getString("requestId")) + .put("toolchainId", "typescript-6.0.3-node-24").put("strict", true).put("success", true).put("diagnostics", new JSONArray()) + .put("bundle", bundle).put("bundleSha256", Protocol.hash(bundle.getBytes(StandardCharsets.UTF_8))).put("sourceMap", "{}"); + send(result); + if("early".equals(mode)) + { + return; + } + JSONObject retirement = new JSONObject(input.readLine()); + Protocol.keys(retirement, "protocol", "type"); + Protocol.equal(retirement, "protocol", "qqq-typescript-child/1"); + Protocol.equal(retirement, "type", "RETIRE"); + send(new JSONObject().put("protocol", "qqq-typescript-child/1").put("type", "EXITING")); + if("extra".equals(mode)) + { + System.out.println("unexpected"); + } + } + } + + /*************************************************************************** + ** Fixed small fixture frame with explicit flush for real pipe coordination. + ***************************************************************************/ + private static void send(JSONObject value) + { + System.out.println(value); + System.out.flush(); + } +} diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompilerImageTest.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompilerImageTest.java new file mode 100644 index 0000000..c2228bd --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompilerImageTest.java @@ -0,0 +1,95 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/******************************************************************************* + ** Image identity tests use inert bytes; no Node process or author code executes. + *******************************************************************************/ +class CompilerImageTest +{ + /*************************************************************************** + ** Exact closure verifies, but changed/extra files fail before child creation. + ***************************************************************************/ + @Test + void immutableClosureAndNoAmbientExecutor(@TempDir Path directory) throws Exception + { + CompilerImage image = fixture(directory); + image.verify(); + assertFalse(image.command().contains("--allow-child-process")); + assertEquals(image.node().toString(), image.command().getFirst()); + Path extra = image.root().resolve("dist/execute.js"); + Files.writeString(extra, "inert"); + assertThrows(Exception.class, image::verify); + Files.delete(extra); + Files.writeString(image.root().resolve("dist/compiler.js"), "changed"); + assertThrows(Exception.class, image::verify); + } + + /*************************************************************************** + ** Hash substitution and absent required libraries are not accepted by manifest. + ***************************************************************************/ + @Test + void executableManifestAndRequiredFilesArePinned(@TempDir Path directory) throws Exception + { + CompilerImage image = fixture(directory); + assertThrows(Exception.class, () -> new CompilerImage(image.node(), "0".repeat(64), image.root(), image.manifestSha256(), 128).verify()); + assertThrows(Exception.class, () -> new CompilerImage(image.node(), image.nodeSha256(), image.root(), "0".repeat(64), 128).verify()); + Path manifest = image.root().resolve("compiler-manifest.json"); + JSONObject value = new JSONObject(Files.readString(manifest)); + value.getJSONObject("files").remove("dist/frames.js"); + Files.writeString(manifest, value.toString()); + var missing = new CompilerImage(image.node(), image.nodeSha256(), image.root(), CompilerImage.hashFile(manifest, 131072), 128); + assertThrows(Exception.class, missing::verify); + assertThrows(IllegalArgumentException.class, () -> new CompilerImage(image.node(), image.nodeSha256(), directory.resolve("wild*"), image.manifestSha256(), 128)); + } + + /*************************************************************************** + ** Construct only the exact minimal legal tree with harmless synthetic bytes. + ***************************************************************************/ + private CompilerImage fixture(Path directory) throws Exception + { + Path node = Files.writeString(directory.resolve("inert-node"), "not executable"); + Path root = Files.createDirectory(directory.resolve("image")); + JSONObject entries = new JSONObject(); + for(String name : List.of("package.json", "dist/compiler-entry.js", "dist/compiler.js", "dist/frames.js", "dist/protocol.js", + "node_modules/typescript/package.json", "node_modules/typescript/lib/typescript.js", "node_modules/typescript/lib/lib.es2022.d.ts")) + { + Path path = root.resolve(name); + Files.createDirectories(path.getParent()); + Files.writeString(path, "fixed inert bytes"); + entries.put(name, CompilerImage.hashFile(path, 1024)); + } + Path manifest = root.resolve("compiler-manifest.json"); + Files.writeString(manifest, new JSONObject().put("protocol", "qqq-typescript-image/1").put("nodeVersion", "24.22.2") + .put("typescriptVersion", "6.0.3").put("files", entries).toString()); + return (new CompilerImage(node, CompilerImage.hashFile(node, 1024), root, CompilerImage.hashFile(manifest, 131072), 128)); + } +} \ No newline at end of file diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompilerServerTest.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompilerServerTest.java new file mode 100644 index 0000000..e4bbe73 --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/CompilerServerTest.java @@ -0,0 +1,434 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/******************************************************************************* + ** Real TLS routing with a non-compiling trusted fixture; no runtime qualification. + *******************************************************************************/ +class CompilerServerTest +{ + @TempDir + static Path directory; + private static TlsIdentity tls; + private static final String TOOLCHAIN = "typescript-6.0.3-node-24"; + + /*************************************************************************** + ** Exact temporary certificate identity, never a deployed credential. + ***************************************************************************/ + @BeforeAll + static void identity() throws Exception + { + tls = TlsIdentity.create(directory); + } + + /*************************************************************************** + ** Result/retirement follows compile return and closes each cold connection. + ***************************************************************************/ + @Test + void twoIndependentColdExchangesMatchIdentityAndRetire() throws Exception + { + try(var host = new Host(tls.fingerprint(), false)) + { + for(int i = 0; i < 2; i++) + { + try(var client = new Client(host.server.port(), "fixture")) + { + client.send(client.request()); + JSONObject result = client.read(); + assertEquals("RESULT", result.getString("type")); + assertEquals(client.session, result.getString("session")); + assertEquals("fixture", result.getString("requestId")); + assertEquals(1, result.getInt("sequence")); + CompilerServer.validateResult(result.getJSONObject("message"), "fixture", TOOLCHAIN); + JSONObject retired = client.read(); + assertEquals("RETIRED", retired.getString("type")); + assertTrue(retired.getBoolean("retired")); + assertEquals(result.getString("session"), retired.getString("session")); + assertThrows(IOException.class, client::read); + } + } + assertEquals(2, host.calls.get()); + } + } + + /*************************************************************************** + ** Wrong namespace/certificate and non-compile envelopes never reach the child. + ***************************************************************************/ + @Test + void authorityAndClosedEnvelopesRefuseBeforeCompiler() throws Exception + { + try(var host = new Host("b".repeat(64), false)) + { + assertThrows(IOException.class, () -> new Client(host.server.port(), "fixture")); + assertEquals(0, host.calls.get()); + } + try(var host = new Host(tls.fingerprint(), false)) + { + assertThrows(IOException.class, () -> new Client(host.server.port(), "other")); + for(String failure : List.of("type", "session", "sequence", "extra", "strict", "innerId")) + { + try(var client = new Client(host.server.port(), "fixture")) + { + JSONObject frame = client.request(); + switch(failure) + { + case "type" -> frame.put("type", "INVOKE"); + case "session" -> frame.put("session", "other"); + case "sequence" -> frame.put("sequence", 2); + case "extra" -> frame.put("capabilityToken", "never-forward"); + case "strict" -> frame.getJSONObject("request").put("strict", "true"); + case "innerId" -> frame.getJSONObject("request").put("requestId", "other"); + default -> throw new AssertionError(); + } + client.send(frame); + assertThrows(IOException.class, client::read); + } + } + assertEquals(0, host.calls.get()); + } + } + + /*************************************************************************** + ** Bad compiler output cannot become an outer result or retirement assertion. + ***************************************************************************/ + @Test + void corruptCompilerResultClosesWithoutSuccess() throws Exception + { + try(var host = new Host(tls.fingerprint(), true); var client = new Client(host.server.port(), "fixture")) + { + client.send(client.request()); + assertThrows(IOException.class, client::read); + assertEquals(1, host.calls.get()); + } + } + + /*************************************************************************** + ** Bound source and result contracts before allocating children or forwarding. + ***************************************************************************/ + @Test + void closedSourceAndResultLimits() throws Exception + { + JSONObject request = NodeColdCompilerTest.request(); + request.getJSONArray("files").getJSONObject(0).put("content", "x".repeat(262145)); + assertThrows(Exception.class, () -> CompilerServer.validateRequest(request, "fixture", TOOLCHAIN)); + JSONObject result = result("fixture"); + result.put("success", false); + assertThrows(Exception.class, () -> CompilerServer.validateResult(result, "fixture", TOOLCHAIN)); + result.remove("bundle"); + result.remove("bundleSha256"); + result.remove("sourceMap"); + CompilerServer.validateResult(result, "fixture", TOOLCHAIN); + result.put("broker", new JSONObject()); + assertThrows(Exception.class, () -> CompilerServer.validateResult(result, "fixture", TOOLCHAIN)); + } + + /*************************************************************************** + ** Exactly N working plus N waiting sockets; close failure still drains both. + ***************************************************************************/ + @Test + void boundedQueueAndFailingCompilerCloseReleaseOwnedResources() throws Exception + { + Set prior = Thread.getAllStackTraces().keySet(); + IOException expected = new IOException("FIXTURE_COMPILER_CLOSE"); + CompilerServer server = idleServer(expected); + Thread listener = listener(server); + try(Socket first = new Socket("127.0.0.1", server.port()); Socket queued = new Socket("127.0.0.1", server.port()); + Socket excess = new Socket("127.0.0.1", server.port())) + { + excess.setSoTimeout(2000); + assertEquals(-1, excess.getInputStream().read(), "No third socket may wait behind one worker and one handoff position"); + assertSame(expected, assertThrows(IOException.class, server::close)); + ended(first, 1000); + ended(queued, 1000); + listener.join(1000); + assertFalse(listener.isAlive()); + assertFalse(Thread.getAllStackTraces().keySet().stream().anyMatch(thread -> !prior.contains(thread) + && thread.isAlive() && thread.getName().equals("script-cold-compiler-deadline"))); + } + finally + { + assertSame(expected, assertThrows(IOException.class, server::close)); + listener.join(1000); + } + } + + /*************************************************************************** + ** The queued handshake expires from acceptance, not after its worker starts. + ***************************************************************************/ + @Test + void waitingSocketDoesNotReceiveANewHandshakeBudget() throws Exception + { + try(var server = idleServer(null)) + { + Thread listener = listener(server); + try(Socket first = new Socket("127.0.0.1", server.port()); Socket queued = new Socket("127.0.0.1", server.port())) + { + ended(queued, 6500); + ended(first, 1000); + } + finally + { + server.close(); + listener.join(1000); + assertFalse(listener.isAlive()); + } + } + } + + /*************************************************************************** + ** No compiler may run before a valid handshake and authorized compile frame. + ***************************************************************************/ + private CompilerServer idleServer(IOException closeFailure) throws Exception + { + ColdCompiler compiler = new ColdCompiler() + { + /******************************************************************* + ** Handshake-only sockets never reach compilation. + *******************************************************************/ + @Override + public JSONObject compile(JSONObject request) + { + throw new AssertionError("UNAUTHENTICATED_COMPILATION"); + } + + /******************************************************************* + ** Inject only a fixed ordinary cleanup failure. + *******************************************************************/ + @Override + public void close() throws IOException + { + if(closeFailure != null) + { + throw closeFailure; + } + } + }; + return (new CompilerServer(new CompilerServer.Configuration(tls.context(), new InetSocketAddress("127.0.0.1", 0), + Map.of(tls.fingerprint(), "fixture"), "profile", TOOLCHAIN, "fixture-only", "fixture-only", 1, 5000), compiler)); + } + + /*************************************************************************** + ** Own one daemon accept thread for the narrow lifecycle tests. + ***************************************************************************/ + private Thread listener(CompilerServer server) + { + return (Thread.ofPlatform().daemon().start(() -> + { + try + { + server.serve(); + } + catch(IOException failure) + { + throw new AssertionError(failure); + } + })); + } + + /*************************************************************************** + ** A TLS close alert is allowed, but no unbounded bytes or live socket remains. + ***************************************************************************/ + private void ended(Socket socket, int timeout) throws IOException + { + socket.setSoTimeout(timeout); + int count = 0; + while(socket.getInputStream().read() != -1) + { + assertTrue(++count <= 4096); + } + } + + /*************************************************************************** + ** Fixed synthetic protocol output, not a compiled author artifact. + ***************************************************************************/ + private static JSONObject result(String id) throws Exception + { + String bundle = "trusted protocol fixture"; + return (new JSONObject().put("protocol", Protocol.WORKER).put("type", "compileResult").put("requestId", id) + .put("strict", true).put("toolchainId", TOOLCHAIN).put("success", true).put("diagnostics", new JSONArray()) + .put("bundle", bundle).put("bundleSha256", Protocol.hash(bundle.getBytes(StandardCharsets.UTF_8))).put("sourceMap", "{}")); + } + + /*************************************************************************** + ** Own only a random loopback listener and one trusted compile fixture. + ***************************************************************************/ + private static final class Host implements AutoCloseable + { + private final AtomicInteger calls = new AtomicInteger(); + private final CompilerServer server; + private final Thread thread; + + /*********************************************************************** + ** Test evidence is explicitly synthetic and cannot qualify a deployment. + ***********************************************************************/ + private Host(String certificate, boolean corrupt) throws Exception + { + ColdCompiler compiler = new ColdCompiler() + { + /******************************************************************* + ** No compiler, source evaluation or subprocess exists in this fixture. + *******************************************************************/ + @Override + public JSONObject compile(JSONObject request) throws Exception + { + calls.incrementAndGet(); + JSONObject value = result(request.getString("requestId")); + if(corrupt) + { + value.put("bundleSha256", "b".repeat(64)); + } + return (value); + } + + /******************************************************************* + ** Fixture owns no resources. + *******************************************************************/ + @Override + public void close() {} + }; + server = new CompilerServer(new CompilerServer.Configuration(tls.context(), new InetSocketAddress("127.0.0.1", 0), + Map.of(certificate, "fixture"), "profile", TOOLCHAIN, "fixture-only", "fixture-only", 2, 5000), compiler); + thread = Thread.ofPlatform().daemon().start(() -> + { + try + { + server.serve(); + } + catch(IOException failure) + { + throw new AssertionError(failure); + } + }); + } + + /*********************************************************************** + ** No daemon survives a fixture, including failed authority checks. + ***********************************************************************/ + @Override + public void close() throws Exception + { + server.close(); + thread.join(1000); + assertFalse(thread.isAlive()); + } + } + + /*************************************************************************** + ** Independent typed frame fixture with real mutual TLS and hostname checks. + ***************************************************************************/ + private static final class Client implements AutoCloseable + { + private final SSLSocket socket; + private final DataInputStream input; + private final DataOutputStream output; + private final Protocol wire = new Protocol(); + private final String session; + + /*********************************************************************** + ** Bind the same certificate/profile handshake used by the production client. + ***********************************************************************/ + private Client(int port, String namespace) throws Exception + { + socket = (SSLSocket) tls.context().getSocketFactory().createSocket("localhost", port); + try + { + socket.setTcpNoDelay(true); + socket.setSoTimeout(2500); + SSLParameters settings = socket.getSSLParameters(); + settings.setEndpointIdentificationAlgorithm("HTTPS"); + settings.setProtocols(new String[] {"TLSv1.3"}); + socket.setSSLParameters(settings); + socket.startHandshake(); + input = new DataInputStream(socket.getInputStream()); + output = new DataOutputStream(socket.getOutputStream()); + send(new JSONObject().put("protocol", Protocol.VERSION).put("type", "HELLO").put("namespace", namespace) + .put("profile", "profile").put("toolchain", TOOLCHAIN).put("evidence", "fixture-only").put("nonce", UUID.randomUUID().toString())); + JSONObject accepted = read(); + assertEquals("ACCEPT", accepted.getString("type")); + session = accepted.getString("session"); + } + catch(Exception failure) + { + socket.close(); + throw failure; + } + } + + /*********************************************************************** + ** Closed ordinary COMPILE envelope with the actual worker request shape. + ***********************************************************************/ + private JSONObject request() + { + return (new JSONObject().put("protocol", Protocol.VERSION).put("type", "COMPILE").put("session", session).put("sequence", 1) + .put("requestId", "fixture").put("request", NodeColdCompilerTest.request())); + } + + /*********************************************************************** + ** Bound length-prefixed bytes over TLS. + ***********************************************************************/ + private void send(JSONObject value) throws Exception + { + wire.write(output, value); + } + + /*********************************************************************** + ** Strict parser checks duplicate fields, limits and EOF. + ***********************************************************************/ + private JSONObject read() throws Exception + { + return (wire.read(input)); + } + + /*********************************************************************** + ** Close only this fixture socket. + ***********************************************************************/ + @Override + public void close() throws IOException + { + socket.close(); + } + } +} diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/ExtismAbiTest.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/ExtismAbiTest.java new file mode 100644 index 0000000..da5bfad --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/ExtismAbiTest.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import com.sun.jna.Memory; +import com.sun.jna.Native; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/******************************************************************************* + ** Exercise JNA's actual public C value representation without executing Wasm. + *******************************************************************************/ +class ExtismAbiTest +{ + /*************************************************************************** + ** The pinned64-bit header declares enum at0, eight-byte union at8, size16. + ***************************************************************************/ + @Test + void actualValueStructureReadsAndWritesI64() + { + assertEquals(8, Native.POINTER_SIZE); + try(Memory memory = new Memory(16)) + { + memory.clear(); + memory.setInt(0, 1); + memory.setLong(8, 0x1122334455667788L); + ExtismApi.Value value = new ExtismApi.Value(memory); + assertEquals(16, value.size()); + assertEquals(1, value.type); + assertEquals(0x1122334455667788L, value.value.i64); + value.value.i64 = 123; + value.write(); + assertEquals(123, memory.getLong(8)); + } + } + + /*************************************************************************** + ** A mismatched operator identity refuses before native library loading. + ***************************************************************************/ + @Test + void wrongIdentityDoesNotLoadAnyNativeCode(@TempDir Path directory) throws Exception + { + Path library = Files.write(directory.resolve("not-a-library.bin"), new byte[] {1, 2, 3}); + Path wasm = Files.write(directory.resolve("not-wasm.bin"), new byte[] {4, 5, 6}); + IOException failure = assertThrows(IOException.class, () -> new JavyExtismRuntime(library, "0".repeat(64), wasm, + "0".repeat(64), 64L * 1024 * 1024, 100)); + assertEquals("NATIVE_IDENTITY", failure.getMessage()); + } +} diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NativeCompilerQualification.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NativeCompilerQualification.java new file mode 100644 index 0000000..3cfb8a7 --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NativeCompilerQualification.java @@ -0,0 +1,213 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicReference; +import org.json.JSONArray; +import org.json.JSONObject; + +/******************************************************************************* + ** Root-run fixed compiler corpus. Compiles trusted source only, never evaluates + ** emitted JavaScript. This does not qualify hostile containment or Kubernetes. + *******************************************************************************/ +public final class NativeCompilerQualification +{ + private static final String TOOLCHAIN = "typescript-6.0.3-node-24"; + + /*************************************************************************** + ** Utility qualification entrypoint. + ***************************************************************************/ + private NativeCompilerQualification() {} + + /*************************************************************************** + ** Exact operator paths/hashes only; temporary fixture certificate never logged. + ***************************************************************************/ + public static void main(String[] args) throws Exception + { + if(args.length != 4) + { + throw new IOException("Expected node-path node-sha256 compiler-root manifest-sha256"); + } + CompilerImage image = new CompilerImage(Path.of(args[0]), args[1], Path.of(args[2]), args[3], 256); + Path temporary = Files.createTempDirectory("qqq-trusted-cold-compiler-"); + try + { + TlsIdentity tls = TlsIdentity.create(temporary); + AtomicReference failure = new AtomicReference<>(); + try(var compiler = new NodeColdCompiler(image, 1, 10000, 20000); + var server = new CompilerServer(new CompilerServer.Configuration(tls.context(), new InetSocketAddress("127.0.0.1", 0), + Map.of(tls.fingerprint(), "fixture"), "trusted-compiler-fixture", TOOLCHAIN, "fixture-only", "fixture-only", 1, 20000), compiler)) + { + Thread listener = Thread.ofPlatform().daemon().start(() -> + { + try + { + server.serve(); + } + catch(Exception exception) + { + failure.set(exception); + } + }); + try + { + JSONObject first = exchange(tls, server.port(), request(Map.of( + "src/value.ts", "export const value: string = 'caf\\u00e9';", + "src/main.ts", "import type { Context, Qqq } from '@qqq/sdk'; import { value } from './value.js'; const top = await Promise.resolve(value); export async function main(context: Context, qqq: Qqq): Promise { return { top, event: context.event, rows: await qqq.query('purchase_order', ['id'], [], 1) }; }")), true); + JSONObject bundle = new JSONObject(first.getString("bundle")); + require("qqq-esm/1".equals(bundle.getString("format")), "BUNDLE_FORMAT"); + require("src/main.js".equals(bundle.getString("entryPoint")), "EMITTED_ENTRY"); + require(bundle.getJSONArray("files").length() == 2, "STATIC_MODULES"); + require(first.getString("sourceMap").contains("src/main.ts"), "SOURCE_MAP"); + exchange(tls, server.port(), request(Map.of("src/main.ts", "export const wrong: number = 'not a number';")), false); + exchange(tls, server.port(), request(Map.of("src/main.ts", "import fs from 'node:fs'; export const wrong = fs;")), false); + JSONObject repeated = exchange(tls, server.port(), request(Map.of("src/main.ts", + "export async function main(): Promise<{ action: 'ALLOW' }> { return { action: 'ALLOW' }; }")), true); + require(!repeated.getString("bundleSha256").equals(first.getString("bundleSha256")), "ARTIFACT_IDENTITY"); + System.out.println(new JSONObject().put("fixture", "trusted-cold-compiler-tls/1").put("compilations", 4) + .put("strictSuccesses", 2).put("strictRefusals", 2).put("retirements", 4) + .put("emittedJavaScriptExecuted", false).put("nodeSha256", args[1]).put("manifestSha256", args[3])); + } + finally + { + server.close(); + listener.join(2000); + require(!listener.isAlive() && failure.get() == null, "LISTENER_DRAIN"); + } + } + } + finally + { + try(var paths = Files.walk(temporary)) + { + for(Path path : paths.sorted(Comparator.reverseOrder()).toList()) + { + Files.delete(path); + } + } + } + } + + /*************************************************************************** + ** Same length-prefixed source identity as the existing compiler/host contract. + ***************************************************************************/ + private static JSONObject request(Map sources) throws Exception + { + JSONArray files = new JSONArray(); + ByteArrayOutputStream canonical = new ByteArrayOutputStream(); + try(var lengths = new DataOutputStream(canonical)) + { + for(var entry : new TreeMap<>(sources).entrySet()) + { + byte[] name = entry.getKey().getBytes(StandardCharsets.UTF_8); + byte[] source = entry.getValue().getBytes(StandardCharsets.UTF_8); + lengths.writeInt(name.length); + lengths.writeInt(source.length); + lengths.write(name); + lengths.write(source); + files.put(new JSONObject().put("path", entry.getKey()).put("content", entry.getValue())); + } + } + return (new JSONObject().put("protocol", Protocol.WORKER).put("type", "compile").put("requestId", UUID.randomUUID().toString()) + .put("toolchainId", TOOLCHAIN).put("strict", true).put("entryPoint", "src/main.ts") + .put("sourceSha256", Protocol.hash(canonical.toByteArray())).put("files", files)); + } + + /*************************************************************************** + ** Real mTLS endpoint transcript; no direct invocation or broker substitution. + ***************************************************************************/ + private static JSONObject exchange(TlsIdentity tls, int port, JSONObject request, boolean success) throws Exception + { + try(SSLSocket socket = (SSLSocket) tls.context().getSocketFactory().createSocket("localhost", port)) + { + socket.setTcpNoDelay(true); + socket.setSoTimeout(30000); + SSLParameters settings = socket.getSSLParameters(); + settings.setEndpointIdentificationAlgorithm("HTTPS"); + settings.setProtocols(new String[] {"TLSv1.3"}); + socket.setSSLParameters(settings); + socket.startHandshake(); + DataInputStream input = new DataInputStream(socket.getInputStream()); + DataOutputStream output = new DataOutputStream(socket.getOutputStream()); + Protocol wire = new Protocol(); + String nonce = UUID.randomUUID().toString(); + wire.write(output, new JSONObject().put("protocol", Protocol.VERSION).put("type", "HELLO").put("namespace", "fixture") + .put("profile", "trusted-compiler-fixture").put("toolchain", TOOLCHAIN).put("evidence", "fixture-only").put("nonce", nonce)); + JSONObject accepted = wire.read(input); + Protocol.equal(accepted, "type", "ACCEPT"); + Protocol.equal(accepted, "nonce", nonce); + String session = accepted.getString("session"); + String id = request.getString("requestId"); + wire.reset(); + wire.write(output, new JSONObject().put("protocol", Protocol.VERSION).put("type", "COMPILE").put("session", session) + .put("sequence", 1).put("requestId", id).put("request", request)); + JSONObject response = wire.read(input); + Protocol.keys(response, "protocol", "type", "session", "sequence", "requestId", "message"); + Protocol.equal(response, "protocol", Protocol.VERSION); + Protocol.equal(response, "type", "RESULT"); + Protocol.equal(response, "session", session); + Protocol.equal(response, "requestId", id); + Protocol.number(response, "sequence", 1, 1); + JSONObject result = response.getJSONObject("message"); + CompilerServer.validateResult(result, id, TOOLCHAIN); + require(result.getBoolean("success") == success, "COMPILE_OUTCOME"); + if(!success) + { + require(!result.getJSONArray("diagnostics").isEmpty(), "STRICT_DIAGNOSTICS"); + } + JSONObject retired = wire.read(input); + Protocol.keys(retired, "protocol", "type", "session", "sequence", "requestId", "retired"); + Protocol.equal(retired, "protocol", Protocol.VERSION); + Protocol.equal(retired, "type", "RETIRED"); + Protocol.equal(retired, "session", session); + Protocol.equal(retired, "requestId", id); + Protocol.number(retired, "sequence", 1, 1); + require(Boolean.TRUE.equals(retired.opt("retired")), "RETIREMENT"); + require(input.read() == -1, "EXTRA_OUTPUT"); + return (result); + } + } + + /*************************************************************************** + ** Report only fixed fixture labels, never source/diagnostics/credential values. + ***************************************************************************/ + private static void require(boolean condition, String code) throws IOException + { + if(!condition) + { + throw new IOException(code); + } + } +} \ No newline at end of file diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NativeSupervisorQualification.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NativeSupervisorQualification.java new file mode 100644 index 0000000..2757a53 --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NativeSupervisorQualification.java @@ -0,0 +1,474 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.json.JSONArray; +import org.json.JSONObject; + +/******************************************************************************* + ** Explicit root-owned native test entrypoint, excluded from ordinary unit tests. + ** Fixed compiler fixtures only; never establishes production qualification. + *******************************************************************************/ +public final class NativeSupervisorQualification +{ + private static final long MEMORY = 128L * 1024 * 1024; + + /*************************************************************************** + ** Utility qualification entrypoint. + ***************************************************************************/ + private NativeSupervisorQualification() {} + + /*************************************************************************** + ** Exact library/hash/Wasm/hash/fixture-directory, with no secret arguments. + ***************************************************************************/ + public static void main(String[] args) throws Exception + { + if(args.length != 5) + { + throw new IOException("QUALIFICATION_ARGUMENTS"); + } + Path library = Path.of(args[0]); + Path wasm = Path.of(args[2]); + Path fixtures = Path.of(args[4]); + if(!fixtures.isAbsolute()) + { + throw new IOException("FIXTURE_PATH"); + } + JSONObject state = fixture(fixtures, "state"); + JSONObject unicode = fixture(fixtures, "unicode"); + JSONObject plain = fixture(fixtures, "plain"); + JSONObject scriptError = fixture(fixtures, "script-error"); + Path ephemeral = Files.createTempDirectory("qqq-supervisor-native-"); + TlsIdentity identity = TlsIdentity.create(ephemeral); + AtomicReference listenerFailure = new AtomicReference<>(); + JSONArray usages = new JSONArray(); + int brokerCalls; + int instances; + try(var runtime = new Counted(new JavyExtismRuntime(library, args[1], wasm, args[3], MEMORY, 1_000_000_000L)); + var server = new SupervisorServer(new SupervisorServer.Configuration(identity.context(), new InetSocketAddress("127.0.0.1", 0), + Map.of(identity.fingerprint(), "native-fixture"), "native-fixture", "typescript-6.0.3-node-24", "trusted-fixture-only", "trusted-fixture-only", + args[3], 1, 2, 1, 8, 16L * 1024 * 1024, MEMORY, 10_000), runtime)) + { + Thread thread = Thread.ofPlatform().daemon().start(() -> + { + try + { + server.serve(); + } + catch(IOException failure) + { + listenerFailure.set(failure); + } + }); + try(var client = new Client(identity, server.port(), runtime)) + { + String ready = client.prepare(state); + Set randoms = new HashSet<>(); + for(int i = 0; i < 4; i++) + { + String actor = "fixture-" + i; + long earliest = System.currentTimeMillis(); + JSONObject message = client.invoke(state, ready, actor, MEMORY); + equal("OK", message.getString("outcome")); + JSONObject result = message.getJSONObject("result"); + equal(1, result.getInt("count")); + equal(73, result.getInt("id")); + equal(actor, result.getString("actor")); + long stamp = result.getLong("stamp"); + require(stamp >= earliest && stamp <= System.currentTimeMillis(), "FRESH_CLOCK"); + double random = result.getDouble("random"); + require(random >= 0 && random < 1, "RANDOM_RANGE"); + randoms.add(random); + usages.put(message.getJSONObject("usage")); + } + require(randoms.size() > 1, "FRESH_RANDOM"); + require(unicode.getString("bundle").getBytes(StandardCharsets.UTF_8).length > 1024 * 1024, "UNICODE_FIXTURE_BOUNDARY"); + JSONObject unicodeMessage = client.invoke(unicode, client.prepare(unicode), "unicode", MEMORY); + equal("café 漢字 Привет", unicodeMessage.getJSONObject("result").getString("label")); + usages.put(unicodeMessage.getJSONObject("usage")); + String errorReady = client.prepare(scriptError); + JSONObject error = client.invoke(scriptError, errorReady, "error", MEMORY); + equal("SCRIPT_ERROR", error.getString("outcome")); + require(!error.toString().contains("fixed trusted fixture"), "ERROR_REDACTION"); + usages.put(error.getJSONObject("usage")); + JSONObject normal = client.invoke(plain, client.prepare(plain), "plain", MEMORY); + equal("plain", normal.getJSONObject("result").getString("label")); + usages.put(normal.getJSONObject("usage")); + int before = runtime.opened.get(); + boolean refused = false; + try + { + client.invoke(state, ready, "memory-mismatch", MEMORY / 2); + } + catch(IOException expected) + { + refused = true; + } + require(refused && before == runtime.opened.get(), "MEMORY_PROFILE_REFUSAL"); + brokerCalls = client.brokerCalls; + } + finally + { + server.close(); + thread.join(1000); + require(!thread.isAlive() && listenerFailure.get() == null, "LISTENER_CLEANUP"); + } + equal(runtime.opened.get(), runtime.freed.get()); + equal(1, runtime.compiled.get()); + instances = runtime.freed.get(); + } + JSONObject prepare = new JSONObject().put("operation", "prepare").put("runtimeSha256", args[3]) + .put("artifactSha256", plain.getString("sha256")).put("bundle", plain.getString("bundle")); + byte[] input = prepare.toString().getBytes(StandardCharsets.UTF_8); + try(var runtime = new JavyExtismRuntime(library, args[1], wasm, args[3], MEMORY, 1_000_000_000L); + var slot = runtime.createSlot(); var invocation = slot.open(NativeSupervisorQualification::denyBroker)) + { + Protocol.parse(invocation.call(input)); + boolean refused = false; + try + { + invocation.call(input); + } + catch(IOException expected) + { + refused = "INVOCATION_STATE".equals(expected.getMessage()); + } + require(refused, "ONE_CALL_PER_INSTANCE"); + } + boolean fuelRefused = false; + try(var runtime = new JavyExtismRuntime(library, args[1], wasm, args[3], MEMORY, 1); + var slot = runtime.createSlot()) + { + try(var invocation = slot.open(NativeSupervisorQualification::denyBroker)) + { + invocation.call(input); + } + catch(IOException expected) + { + fuelRefused = "INSTANCE_CREATE".equals(expected.getMessage()) || "GUEST_FAILED".equals(expected.getMessage()); + } + } + require(fuelRefused, "NORMAL_FIXTURE_LOW_FUEL_REFUSAL"); + Files.deleteIfExists(ephemeral.resolve("identity.p12")); + Files.deleteIfExists(ephemeral); + System.out.println(new JSONObject().put("scope", "fixed trusted native supervisor corpus; not production qualification") + .put("runtimeSha256", args[3]).put("nativeSha256", args[1]).put("brokerCalls", brokerCalls).put("closedInstances", instances) + .put("usageSamples", usages).put("lowFuelRefused", true).put("productionQualified", false)); + } + + /*************************************************************************** + ** Direct preparation fixtures cannot access any application capability. + ***************************************************************************/ + private static byte[] denyBroker(byte[] ignored) throws IOException + { + throw new IOException("UNEXPECTED_BROKER"); + } + + /*************************************************************************** + ** Read only fixed named compiler artifacts with bounded exact identities. + ***************************************************************************/ + private static JSONObject fixture(Path root, String name) throws Exception + { + Path file = root.resolve(name + ".json"); + require(Files.size(file) <= Protocol.MAX_FRAME, "FIXTURE_BYTES"); + JSONObject value = Protocol.parse(Files.readAllBytes(file)); + equal(name, value.getString("name")); + equal("main.ts", value.getString("entryPoint")); + equal("typescript-6.0.3-node-24", value.getString("toolchainId")); + equal(value.getString("sha256"), Protocol.hash(value.getString("bundle").getBytes(StandardCharsets.UTF_8))); + return (value); + } + + /*************************************************************************** + ** Fixed assertion labels avoid printing broker/source/config contents. + ***************************************************************************/ + private static void require(boolean condition, String code) throws IOException + { + if(!condition) + { + throw new IOException(code); + } + } + + /*************************************************************************** + ** Do not echo differing payloads when a qualification assertion fails. + ***************************************************************************/ + private static void equal(Object expected, Object actual) throws IOException + { + require(expected.equals(actual), "FIXTURE_VALUE"); + } + + /*************************************************************************** + ** Observe real native lifecycle completion; no stub result or fake usage. + ***************************************************************************/ + private static final class Counted implements GuestRuntime + { + private final GuestRuntime delegate; + private final AtomicInteger compiled = new AtomicInteger(); + private final AtomicInteger opened = new AtomicInteger(); + private final AtomicInteger freed = new AtomicInteger(); + + /*********************************************************************** + ** Only a real pinned runtime is passed by this fixed entrypoint. + ***********************************************************************/ + private Counted(GuestRuntime delegate) + { + this.delegate = delegate; + } + + /*********************************************************************** + ** Forward actual runtime content identity. + ***********************************************************************/ + @Override + public String sha256() + { + return (delegate.sha256()); + } + + /*********************************************************************** + ** Count actual engine compilation and delegate every operation. + ***********************************************************************/ + @Override + public Slot createSlot() throws Exception + { + Slot slot = delegate.createSlot(); + compiled.incrementAndGet(); + return (new Slot() + { + /***************************************************************** + ** New native instance, with the actual current broker callback. + *****************************************************************/ + @Override + public Invocation open(Broker broker) throws Exception + { + Invocation invocation = slot.open(broker); + opened.incrementAndGet(); + return (new Invocation() + { + /*********************************************************** + ** Execute the fixed native export through the real binding. + ***********************************************************/ + @Override + public byte[] call(byte[] input) throws Exception + { + return (invocation.call(input)); + } + + /*********************************************************** + ** Forward only this native instance's cancellation handle. + ***********************************************************/ + @Override + public void cancel() + { + invocation.cancel(); + } + + /*********************************************************** + ** Count only after actual native Store destruction returns. + ***********************************************************/ + @Override + public void close() throws Exception + { + invocation.close(); + freed.incrementAndGet(); + } + }); + } + + /***************************************************************** + ** Release actual compiled engine after all active stores close. + *****************************************************************/ + @Override + public void close() throws Exception + { + slot.close(); + } + }); + } + + /*********************************************************************** + ** Delegate native owner cleanup. + ***********************************************************************/ + @Override + public void close() throws Exception + { + delegate.close(); + } + } + + /*************************************************************************** + ** Actual TLS peer, independent of the production provider implementation. + ***************************************************************************/ + private static final class Client implements AutoCloseable + { + private final SSLSocket socket; + private final DataInputStream input; + private final DataOutputStream output; + private final Protocol wire = new Protocol(); + private final Counted runtime; + private final String session; + private long sequence; + private int brokerCalls; + + /*********************************************************************** + ** Exact ephemeral client certificate and verified localhost server name. + ***********************************************************************/ + private Client(TlsIdentity identity, int port, Counted runtime) throws Exception + { + this.runtime = runtime; + socket = (SSLSocket) identity.context().getSocketFactory().createSocket("localhost", port); + socket.setTcpNoDelay(true); + SSLParameters parameters = socket.getSSLParameters(); + parameters.setEndpointIdentificationAlgorithm("HTTPS"); + parameters.setProtocols(new String[] {"TLSv1.3"}); + socket.setSSLParameters(parameters); + socket.setSoTimeout(15_000); + socket.startHandshake(); + input = new DataInputStream(socket.getInputStream()); + output = new DataOutputStream(socket.getOutputStream()); + wire.write(output, new JSONObject().put("protocol", Protocol.VERSION).put("type", "HELLO").put("namespace", "native-fixture") + .put("profile", "native-fixture").put("toolchain", "typescript-6.0.3-node-24").put("evidence", "trusted-fixture-only").put("nonce", UUID.randomUUID().toString())); + JSONObject accept = wire.read(input); + equal("ACCEPT", accept.getString("type")); + session = accept.getString("session"); + } + + /*********************************************************************** + ** Actual cold preparation retains original TypeScript entry identity. + ***********************************************************************/ + private String prepare(JSONObject fixture) throws Exception + { + JSONObject request = envelope("PREPARE").put("sha256", fixture.getString("sha256")) + .put("entryPoint", fixture.getString("entryPoint")).put("bundle", fixture.getString("bundle")); + wire.write(output, request); + JSONObject ready = response(request, "READY"); + equal(fixture.getString("entryPoint"), ready.getString("entryPoint")); + equal(fixture.getString("sha256"), ready.getString("sha256")); + retired(request); + return (ready.getString("readyId")); + } + + /*********************************************************************** + ** Fresh token/context, real broker and host-reported usage on every call. + ***********************************************************************/ + private JSONObject invoke(JSONObject fixture, String ready, String actor, long memory) throws Exception + { + JSONObject outer = envelope("INVOKE").put("readyId", ready); + String token = UUID.randomUUID().toString(); + JSONObject request = new JSONObject().put("protocol", Protocol.WORKER).put("type", "execute").put("requestId", outer.getString("requestId")) + .put("artifactId", "fixed-fixture").put("artifactSha256", fixture.getString("sha256")).put("entryPoint", fixture.getString("entryPoint")) + .put("event", "CONTEXT").put("payload", new JSONObject().put("actor", actor)).put("capabilityToken", token) + .put("limits", new JSONObject().put("cpuMillis", 5000).put("wallMillis", 10_000).put("memoryBytes", memory).put("outputBytes", 65536).put("brokerCalls", 2)); + wire.write(output, outer.put("request", request)); + JSONObject frame = wire.read(input); + while("BROKER".equals(frame.getString("type"))) + { + correlated(frame, outer); + JSONObject broker = frame.getJSONObject("message"); + equal(token, broker.getString("token")); + equal(outer.getString("requestId"), broker.getString("requestId")); + equal("purchaseOrder", broker.getString("table")); + equal("[\"id\"]", broker.getJSONArray("fields").toString()); + require(++brokerCalls <= 4, "BROKER_CALL_COUNT"); + JSONObject answer = new JSONObject(outer.toString()); + answer.remove("request"); + answer.remove("readyId"); + answer.put("type", "BROKER_REPLY").put("message", new JSONObject().put("protocol", Protocol.WORKER).put("type", "brokerResult") + .put("ok", true).put("rows", new JSONArray().put(new JSONObject().put("id", 73))).put("truncated", false)); + wire.write(output, answer); + frame = wire.read(input); + } + correlated(frame, outer); + equal("RESULT", frame.getString("type")); + JSONObject message = frame.getJSONObject("message"); + JSONObject usage = message.getJSONObject("usage"); + require(usage.getInt("cpuMillis") > 0 && usage.getInt("cpuMillis") <= 5000, "ACTUAL_CPU"); + require(usage.getInt("wallMillis") >= usage.getInt("cpuMillis") && usage.getInt("wallMillis") <= 10_000, "ACTUAL_WALL"); + retired(outer); + return (message); + } + + /*********************************************************************** + ** Retirement is observed only after the counted real native close. + ***********************************************************************/ + private void retired(JSONObject request) throws Exception + { + require(response(request, "RETIRED").getBoolean("retired"), "RETIREMENT"); + equal(runtime.opened.get(), runtime.freed.get()); + } + + /*********************************************************************** + ** Exact reply binding independent from guest-controlled result contents. + ***********************************************************************/ + private JSONObject response(JSONObject request, String type) throws Exception + { + JSONObject value = wire.read(input); + correlated(value, request); + equal(type, value.getString("type")); + return (value); + } + + /*********************************************************************** + ** Verify every server control frame's operation identity. + ***********************************************************************/ + private void correlated(JSONObject frame, JSONObject request) throws Exception + { + equal(Protocol.VERSION, frame.getString("protocol")); + equal(session, frame.getString("session")); + equal(request.getLong("sequence"), frame.getLong("sequence")); + equal(request.getString("requestId"), frame.getString("requestId")); + } + + /*********************************************************************** + ** Start a new bounded transcript with a distinct operation identity. + ***********************************************************************/ + private JSONObject envelope(String type) + { + wire.reset(); + return (new JSONObject().put("protocol", Protocol.VERSION).put("type", type).put("session", session) + .put("sequence", ++sequence).put("requestId", UUID.randomUUID().toString())); + } + + /*********************************************************************** + ** Close only this test-owned peer. + ***********************************************************************/ + @Override + public void close() throws IOException + { + socket.close(); + } + } +} diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NodeColdCompilerTest.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NodeColdCompilerTest.java new file mode 100644 index 0000000..62d8464 --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/NodeColdCompilerTest.java @@ -0,0 +1,166 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/******************************************************************************* + ** Actual child-process/CPU/pipe tests with a fixed non-compiling Java fixture. + *******************************************************************************/ +class NodeColdCompilerTest +{ + /*************************************************************************** + ** A normal return requires actual process death; repeated calls use new PIDs. + ***************************************************************************/ + @Test + void realChildRetiresBeforeReturningAndCanBeRepeated(@TempDir Path directory) throws Exception + { + try(var compiler = child(directory, "normal", 5000)) + { + long prior = -1; + for(int i = 0; i < 2; i++) + { + JSONObject result = compiler.compile(request()); + CompilerServer.validateResult(result, "fixture", "typescript-6.0.3-node-24"); + assertEquals(true, result.getBoolean("success")); + long pid = Long.parseLong(Files.readString(directory.resolve("pid"))); + assertFalse(pid == prior); + assertFalse(ProcessHandle.of(pid).map(ProcessHandle::isAlive).orElse(false)); + prior = pid; + } + } + } + + /*************************************************************************** + ** Partial/extra/early/stderr frames all refuse and leave no living owned child. + ***************************************************************************/ + @Test + void malformedTranscriptsNeverReturnAndChildrenAreReaped(@TempDir Path directory) throws Exception + { + for(String mode : List.of("partial", "extra", "early", "stderr")) + { + try(var compiler = child(directory, mode, 5000)) + { + assertThrows(Exception.class, () -> compiler.compile(request())); + } + long pid = Long.parseLong(Files.readString(directory.resolve("pid"))); + assertFalse(ProcessHandle.of(pid).map(ProcessHandle::isAlive).orElse(false)); + } + } + + /*************************************************************************** + ** Bounded ordinary wait exceeds wall budget and must be killed/reaped. + ***************************************************************************/ + @Test + void wallLimitAndClosedServiceRefuse(@TempDir Path directory) throws Exception + { + try(var compiler = child(directory, "wait", 500)) + { + assertThrows(Exception.class, () -> compiler.compile(request())); + long pid = Long.parseLong(Files.readString(directory.resolve("pid"))); + assertFalse(ProcessHandle.of(pid).map(ProcessHandle::isAlive).orElse(false)); + compiler.close(); + assertThrows(Exception.class, () -> compiler.compile(request())); + } + } + + /*************************************************************************** + ** A second call cannot queue behind the single occupied child-process slot. + ***************************************************************************/ + @Test + void occupiedCapacityRefusesWithoutStartingAnotherChild(@TempDir Path directory) throws Exception + { + try(var compiler = child(directory, "wait", 5000); var executor = Executors.newSingleThreadExecutor()) + { + var running = executor.submit(() -> compiler.compile(request())); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while(!Files.exists(directory.resolve("pid")) && System.nanoTime() < deadline) + { + Thread.sleep(5); + } + long pid = Long.parseLong(Files.readString(directory.resolve("pid"))); + assertThrows(Exception.class, () -> compiler.compile(request())); + assertEquals(pid, Long.parseLong(Files.readString(directory.resolve("pid")))); + CompilerServer.validateResult(running.get(5, TimeUnit.SECONDS), "fixture", "typescript-6.0.3-node-24"); + assertFalse(ProcessHandle.of(pid).map(ProcessHandle::isAlive).orElse(false)); + } + } + + /*************************************************************************** + ** The same accounting validator used by the watchdog rejects absent samples. + ***************************************************************************/ + @Test + void unavailableCpuNeverBecomesSyntheticUsage() throws Exception + { + assertThrows(Exception.class, () -> NodeColdCompiler.requireCpuNanos(Optional.empty())); + assertThrows(Exception.class, () -> NodeColdCompiler.requireCpuNanos(Optional.of(Duration.ofNanos(-1)))); + assertEquals(123, NodeColdCompiler.requireCpuNanos(Optional.of(Duration.ofNanos(123)))); + } + + /*************************************************************************** + ** Reproduce OS sample disappearance after normal exit without author code. + ***************************************************************************/ + @Test + void accountingExitRaceRetainsOnlyVerifiedPriorLiveCpu() throws Exception + { + assertEquals(123, NodeColdCompiler.accountCpu(Optional.empty(), 123, true)); + assertEquals(123, NodeColdCompiler.accountCpu(Optional.of(Duration.ZERO), 123, true)); + assertEquals(200, NodeColdCompiler.accountCpu(Optional.of(Duration.ofNanos(200)), 123, true)); + assertThrows(IOException.class, () -> NodeColdCompiler.accountCpu(Optional.empty(), 123, false)); + assertThrows(IOException.class, () -> NodeColdCompiler.accountCpu(Optional.of(Duration.ZERO), 123, false)); + assertThrows(IOException.class, () -> NodeColdCompiler.accountCpu(Optional.empty(), -1, true)); + assertThrows(IOException.class, () -> NodeColdCompiler.accountCpu(Optional.of(Duration.ofNanos(-1)), 123, true)); + } + + /*************************************************************************** + ** Fixed current JDK child command: no source-generated shell/arguments. + ***************************************************************************/ + private NodeColdCompiler child(Path directory, String mode, int wall) + { + Path executable = Path.of(System.getProperty("java.home"), "bin", System.getProperty("os.name").startsWith("Windows") ? "java.exe" : "java"); + return (new NodeColdCompiler(List.of(executable.toString(), "-cp", System.getProperty("java.class.path"), CompileChildFixture.class.getName(), mode, + directory.resolve("pid").toString()), directory, 1, 5000, wall)); + } + + /*************************************************************************** + ** Fixed protocol input, never evaluated by the test child. + ***************************************************************************/ + static JSONObject request() + { + return (new JSONObject().put("protocol", Protocol.WORKER).put("type", "compile").put("requestId", "fixture").put("strict", true) + .put("toolchainId", "typescript-6.0.3-node-24").put("entryPoint", "main.ts").put("sourceSha256", "a".repeat(64)) + .put("files", new JSONArray().put(new JSONObject().put("path", "main.ts").put("content", "fixed fixture")))); + } +} diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/SupervisorServerTest.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/SupervisorServerTest.java new file mode 100644 index 0000000..e8026d5 --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/SupervisorServerTest.java @@ -0,0 +1,740 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import org.json.JSONArray; +import org.json.JSONObject; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/******************************************************************************* + ** Real mTLS server lifecycle with a non-executing trusted GuestRuntime fixture. + ** No test result qualifies Extism/Javy, containment or Kubernetes performance. + *******************************************************************************/ +class SupervisorServerTest +{ + @TempDir + static Path directory; + private static TlsIdentity tls; + private static final String BUNDLE = "{\"format\":\"qqq-esm/1\",\"entryPoint\":\"src/index.js\",\"files\":[{\"path\":\"src/index.js\",\"content\":\"export async function main(context, qqq) { return {action: 'ALLOW'}; }\"}]}"; + + /*************************************************************************** + ** One ephemeral certificate, mapped by exact DER digest rather than its name. + ***************************************************************************/ + @BeforeAll + static void identity() throws Exception + { + tls = TlsIdentity.create(directory); + } + + /*************************************************************************** + ** Common engine is shared, but prepare and each run close a distinct instance. + ***************************************************************************/ + @Test + void prepareThenTwoCallsRetireFreshInstances() throws Exception + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + String ready = client.prepare(); + assertEquals(1, runtime.compiled.get()); + assertEquals(1, runtime.created.get()); + assertEquals(1, runtime.freed.get()); + for(String token : List.of("first-token", "second-token")) + { + JSONObject result = client.invoke(ready, token, 1000, 2000); + assertEquals("OK", result.getString("outcome")); + assertTrue(result.getJSONObject("usage").getInt("cpuMillis") >= 0); + assertTrue(result.getJSONObject("usage").getInt("wallMillis") < 2000); + assertFalse(result.toString().contains("987654"), "Guest wall hint is not authoritative usage"); + assertEquals(runtime.created.get(), runtime.freed.get(), "Retirement must follow actual close"); + } + assertEquals(3, runtime.created.get()); + assertEquals(1, runtime.compiled.get()); + } + } + + /*************************************************************************** + ** A rejected peer that never reads TLS close alerts cannot retain an engine. + ***************************************************************************/ + @Test + void rejectedConnectedPeerDoesNotDelayCleanEngineRecovery() throws Exception + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint(), 10_000); var rejected = new Client(host.server.port(), "fixture")) + { + rejected.prepare(); + rejected.write(rejected.envelope("PING").put("unexpected", true)); + // Deliberately do not read or close this peer. Give the protocol thread + // a bounded scheduling window, far below its ten-second idle timeout. + Thread.sleep(500); + try(var next = new Client(host.server.port(), "fixture")) + { + String ready = next.prepare(); + assertEquals("OK", next.invoke(ready, "fresh-token", 1000, 2000).getString("outcome")); + } + assertEquals(1, runtime.compiled.get()); + assertEquals(runtime.created.get(), runtime.freed.get()); + } + } + + /*************************************************************************** + ** Failed startup retires every acquired engine even when earlier cleanup fails. + ***************************************************************************/ + @Test + void startupPreservesPrimaryFailureAndClosesEveryAcquiredSlot() throws Exception + { + for(boolean sameFailure : List.of(false, true)) + { + IOException primary = new IOException("STARTUP"); + IOException cleanup = sameFailure ? primary : new IOException("CLEANUP"); + AtomicInteger acquired = new AtomicInteger(); + AtomicInteger closed = new AtomicInteger(); + GuestRuntime runtime = new GuestRuntime() + { + /*************************************************************************** + ** Fixed identity is only a trusted protocol fixture. + ***************************************************************************/ + @Override + public String sha256() + { + return ("a".repeat(64)); + } + + /*************************************************************************** + ** Fail after two successful acquisitions, before listener binding. + ***************************************************************************/ + @Override + public Slot createSlot() throws Exception + { + int ordinal = acquired.incrementAndGet(); + if(ordinal == 3) + { + throw primary; + } + return (new Slot() + { + /*************************************************************************** + ** Startup never opens an invocation. + ***************************************************************************/ + @Override + public Invocation open(Broker broker) + { + throw new AssertionError("No invocation during failed startup"); + } + + /*************************************************************************** + ** The first cleanup throws; the second must still run. + ***************************************************************************/ + @Override + public void close() throws Exception + { + closed.incrementAndGet(); + if(ordinal == 1) + { + throw cleanup; + } + } + }); + } + + /*************************************************************************** + ** The caller retains runtime ownership when construction fails. + ***************************************************************************/ + @Override + public void close() {} + }; + var config = new SupervisorServer.Configuration(tls.context(), new InetSocketAddress("127.0.0.1", 0), + Map.of(tls.fingerprint(), "fixture"), "profile", "toolchain", "fixture-evidence", "fixture-evidence", runtime.sha256(), + 3, 3, 1, 2, 8192, 64L * 1024 * 1024, 2000); + assertSame(primary, assertThrows(IOException.class, () -> new SupervisorServer(config, runtime))); + assertEquals(2, closed.get()); + assertEquals(sameFailure ? 0 : 1, primary.getSuppressed().length); + if(!sameFailure) + { + assertSame(cleanup, primary.getSuppressed()[0]); + } + } + } + + /*************************************************************************** + ** A failed call loses its session, not the only engine after proven cleanup. + ***************************************************************************/ + @Test + void cleanedIoFailureAllowsFreshSessionOnTheOnlyEngine() throws Exception + { + for(boolean prepareFailure : List.of(true, false)) + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint())) + { + try(var client = new Client(host.server.port(), "fixture")) + { + String ready = prepareFailure ? null : client.prepare(); + runtime.failWithIo = true; + if(prepareFailure) + { + assertThrows(IOException.class, client::prepare); + } + else + { + assertThrows(IOException.class, () -> client.invoke(ready, "failed", 1000, 2000)); + } + assertEquals(runtime.created.get(), runtime.freed.get()); + } + runtime.failWithIo = false; + try(var next = new Client(host.server.port(), "fixture")) + { + String ready = next.prepare(); + assertEquals("OK", next.invoke(ready, "next", 1000, 2000).getString("outcome")); + assertEquals(1, runtime.compiled.get(), "Only the compiled runtime is reused, never a guest instance"); + assertEquals(runtime.created.get(), runtime.freed.get()); + } + } + } + } + + /*************************************************************************** + ** Invalid prepared data is refused even though its destroyed guest is reusable. + ***************************************************************************/ + @Test + void malformedPreparedResultDoesNotPermanentlyConsumeCapacity() throws Exception + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint())) + { + runtime.corruptPrepared = true; + try(var client = new Client(host.server.port(), "fixture")) + { + assertThrows(IOException.class, client::prepare); + } + assertEquals(1, runtime.freed.get()); + runtime.corruptPrepared = false; + try(var next = new Client(host.server.port(), "fixture")) + { + assertEquals("OK", next.invoke(next.prepare(), "next", 1000, 2000).getString("outcome")); + assertEquals(1, runtime.compiled.get()); + } + } + } + + /*************************************************************************** + ** Heartbeats keep the same cache epoch without starting or preparing a guest. + ***************************************************************************/ + @Test + void heartbeatPreservesReadinessWithoutGuestWork() throws Exception + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + client.ping(); + assertEquals(0, runtime.created.get()); + String ready = client.prepare(); + client.ping(); + assertEquals(1, runtime.created.get()); + client.invoke(ready, "first", 1000, 2000); + client.ping(); + assertEquals(2, runtime.created.get()); + client.invoke(ready, "second", 1000, 2000); + assertEquals(3, runtime.freed.get()); + assertEquals(1, runtime.compiled.get()); + } + } + + /*************************************************************************** + ** Heartbeats cannot carry capability/code data or weaken operation identity. + ***************************************************************************/ + @Test + void heartbeatRejectsExtraDataWrongSessionAndReplay() throws Exception + { + for(String invalid : List.of("capabilityToken", "session", "sequence")) + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + JSONObject request = client.envelope("PING"); + request.put(invalid, "sequence".equals(invalid) ? 0 : "invalid"); + client.write(request); + assertThrows(IOException.class, client::read); + assertEquals(0, runtime.created.get()); + } + } + } + + /*************************************************************************** + ** Certificate trust alone does not authorize an unconfigured namespace. + ***************************************************************************/ + @Test + void exactCertificateAndNamespaceAreRequired() throws Exception + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, "b".repeat(64))) + { + assertThrows(IOException.class, () -> new Client(host.server.port(), "fixture")); + assertEquals(0, runtime.created.get()); + } + try(var host = new Host(runtime, tls.fingerprint())) + { + assertThrows(IOException.class, () -> new Client(host.server.port(), "other")); + assertEquals(0, runtime.created.get()); + } + } + + /*************************************************************************** + ** Missing readiness never starts a guest, and warm-session slots are bounded. + ***************************************************************************/ + @Test + void readinessAndGlobalSlotsCannotBeBypassed() throws Exception + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + assertThrows(IOException.class, () -> new Client(host.server.port(), "fixture")); + assertThrows(IOException.class, () -> client.invoke("missing", "token", 1000, 2000)); + assertEquals(0, runtime.created.get()); + } + } + + /*************************************************************************** + ** No READY follows corrupt prepared data; no RETIRED follows cleanup failure. + ***************************************************************************/ + @Test + void integrityAndCloseFailuresNeverAttestRetirement() throws Exception + { + TrustedRuntime corrupt = new TrustedRuntime(); + corrupt.corruptPrepared = true; + try(var host = new Host(corrupt, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + assertThrows(IOException.class, client::prepare); + assertEquals(1, corrupt.freed.get()); + } + TrustedRuntime failed = new TrustedRuntime(); + try(var host = new Host(failed, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + String ready = client.prepare(); + failed.failClose = true; + assertThrows(IOException.class, () -> client.invoke(ready, "token", 1000, 2000)); + assertEquals(2, failed.created.get()); + assertEquals(1, failed.freed.get()); + host.expectExhaustion(); + assertThrows(IOException.class, () -> new Client(host.server.port(), "fixture"), "Uncertain engine cannot return to capacity"); + } + } + + /*************************************************************************** + ** Even an unchecked runtime Error closes the instance and retires its engine. + ***************************************************************************/ + @Test + void uncheckedFailureCannotReturnEngineToAdmission() throws Exception + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + String ready = client.prepare(); + runtime.failWithError = true; + assertThrows(IOException.class, () -> client.invoke(ready, "token", 1000, 2000)); + assertEquals(2, runtime.freed.get()); + host.expectExhaustion(); + assertThrows(IOException.class, () -> new Client(host.server.port(), "fixture")); + } + } + + /*************************************************************************** + ** Cache fullness refuses before creating a third preparation instance. + ***************************************************************************/ + @Test + void boundedCacheNeverEvictsPreparedCode() throws Exception + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + String first = client.prepare(); + client.prepare(); + client.invoke(first, "token", 1000, 2000); + assertThrows(IOException.class, client::prepare); + assertEquals(3, runtime.created.get()); + assertEquals(3, runtime.freed.get()); + } + } + + /*************************************************************************** + ** Wrong broker authority is denied before it reaches the application peer. + ***************************************************************************/ + @Test + void forgedBrokerAndWallTimeoutDiscardSession() throws Exception + { + for(boolean timeout : List.of(false, true)) + { + TrustedRuntime runtime = new TrustedRuntime(); + try(var host = new Host(runtime, tls.fingerprint()); var client = new Client(host.server.port(), "fixture")) + { + String ready = client.prepare(); + runtime.pending = timeout; + runtime.wrongToken = !timeout; + assertThrows(IOException.class, () -> client.invoke(ready, "token", 1000, timeout ? 20 : 2000)); + if(timeout) + { + assertTrue(runtime.canceled.get() > 0); + } + } + assertEquals(runtime.created.get(), runtime.freed.get()); + } + } + + /*************************************************************************** + ** Actual thread CPU is measured; waiting wall time is not charged as CPU. + ***************************************************************************/ + @Test + void actualCpuBudgetCancelsAndCannotReportSuccess() throws Exception + { + AtomicInteger canceled = new AtomicInteger(); + AtomicInteger freed = new AtomicInteger(); + try(var scheduler = new ScheduledThreadPoolExecutor(1)) + { + InvocationBudget budget = new InvocationBudget(scheduler, 1, 500, () -> {}); + GuestRuntime.Invocation fixture = new GuestRuntime.Invocation() + { + /***************************************************************** + ** Fixed trusted loop, independently bounded even if cancel fails. + *****************************************************************/ + @Override + public byte[] call(byte[] ignored) + { + return (new byte[0]); + } + + /***************************************************************** + ** Observe cancellation without any native execution. + *****************************************************************/ + @Override + public void cancel() + { + canceled.incrementAndGet(); + } + + /***************************************************************** + ** Actual cleanup observation. + *****************************************************************/ + @Override + public void close() + { + freed.incrementAndGet(); + } + }; + try + { + budget.attach(fixture); + } + catch(IOException expiredDuringAttach) + { + assertEquals("INVOCATION_LIMIT", expiredDuringAttach.getMessage()); + } + long cpu = InvocationBudget.requireCpu().getCurrentThreadCpuTime(); + long end = System.nanoTime() + 100_000_000L; + while(InvocationBudget.requireCpu().getCurrentThreadCpuTime() - cpu < 5_000_000 && System.nanoTime() < end) + { + Thread.onSpinWait(); + } + assertThrows(IOException.class, budget::check); + assertThrows(IOException.class, budget::close); + assertThrows(IOException.class, budget::cpuMillis); + assertTrue(canceled.get() > 0); + assertEquals(1, freed.get()); + } + } + + /*************************************************************************** + ** Standalone outer protocol rejects malformed data before guest admission. + ***************************************************************************/ + @Test + void strictProtocolAndConfigurationBounds() throws Exception + { + for(String invalid : List.of("{x:1}", "{\"x\":1,\"x\":2}", "{} trailing", "{".repeat(17))) + { + assertThrows(Exception.class, () -> Protocol.parse(invalid.getBytes(StandardCharsets.UTF_8))); + } + assertThrows(IOException.class, () -> Protocol.number(new JSONObject().put("n", "1"), "n", 1, 2)); + assertThrows(IOException.class, () -> Protocol.number(new JSONObject().put("n", 1.2), "n", 1, 2)); + assertThrows(IllegalArgumentException.class, () -> new SupervisorServer.Configuration(tls.context(), new InetSocketAddress("127.0.0.1", 0), + Map.of(tls.fingerprint(), "fixture"), "profile", "toolchain", "", "", "a".repeat(64), 1, 2, 1, 2, 4096, 64L * 1024 * 1024, 1000)); + } + + /*************************************************************************** + ** An already dequeued old timer cannot close a newly active exchange. + ***************************************************************************/ + @Test + void retiredDeadlineCannotCancelFollowingOperation() + { + AtomicInteger canceled = new AtomicInteger(); + try(var scheduler = new ScheduledThreadPoolExecutor(1)) + { + SocketDeadline prior = new SocketDeadline(scheduler, 60_000, canceled::incrementAndGet); + prior.close(); + try(var current = new SocketDeadline(scheduler, 60_000, canceled::incrementAndGet)) + { + prior.expire(); + assertEquals(0, canceled.get()); + current.expire(); + assertEquals(1, canceled.get()); + prior.expire(); + assertEquals(1, canceled.get()); + } + } + } + + /*************************************************************************** + ** Owned ephemeral service/thread, never a shared port or installed daemon. + ***************************************************************************/ + private static final class Host implements AutoCloseable + { + private final SupervisorServer server; + private final Thread thread; + private final AtomicReference listenerFailure = new AtomicReference<>(); + private boolean exhaustionExpected; + + /*********************************************************************** + ** Test-only evidence identity and fixed trusted runtime supplied directly. + ***********************************************************************/ + private Host(TrustedRuntime runtime, String fingerprint) throws Exception + { + this(runtime, fingerprint, 2000); + } + + /*********************************************************************** + ** Separate idle timeout makes TLS-drain retention observable in real mTLS. + ***********************************************************************/ + private Host(TrustedRuntime runtime, String fingerprint, int idleMillis) throws Exception + { + server = new SupervisorServer(new SupervisorServer.Configuration(tls.context(), new InetSocketAddress("127.0.0.1", 0), + Map.of(fingerprint, "fixture"), "profile", "toolchain", "fixture-evidence", "fixture-evidence", runtime.sha256(), + 1, 3, 1, 2, 8192, 64L * 1024 * 1024, idleMillis), runtime); + thread = Thread.ofPlatform().daemon().start(() -> + { + try + { + server.serve(); + } + catch(IOException failure) + { + listenerFailure.set(failure); + } + }); + } + + /*********************************************************************** + ** Exhaustion must end the public service loop, not leave a live empty pool. + ***********************************************************************/ + private void expectExhaustion() throws Exception + { + exhaustionExpected = true; + thread.join(1000); + assertFalse(thread.isAlive()); + assertTrue(listenerFailure.get() != null); + assertEquals("ENGINE_CAPACITY_EXHAUSTED", listenerFailure.get().getMessage()); + } + + /*********************************************************************** + ** Release only owned listener/threads/engines; no other host resources. + ***********************************************************************/ + @Override + public void close() throws Exception + { + server.close(); + thread.join(1000); + assertFalse(thread.isAlive()); + if(exhaustionExpected) + { + assertEquals("ENGINE_CAPACITY_EXHAUSTED", listenerFailure.get().getMessage()); + } + else + { + assertEquals(null, listenerFailure.get()); + } + } + } + + /*************************************************************************** + ** Independent real TLS client using the already accepted outer wire contract. + ***************************************************************************/ + private static final class Client implements AutoCloseable + { + private final SSLSocket socket; + private final DataInputStream input; + private final DataOutputStream output; + private final String session; + private long sequence; + + /*********************************************************************** + ** Verify hostname and exchange exact certificate-authorized HELLO. + ***********************************************************************/ + private Client(int port, String namespace) throws Exception + { + socket = (SSLSocket) tls.context().getSocketFactory().createSocket("localhost", port); + try + { + SSLParameters parameters = socket.getSSLParameters(); + parameters.setEndpointIdentificationAlgorithm("HTTPS"); + parameters.setProtocols(new String[] {"TLSv1.3"}); + socket.setSSLParameters(parameters); + socket.setSoTimeout(2500); + socket.startHandshake(); + input = new DataInputStream(socket.getInputStream()); + output = new DataOutputStream(socket.getOutputStream()); + write(new JSONObject().put("protocol", Protocol.VERSION).put("type", "HELLO").put("namespace", namespace) + .put("profile", "profile").put("toolchain", "toolchain").put("evidence", "fixture-evidence").put("nonce", UUID.randomUUID().toString())); + JSONObject accepted = read(); + assertEquals("ACCEPT", accepted.getString("type")); + session = accepted.getString("session"); + } + catch(Exception failure) + { + socket.close(); + throw failure; + } + } + + /*********************************************************************** + ** Compile only fixed synthetic bytecode through the trusted runtime seam. + ***********************************************************************/ + private String prepare() throws Exception + { + JSONObject request = envelope("PREPARE").put("sha256", Protocol.hash(BUNDLE.getBytes(StandardCharsets.UTF_8))) + .put("entryPoint", "src/index.ts").put("bundle", BUNDLE); + write(request); + JSONObject ready = read(); + assertEquals("READY", ready.getString("type")); + assertEquals("RETIRED", read().getString("type")); + return (ready.getString("readyId")); + } + + /*********************************************************************** + ** Empty bounded response confirms this exact session, not cache inventory. + ***********************************************************************/ + private void ping() throws Exception + { + JSONObject request = envelope("PING"); + write(request); + JSONObject pong = read(); + assertEquals(Set.of("protocol", "type", "session", "sequence", "requestId"), pong.keySet()); + assertEquals("PONG", pong.getString("type")); + JSONObject retired = read(); + assertEquals(Set.of("protocol", "type", "session", "sequence", "requestId", "retired"), retired.keySet()); + assertEquals("RETIRED", retired.getString("type")); + assertTrue(retired.getBoolean("retired")); + for(JSONObject response : List.of(pong, retired)) + { + assertEquals(request.getLong("sequence"), response.getLong("sequence")); + for(String key : List.of("protocol", "session", "requestId")) + { + assertEquals(request.get(key), response.get(key)); + } + } + } + + /*********************************************************************** + ** Return only after broker, actual result and distinct retirement frame. + ***********************************************************************/ + private JSONObject invoke(String ready, String token, int cpu, int wall) throws Exception + { + JSONObject outer = envelope("INVOKE").put("readyId", ready); + JSONObject request = new JSONObject().put("protocol", Protocol.WORKER).put("type", "execute").put("requestId", outer.getString("requestId")) + .put("artifactId", "artifact").put("artifactSha256", Protocol.hash(BUNDLE.getBytes(StandardCharsets.UTF_8))).put("entryPoint", "src/index.ts") + .put("event", "PRE_INSERT").put("payload", new JSONObject().put("proposed", new JSONObject())).put("capabilityToken", token) + .put("limits", new JSONObject().put("cpuMillis", cpu).put("wallMillis", wall).put("memoryBytes", 64L * 1024 * 1024).put("outputBytes", 65536).put("brokerCalls", 2)); + write(outer.put("request", request)); + JSONObject broker = read(); + assertEquals("BROKER", broker.getString("type")); + assertEquals(token, broker.getJSONObject("message").getString("token")); + write(new JSONObject().put("protocol", Protocol.VERSION).put("type", "BROKER_REPLY").put("session", session).put("sequence", sequence) + .put("requestId", outer.getString("requestId")).put("message", new JSONObject().put("protocol", Protocol.WORKER) + .put("type", "brokerResult").put("ok", true).put("rows", new JSONArray()).put("truncated", false))); + JSONObject result = read(); + assertEquals("RESULT", result.getString("type")); + assertEquals("RETIRED", read().getString("type")); + return (result.getJSONObject("message")); + } + + /*********************************************************************** + ** Fresh exact operation correlation, independent from production helper. + ***********************************************************************/ + private JSONObject envelope(String type) + { + return (new JSONObject().put("protocol", Protocol.VERSION).put("type", type).put("session", session) + .put("sequence", ++sequence).put("requestId", UUID.randomUUID().toString())); + } + + /*********************************************************************** + ** Independent frame writer. + ***********************************************************************/ + private void write(JSONObject object) throws IOException + { + byte[] bytes = object.toString().getBytes(StandardCharsets.UTF_8); + output.writeInt(bytes.length); + output.write(bytes); + output.flush(); + } + + /*********************************************************************** + ** Small trusted response bound prevents fixture hangs/large allocation. + ***********************************************************************/ + private JSONObject read() throws IOException + { + int size = input.readInt(); + if(size < 2 || size > 64 * 1024) + { + throw new IOException("TEST_FRAME"); + } + byte[] bytes = new byte[size]; + input.readFully(bytes); + return (new JSONObject(new String(bytes, StandardCharsets.UTF_8))); + } + + /*********************************************************************** + ** Release only this owned client connection. + ***********************************************************************/ + @Override + public void close() throws IOException + { + socket.close(); + } + } +} diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/SupervisorStartupCheck.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/SupervisorStartupCheck.java new file mode 100644 index 0000000..01edd73 --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/SupervisorStartupCheck.java @@ -0,0 +1,103 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import org.json.JSONObject; + +/******************************************************************************* + ** Root-run startup-only diagnostic. Fixed labels only; no author code, credentials, + ** configuration values, native error strings or arbitrary exception text is logged. + *******************************************************************************/ +public final class SupervisorStartupCheck +{ + /*************************************************************************** + ** Utility diagnostic entrypoint. + ***************************************************************************/ + private SupervisorStartupCheck() {} + + /*************************************************************************** + ** Exercise the real production startup stages, then close the unused listener. + ***************************************************************************/ + public static void main(String[] args) + { + String stage = "CONFIG"; + try + { + if(args.length != 1) + { + throw new IOException("CONFIG_REQUIRED"); + } + Path file = Path.of(args[0]); + if(!file.isAbsolute() || !Files.isRegularFile(file) || Files.size(file) > 64 * 1024) + { + throw new IOException("CONFIG_FILE"); + } + JSONObject config = Protocol.parse(Files.readAllBytes(file)); + Protocol.keys(config, "bind", "port", "keyStore", "trustStore", "certificateNamespaces", "profile", "toolchain", "evidence", "requiredEvidence", + "runtimePath", "runtimeSha256", "nativePath", "nativeSha256", "slots", "connections", "coldOperations", "entries", "codeBytes", "memoryBytes", "idleMillis", "fuel"); + stage = "TLS"; + var tls = SupervisorMain.tls(Path.of(config.getString("keyStore")), Path.of(config.getString("trustStore"))); + stage = "SETTINGS"; + Map certificates = new HashMap<>(); + JSONObject mappings = config.getJSONObject("certificateNamespaces"); + for(String key : mappings.keySet()) + { + certificates.put(key, mappings.getString(key)); + } + long memory = Protocol.number(config, "memoryBytes", 64L * 1024 * 1024, 256L * 1024 * 1024); + var settings = new SupervisorServer.Configuration(tls, new InetSocketAddress(config.getString("bind"), (int) Protocol.number(config, "port", 1, 65535)), + certificates, config.getString("profile"), config.getString("toolchain"), config.getString("evidence"), config.getString("requiredEvidence"), + config.getString("runtimeSha256"), (int) Protocol.number(config, "slots", 1, 64), (int) Protocol.number(config, "connections", 1, 128), + (int) Protocol.number(config, "coldOperations", 1, 8), (int) Protocol.number(config, "entries", 1, 256), + Protocol.number(config, "codeBytes", 1, 64L * 1024 * 1024), memory, (int) Protocol.number(config, "idleMillis", 100, 60000)); + stage = "NATIVE_BINDING"; + try(var runtime = new JavyExtismRuntime(Path.of(config.getString("nativePath")), config.getString("nativeSha256"), + Path.of(config.getString("runtimePath")), config.getString("runtimeSha256"), memory, Protocol.number(config, "fuel", 1, 1_000_000_000L))) + { + stage = "RUNTIME_COMPILE_AND_BIND"; + try(var server = new SupervisorServer(settings, runtime)) + { + stage = "CLEANUP"; + } + } + System.out.println(new JSONObject().put("diagnostic", "supervisor-startup/1").put("stage", "COMPLETE").put("success", true) + .put("authorCodeExecuted", false)); + } + catch(Throwable failure) + { + String category = failure instanceof IOException ? "IO" : failure instanceof IllegalArgumentException ? "CONFIGURATION" + : failure instanceof LinkageError ? "LINKAGE" : "OTHER"; + Set codes = Set.of("CONFIG_REQUIRED", "CONFIG_FILE", "TLS_SECRET_REQUIRED", "TLS_STORE", "NATIVE_CONFIGURATION", "NATIVE_IDENTITY", + "NATIVE_VERSION", "HOST_FUNCTION", "RUNTIME_COMPILE", "SUPERVISOR_CONFIGURATION", "CERTIFICATE_NAMESPACE", "SUPERVISOR_DRAIN_UNKNOWN"); + String code = failure.getMessage() != null && codes.contains(failure.getMessage()) ? failure.getMessage() : "UNCLASSIFIED"; + System.out.println(new JSONObject().put("diagnostic", "supervisor-startup/1").put("stage", stage).put("category", category) + .put("code", code).put("success", false).put("authorCodeExecuted", false)); + System.exit(1); + } + } +} \ No newline at end of file diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/TlsIdentity.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/TlsIdentity.java new file mode 100644 index 0000000..946a4fd --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/TlsIdentity.java @@ -0,0 +1,80 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManagerFactory; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyStore; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +/******************************************************************************* + ** Ephemeral real certificate for trusted loopback tests, never committed keys. + *******************************************************************************/ +record TlsIdentity(SSLContext context, String fingerprint) +{ + /*************************************************************************** + ** Current JDK creates a local certificate; secrets never enter command output. + ***************************************************************************/ + static TlsIdentity create(Path directory) throws Exception + { + String password = UUID.randomUUID().toString(); + Path path = directory.resolve("identity.p12"); + String keytool = Path.of(System.getProperty("java.home"), "bin", System.getProperty("os.name").startsWith("Windows") ? "keytool.exe" : "keytool").toString(); + ProcessBuilder builder = new ProcessBuilder(keytool, "-genkeypair", "-alias", "fixture", "-keyalg", "EC", "-groupname", "secp256r1", + "-dname", "CN=localhost", "-ext", "SAN=dns:localhost", "-validity", "2", "-storetype", "PKCS12", "-keystore", path.toString(), + "-storepass:env", "QQQ_TEST_PASSWORD", "-keypass:env", "QQQ_TEST_PASSWORD", "-noprompt"); + builder.environment().put("QQQ_TEST_PASSWORD", password); + builder.redirectOutput(ProcessBuilder.Redirect.DISCARD); + builder.redirectError(ProcessBuilder.Redirect.DISCARD); + Process process = builder.start(); + try + { + if(!process.waitFor(15, TimeUnit.SECONDS) || process.exitValue() != 0) + { + throw new IOException("FIXTURE_IDENTITY"); + } + } + finally + { + if(process.isAlive()) + { + process.destroyForcibly(); + } + } + KeyStore store = KeyStore.getInstance("PKCS12"); + try(var input = Files.newInputStream(path)) + { + store.load(input, password.toCharArray()); + } + KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + keys.init(store, password.toCharArray()); + TrustManagerFactory trust = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + trust.init(store); + SSLContext context = SSLContext.getInstance("TLSv1.3"); + context.init(keys.getKeyManagers(), trust.getTrustManagers(), null); + return (new TlsIdentity(context, Protocol.hash(store.getCertificate("fixture").getEncoded()))); + } +} diff --git a/supervisor/src/test/java/com/kingsrook/scripting/supervisor/TrustedRuntime.java b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/TrustedRuntime.java new file mode 100644 index 0000000..fecf2b0 --- /dev/null +++ b/supervisor/src/test/java/com/kingsrook/scripting/supervisor/TrustedRuntime.java @@ -0,0 +1,199 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.scripting.supervisor; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.HexFormat; +import java.util.concurrent.atomic.AtomicInteger; +import org.json.JSONArray; +import org.json.JSONObject; + +/******************************************************************************* + ** Non-executing lifecycle fixture only, not an engine or qualification shortcut. + *******************************************************************************/ +final class TrustedRuntime implements GuestRuntime +{ + final AtomicInteger compiled = new AtomicInteger(); + final AtomicInteger created = new AtomicInteger(); + final AtomicInteger freed = new AtomicInteger(); + final AtomicInteger canceled = new AtomicInteger(); + volatile boolean failClose; + volatile boolean wrongToken; + volatile boolean pending; + volatile boolean corruptPrepared; + volatile boolean failWithError; + volatile boolean failWithIo; + + /*************************************************************************** + ** Fixed test-only runtime identity; production main cannot select this class. + ***************************************************************************/ + @Override + public String sha256() + { + return ("a".repeat(64)); + } + + /*************************************************************************** + ** Track common compiled engine count separately from fresh invocation count. + ***************************************************************************/ + @Override + public Slot createSlot() + { + compiled.incrementAndGet(); + return (new Slot() + { + /******************************************************************** + ** Fresh one-shot trusted protocol fixture, never evaluating input code. + ********************************************************************/ + @Override + public Invocation open(Broker broker) + { + created.incrementAndGet(); + return (new Invocation() + { + private boolean used; + private volatile boolean canceledCall; + + /************************************************************** + ** Generate fixed prepared bytes or relay a controlled broker read. + **************************************************************/ + @Override + public byte[] call(byte[] bytes) throws Exception + { + if(used) + { + throw new IOException("REUSED"); + } + used = true; + if(failWithError) + { + throw new AssertionError("TRUSTED_RUNTIME_FAILURE"); + } + if(failWithIo) + { + throw new IOException("TRUSTED_IO_FAILURE"); + } + JSONObject input = Protocol.parse(bytes); + if("prepare".equals(input.getString("operation"))) + { + JSONObject result = prepared(input.getString("artifactSha256")); + if(corruptPrepared) + { + result.put("preparedSha256", "0".repeat(64)); + } + return (result.toString().getBytes(StandardCharsets.UTF_8)); + } + if(pending) + { + long end = System.nanoTime() + 500_000_000L; + while(!canceledCall && System.nanoTime() < end) + { + Thread.sleep(1); + } + throw new IOException("CANCELED"); + } + JSONObject request = input.getJSONObject("request"); + JSONObject read = new JSONObject().put("protocol", Protocol.WORKER).put("type", "broker") + .put("requestId", request.getString("requestId")).put("operation", "query").put("table", "purchase_order") + .put("fields", new JSONArray().put("id")).put("criteria", new JSONArray()).put("limit", 1) + .put("token", wrongToken ? "wrong" : request.getString("capabilityToken")); + broker.exchange(read.toString().getBytes(StandardCharsets.UTF_8)); + JSONObject message = new JSONObject().put("protocol", Protocol.WORKER).put("type", "executeResult") + .put("requestId", request.getString("requestId")).put("outcome", "OK").put("result", new JSONObject().put("action", "ALLOW")); + return (new JSONObject().put("message", message).put("guestWallMillis", 987654).toString().getBytes(StandardCharsets.UTF_8)); + } + + /************************************************************** + ** Record only this invocation's cancellation. + **************************************************************/ + @Override + public void cancel() + { + canceledCall = true; + canceled.incrementAndGet(); + } + + /************************************************************** + ** Inject actual lifecycle refusal before any retirement ack. + **************************************************************/ + @Override + public void close() throws Exception + { + if(failClose) + { + throw new IOException("CLOSE_FAILED"); + } + freed.incrementAndGet(); + } + }); + } + + /******************************************************************** + ** No native resource exists in this trusted fixture. + ********************************************************************/ + @Override + public void close() {} + }); + } + + /*************************************************************************** + ** Independently construct the private Rust ABI's fixed digest/vector. + ***************************************************************************/ + private JSONObject prepared(String artifact) throws Exception + { + byte[] bytecode = "fixed-trusted-bytecode".getBytes(StandardCharsets.UTF_8); + JSONObject module = new JSONObject().put("path", "src/index.js").put("sha256", Protocol.hash(bytecode)) + .put("bytecode", Base64.getEncoder().encodeToString(bytecode)); + JSONObject prepared = new JSONObject().put("profile", "qqq-javy8.1-rquickjs0.12-esm/1").put("runtimeSha256", sha256()) + .put("artifactSha256", artifact).put("entryPoint", "src/index.js").put("modules", new JSONArray().put(module)); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + digest.update("qqq-javy-prepared-v1\0".getBytes(StandardCharsets.UTF_8)); + for(String field : new String[] {"profile", "runtimeSha256", "artifactSha256", "entryPoint"}) + { + append(digest, prepared.getString(field)); + } + for(String field : new String[] {"path", "sha256", "bytecode"}) + { + append(digest, module.getString(field)); + } + return (prepared.put("preparedSha256", HexFormat.of().formatHex(digest.digest()))); + } + + /*************************************************************************** + ** Exact UTF8 byte length framing for the independent fixture vector. + ***************************************************************************/ + private void append(MessageDigest hash, String text) + { + byte[] bytes = text.getBytes(StandardCharsets.UTF_8); + hash.update(ByteBuffer.allocate(8).putLong(bytes.length).array()); + hash.update(bytes); + } + + /*************************************************************************** + ** No global state survives fixture teardown. + ***************************************************************************/ + @Override + public void close() {} +} diff --git a/wasm-runtime/BUILD-RECIPE.md b/wasm-runtime/BUILD-RECIPE.md new file mode 100644 index 0000000..97c4ba6 --- /dev/null +++ b/wasm-runtime/BUILD-RECIPE.md @@ -0,0 +1,20 @@ +# Fixed Wasm guest: locked reproducible build and pin (updated 2026-11-02) + +The fixed Javy/QuickJS guest `qqq_javy_runtime.wasm` is pinned in `PINNED-WASM.json` (sha256, length, the exact source inputs and their hash, Cargo.lock hash, Rust commit, WASI SDK identity). `deploy/snapshot_execution.py` (`WASM_SHA`), the license inventory, import inventory and review doc all carry the same hash, and `deploy/test_deploy.py` (`PinnedWasmTest`) fails if any of them, the Wasm source inputs or Cargo.lock drift apart. Change the guest source only together with a reviewed re-pin. + +## Reproduce (operator, offline, no network) + +```text +pwsh -File wasm-runtime/Build-Wasm.ps1 -SdkRoot -OutputDir +``` + +`Build-Wasm.ps1` refuses an existing output directory, verifies the source-input hash and Cargo.lock against the pin, the Rust 1.97.1 compiler commit, and the SDK's clang version line plus `clang.exe`, `libclang.dll` and WASI libc hashes; sets `WASI_SDK`, `LIBCLANG_PATH` and `BINDGEN_EXTRA_CLANG_ARGS` (forward slashes; the rquickjs-sys build script must never download an SDK); runs `cargo +1.97.1 build --release --target wasm32-wasip1 --offline --locked`; then FAILS unless the artifact equals the pinned sha256 and length. It writes `provenance.json` next to the artifact. Inputs: WASI SDK 33.0 archive sha256 `df14ca2a2127c2d6b6be07e6f5549b3af9c1b3c0112430c200a4749970c59f06` (extracted copy used here: root's `qqq-javy-wasi-tools-20261031/sdk-extracted`), Rust 1.97.1 with the `wasm32-wasip1` target. + +## Current pin +`4b247288d6cdff6783f8087a2a9039935f50af9099451cc841a3639c9625a18c`, 1,567,189 bytes, source inputs `ec834dc9d9d10e360c7abcb10c811e6dd892ce744bb40974a068e01908530cb7`. Reproduced twice from clean copies in different directories and different output directories with identical hash (`repro-A`, `repro-B` logs). A one-byte source change is refused before any build. + +## Why the pin changed (finding) +The earlier pin `ca72ac7d72a0876a2c602fe6b73c9ded7ead35fbba6f4034ee544345a38557af` was built before a standards pass added a 3-line license header to each `src/*.rs` on 2026-11-01. Rebuilding the CURRENT inputs with those 3 lines removed, using this same recipe and toolchain, reproduces `ca72...` exactly, so the original recipe is recovered. The two artifacts differ in exactly 9 bytes, all Rust panic-location line numbers (each +3); the type, import, function, export, code and custom Wasm sections are byte-identical (only the data section differs). Reverting the headers would violate the repository's header standard, so the owned source is re-pinned to the artifact built from current source. The old artifact is still the one inside the existing image `qqq-script-execution:qualification-20261031-1` and the root's old context; they were not modified, and `snapshot_execution.py --verify` of that old context now refuses under the new pin by design. + +## Evidence (trusted local only; `productionQualified=false`) +12 native tests and clippy `-D warnings` pass from the clean checkout; the native fixed-Wasm + Extism supervisor corpus passes with the new artifact in the existing image (private evidence, not carried in this tree). diff --git a/wasm-runtime/Build-Wasm.ps1 b/wasm-runtime/Build-Wasm.ps1 new file mode 100644 index 0000000..ba7728d --- /dev/null +++ b/wasm-runtime/Build-Wasm.ps1 @@ -0,0 +1,86 @@ +# Copyright (C) 2026 Kingsrook, LLC. +# SPDX-License-Identifier: Apache-2.0 +<# +.SYNOPSIS + Reproduce the fixed Javy/QuickJS Wasm guest from locked inputs and verify it against PINNED-WASM.json. +.DESCRIPTION + Offline only. Never downloads an SDK or crate. Refuses an existing output directory. + Verifies, before building: Cargo.lock/source-input hashes, the Rust compiler commit, and the + already-extracted WASI SDK's clang version line plus clang.exe, libclang.dll and wasi libc hashes. + After building: the artifact must match the pinned sha256 and length, otherwise the script fails. + Run it twice with different -OutputDir values (and, ideally, from different checkouts) to prove + reproducibility. Writes provenance.json next to the artifact. +.EXAMPLE + pwsh -File Build-Wasm.ps1 -SdkRoot C:/tools/wasi-sdk-33.0-x86_64-windows -OutputDir C:/new/output-1 +#> +param( + [Parameter(Mandatory)][string]$SdkRoot, + [Parameter(Mandatory)][string]$OutputDir +) +$ErrorActionPreference = 'Stop' +$root = $PSScriptRoot +$pin = Get-Content -Raw (Join-Path $root 'PINNED-WASM.json') | ConvertFrom-Json + +function Sha256Of([string]$path) { (Get-FileHash -Algorithm SHA256 -LiteralPath $path).Hash.ToLowerInvariant() } +function Require([bool]$condition, [string]$message) { if (-not $condition) { throw "Build-Wasm refused: $message" } } + +Require ([IO.Path]::IsPathRooted($OutputDir)) 'OutputDir must be absolute' +Require (-not (Test-Path -LiteralPath $OutputDir)) 'OutputDir must not already exist' +Require (Test-Path -LiteralPath $SdkRoot) 'SdkRoot not found' + +# 1. locked source inputs +$combined = New-Object System.Text.StringBuilder +foreach ($name in $pin.sourceInputs) { + [void]$combined.Append($name + "`n" + (Sha256Of (Join-Path $root $name)) + "`n") +} +$sha = [Security.Cryptography.SHA256]::Create() +$inputsHash = ([BitConverter]::ToString($sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($combined.ToString()))) -replace '-', '').ToLowerInvariant() +Require ($inputsHash -eq $pin.sourceInputsSha256) "source inputs differ from PINNED-WASM.json (have $inputsHash); re-pin only after a reviewed source change" +Require ((Sha256Of (Join-Path $root 'Cargo.lock')) -eq $pin.cargoLockSha256) 'Cargo.lock differs from the pin' + +# 2. toolchain +$rustc = (& rustc "+$($pin.toolchain.rust)" -vV) -join "`n" +Require ($rustc -match "commit-hash: $($pin.toolchain.rustcCommitHash)") 'rustc commit differs from the pin' +$sdkInfo = $pin.toolchain.wasiSdk +$clang = Join-Path $SdkRoot 'bin/clang.exe' +Require ((Sha256Of $clang) -eq $sdkInfo.clangExeSha256) 'SDK clang.exe differs from the pin' +Require ((Sha256Of (Join-Path $SdkRoot 'bin/libclang.dll')) -eq $sdkInfo.libclangDllSha256) 'SDK libclang.dll differs from the pin' +Require ((Sha256Of (Join-Path $SdkRoot 'share/wasi-sysroot/lib/wasm32-wasip1/libc.a')) -eq $sdkInfo.wasiLibcSha256) 'SDK wasi libc differs from the pin' +Require (((& $clang --version) | Select-Object -First 1) -eq $sdkInfo.clangVersionLine) 'SDK clang version line differs from the pin' +$clangInclude = Get-ChildItem (Join-Path $SdkRoot 'lib/clang') -Directory | Select-Object -First 1 +Require ($null -ne $clangInclude) 'SDK has no clang resource directory' + +# 3. build (new output directory; offline; locked) +New-Item -ItemType Directory -Path $OutputDir | Out-Null +$saved = @{} +foreach ($key in 'CARGO_TARGET_DIR', 'WASI_SDK', 'LIBCLANG_PATH', 'BINDGEN_EXTRA_CLANG_ARGS') { $saved[$key] = [Environment]::GetEnvironmentVariable($key) } +try { + $env:CARGO_TARGET_DIR = (Join-Path $OutputDir 'cargo-target') + $env:WASI_SDK = $SdkRoot + $env:LIBCLANG_PATH = (Join-Path $SdkRoot 'bin') + # forward slashes: the clang-args parser treats backslashes as escapes + $env:BINDGEN_EXTRA_CLANG_ARGS = '-isystem ' + ((Join-Path $clangInclude.FullName 'include') -replace '\\', '/') + Push-Location $root + try { + & cargo "+$($pin.toolchain.rust)" build --release --target $pin.toolchain.target --offline --locked + if ($LASTEXITCODE -ne 0) { throw "cargo build failed ($LASTEXITCODE)" } + } finally { Pop-Location } +} finally { + foreach ($key in $saved.Keys) { [Environment]::SetEnvironmentVariable($key, $saved[$key]) } +} + +# 4. verify against the pin and record provenance +$built = Join-Path $OutputDir "cargo-target/$($pin.toolchain.target)/release/qqq_javy_runtime.wasm" +$artifact = Join-Path $OutputDir $pin.artifact +Copy-Item -LiteralPath $built -Destination $artifact +$actual = Sha256Of $artifact +$length = (Get-Item -LiteralPath $artifact).Length +$result = [ordered]@{ + artifact = $pin.artifact; sha256 = $actual; bytes = $length; pinnedSha256 = $pin.sha256; matchesPin = ($actual -eq $pin.sha256 -and $length -eq $pin.bytes) + sourceInputsSha256 = $inputsHash; cargoLockSha256 = $pin.cargoLockSha256 + rustc = ($rustc -split "`n" | Select-Object -First 1); wasiSdkClang = $sdkInfo.clangVersionLine + builtAtUtc = [DateTime]::UtcNow.ToString('o') +} +($result | ConvertTo-Json) | Set-Content -LiteralPath (Join-Path $OutputDir 'provenance.json') -Encoding utf8 +Require $result.matchesPin "built artifact $actual ($length bytes) does not match the pin $($pin.sha256)" +Write-Output ($result | ConvertTo-Json -Compress) diff --git a/wasm-runtime/Cargo.lock b/wasm-runtime/Cargo.lock new file mode 100644 index 0000000..1dd7bba --- /dev/null +++ b/wasm-runtime/Cargo.lock @@ -0,0 +1,649 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "convert_case" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "env_filter", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "javy" +version = "8.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5679b8c3d13df0aa91ed63adc87b98fa1a4e067634db394b694ea453c8291cb" +dependencies = [ + "anyhow", + "bitflags", + "fastrand", + "quickcheck", + "rquickjs", + "serde", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "log" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "qqq-javy-runtime" +version = "0.1.0" +dependencies = [ + "base64", + "javy", + "rquickjs", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "quickcheck" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b" +dependencies = [ + "env_logger", + "log", + "rand", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "relative-path" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38540a682c6e9bca04a8bca9e5ec5a6ba168dfbb04c9998fe517a9befb92eb68" +dependencies = [ + "serde_core", +] + +[[package]] +name = "rquickjs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0688f8b0192998cca685adefdfad3483da295fa40a0ec406b4c14ecd729e858" +dependencies = [ + "rquickjs-core", + "rquickjs-macro", +] + +[[package]] +name = "rquickjs-core" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e4f499ac5b943d97ee6dbc44f23c2c10426f420f7d2f1793d6318911b6608c" +dependencies = [ + "hashbrown", + "relative-path", + "rquickjs-sys", +] + +[[package]] +name = "rquickjs-macro" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbcc8219b70ee2faa08d5339f47f15741cba2ce0cb9640e8495f2ab51293f50" +dependencies = [ + "convert_case", + "fnv", + "ident_case", + "indexmap", + "proc-macro-crate", + "proc-macro2", + "quote", + "rquickjs-core", + "syn", +] + +[[package]] +name = "rquickjs-sys" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13ac243b86a74120814ef7e9e30ad5a2c1199b7b9963b1cf7c84e4cdc1cad99" +dependencies = [ + "bindgen", + "cc", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml_datetime" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.17+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] diff --git a/wasm-runtime/Cargo.toml b/wasm-runtime/Cargo.toml new file mode 100644 index 0000000..e8adca8 --- /dev/null +++ b/wasm-runtime/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "qqq-javy-runtime" +version = "0.1.0" +edition = "2024" +rust-version = "1.85" +publish = false +license = "Apache-2.0" + +[lib] +crate-type = ["cdylib", "rlib"] + +[dependencies] +base64 = "=0.22.1" +javy = "=8.1.0" +rquickjs = { version = "=0.12.0", features = ["loader", "array-buffer"] } +serde = { version = "=1.0.228", features = ["derive"] } +serde_json = "=1.0.145" +sha2 = "=0.10.9" + +[profile.release] +panic = "abort" +lto = true +opt-level = 3 +codegen-units = 1 +strip = "debuginfo" + +[lints.rust] +missing_docs = "warn" +unsafe_op_in_unsafe_fn = "deny" + +[lints.clippy] +unwrap_used = "deny" +expect_used = "deny" +print_stdout = "deny" +print_stderr = "deny" diff --git a/wasm-runtime/PINNED-WASM.json b/wasm-runtime/PINNED-WASM.json new file mode 100644 index 0000000..adab201 --- /dev/null +++ b/wasm-runtime/PINNED-WASM.json @@ -0,0 +1,31 @@ +{ + "schema": "qqq-pinned-wasm/1", + "artifact": "qqq_javy_runtime.wasm", + "sha256": "4b247288d6cdff6783f8087a2a9039935f50af9099451cc841a3639c9625a18c", + "bytes": 1567189, + "sourceInputs": ["Cargo.toml", "Cargo.lock", "rust-toolchain.toml", "rustfmt.toml", "src/abi.rs", "src/lib.rs", "src/model.rs", "src/runtime.rs", "src/sdk.js"], + "sourceInputsSha256": "ec834dc9d9d10e360c7abcb10c811e6dd892ce744bb40974a068e01908530cb7", + "sourceInputsAlgorithm": "sha256 over, for each listed path in order, the UTF-8 text '\\n\\n'", + "cargoLockSha256": "f077884125e23e5989d158fe6fe86df9009f5842076609cedf2da6f4951021a1", + "toolchain": { + "rust": "1.97.1", + "rustcCommitHash": "8bab26f4f68e0e26f0bb7960be334d5b520ea452", + "target": "wasm32-wasip1", + "wasiSdk": { + "version": "33.0", + "archiveSha256": "df14ca2a2127c2d6b6be07e6f5549b3af9c1b3c0112430c200a4749970c59f06", + "clangVersionLine": "clang version 22.1.0 (https://github.com/llvm/llvm-project 4434dabb69916856b824f68a64b029c67175e532)", + "clangExeSha256": "6d9b6b13a9d6c0e64765fe85c4ba86b605f9b65e49a0c28703c85ba66038a1df", + "libclangDllSha256": "4e449c403e2fd28fb6259e54b7250dec1ec3a8f5d4d76aedae0eb4802a356c4d", + "wasiLibcSha256": "918d0741c0bcc6d9048cc7d7becd71ab35a7606925e546911462bc9f2333cbc9" + } + }, + "command": "cargo +1.97.1 build --release --target wasm32-wasip1 --offline --locked, with WASI_SDK, LIBCLANG_PATH and BINDGEN_EXTRA_CLANG_ARGS set by Build-Wasm.ps1 (no network; the rquickjs-sys build script must never download an SDK)", + "previous": { + "sha256": "ca72ac7d72a0876a2c602fe6b73c9ded7ead35fbba6f4034ee544345a38557af", + "bytes": 1567189, + "provenance": "Built before the 3-line license header was added to each src/*.rs on 2026-11-01. Rebuilding the CURRENT inputs with the first 3 lines of each src/*.rs removed, using the same toolchain and this recipe, reproduces this hash exactly. It differs from the current artifact in exactly 9 bytes, all Rust panic-location line numbers (each +3), no code. Retained in existing images/contexts; do not retag or rewrite them.", + "status": "superseded for the owned source; still the bytes inside qqq-script-execution:qualification-20261031-1" + }, + "qualification": "trusted-lab fixed artifact only; productionQualified=false and redistributionCleared=false are unchanged" +} diff --git a/wasm-runtime/README.md b/wasm-runtime/README.md new file mode 100644 index 0000000..898a656 --- /dev/null +++ b/wasm-runtime/README.md @@ -0,0 +1,63 @@ +# Fixed Javy/Wasm runtime adapter — not qualified + +This new module is a standalone runtime plugin. It does not import QQQ, modify the existing TypeScript compiler or change `qbit-scripting-worker/1`. No product provider may advertise it as qualified until the exact built artifact and complete host lifecycle pass the required tests. + +## Dependency and build boundary + +Javy CLI release9.1.0 corresponds to public runtime crate `javy8.1.0`, using `rquickjs0.12.0` ([tagged Cargo.toml](https://github.com/bytecodealliance/javy/blob/v9.1.0/crates/javy/Cargo.toml)). This module uses those public crates directly. **It requires no Javy CLI, Wizer or author snapshot.** `Runtime::new` constructs a new QuickJS runtime during each invocation. The external host caches the common compiled Wasm code; publication caches contain only private bytecode and immutable identity metadata. + +Rust1.97.1 is pinned. Direct dependencies are exact-version pins in Cargo.toml. Root resolved the 73-package Cargo.lock (SHA256 `F077884125E23E5989D158FE6FE86DF9009F5842076609CEDF2DA6F4951021A1`); the rquickjs facade is0.12.0 and its compatible core/sys dependencies resolve to0.12.2. Root owns public tool acquisition, Linux/WASI SDK/libclang setup and actual Wasm execution. Building rquickjs requires its documented C/WASI compiler and bindgen prerequisites; do not silently substitute an engine or crate version. + +Planned root commands after the pinned environment is ready: + +```text +cargo generate-lockfile +cargo fmt --check +cargo test --locked --test trusted_contract +cargo clippy --all-targets --locked -- -D warnings +cargo build --locked --release --target wasm32-wasip1 +``` + +The native tests execute only fixed trusted fixtures. Native tests cannot qualify Wasm import capabilities, fuel, actual fresh Store state or host cancellation. A subsequent exact-Wasm run must verify those independently. The common Wasm plugin must be instantiated with no WASI preopens, no inherited environment/stdin credentials, no network capability, bounded linear memory/fuel and an outer deadline. Native tests are not permission to run author submissions. + +## Private supervisor ABI + +Extism export `run() -> i32` reads and writes one UTF-8 JSON frame through the ordinary Extism input/output ABI. Exactly one call is permitted per Wasm instance; the host must free that Plugin/Store before reporting RETIRED. Return zero means a private result was produced; a nonzero return is a refusal/trap with no successful result. + +Prepare input (cold path): + +```json +{"operation":"prepare","runtimeSha256":"","artifactSha256":"","bundle":""} +``` + +Prepare returns `profile`, `runtimeSha256`, `artifactSha256`, `entryPoint`, sorted `modules[{path,bytecode,sha256}]` and `preparedSha256`. Bytecode is canonical standard base64; the prepared hash is domain-separated and length-framed across all immutable fields. This digest proves consistency, not authorization: the supervisor must verify the actual loaded runtime binary and original publication, and retain the resulting bytecode itself. **Never accept arbitrary client-supplied bytecode merely because its self-declared hashes match.** QuickJS's public `Module::load` is an unsafe bytecode reader; only the trusted compiler's exact output is an allowed input, inside the Wasm boundary. + +Run input (hot path): + +```json +{"operation":"run","runtimeSha256":"","preparedSha256":"","prepared":{},"request":{}} +``` + +`prepared` is the complete cached prepare result. `request` is the existing execute object **without bundle**, matching the persistent transport's INVOKE shape; it retains digest, `.ts` entrypoint, request ID, event/payload, fresh opaque capability token and limits. Run output is `{message,guestWallMillis}`. `message` has the existing executeResult fields but deliberately omits `usage`. The supervisor adds authoritative CPU/wall usage before forwarding the unchanged WorkerProtocol result. Guest wall time is diagnostic, not CPU accounting or a trusted enforcement signal. + +Imported application function: `extism:host/user::qqq_broker(i64 requestOffset) -> i64 responseOffset`. Offsets refer to Extism-owned memory, not host pointers. Input is existing bounded broker JSON; output must be a newly allocated existing brokerResult JSON block. The plugin copies and frees both allocations; returning the request allocation itself is refused. The supervisor callback runs synchronously on the active invocation thread and supplies current authority from that invocation only. It must validate frames and enforce its own I/O deadline. Zero offset is refusal. No callback, token or actor is retained in compiled code/cache. + +## Semantic/resource contract + +Preparation verifies the existing emitted artifact hash, validates the closed path/module set and calls Javy `compile_to_bytecode`, which declares/writes modules without evaluation. Execution installs a restricted relative resolver and loads those exact modules. It awaits module evaluation and exported `main` until actually settled; pending with no work, detached rejection, job failure and bounded job exhaustion are failures. It does not use default Javy invoke's pending-Promise success behavior. + +The trusted SDK factory captures its primitives before author evaluation, returns a frozen Promise-based `qqq.query` and preserves one-pending-call and call-count constraints. The opaque token remains in the native broker closure, not a guest global. Broker denial and author errors produce generic host-owned error text. Output is a bounded JSON object. + +QuickJS heap is capped by the requested memory limit, stack at256KiB; wall checks and engine interrupt cover interpretation. A fixed10,000 pending-job-turn ceiling is an additional explicit runtime profile bound. Extism/Wasmtime must separately enforce linear-memory/fuel, and the supervisor must enforce actual CPU and whole-operation wall limits including initialization and broker waits. Fuel does not meter native host callback work. One Engine slot has one active invocation; ambiguous cancellation retires the slot rather than reusing it speculatively. + +The few `unsafe` blocks are narrowly documented public FFI/bytecode boundaries, required by the chosen APIs. There is no engine fork, reflection or raw host-pointer access. This is an explicit scoped exception to the personal default of forbidding unsafe Rust; it does not permit unchecked external bytecode. + +## Current evidence + +Twelve fixed trusted native tests pass with `cargo +1.97.1 test --offline --locked --test trusted_contract`, including fresh state/time/random, relative imports/live bindings/TLA, awaited broker, Unicode source above1MiB UTF-8, pending/error refusal and attributed-import rejection. `cargo-native-serialization-red.log` preserves the demonstrated toJSON detached-rejection bug; `cargo-native-serialization-verification.log` records all12 passing after adding the final bounded job/rejection check after serialization. No Wasm execution, performance or hostile-code qualification has passed for this module yet. The earlier Extism JS-PDK runtime-floor experiment is **not** evidence for this implementation. Exact-Wasm proof must include these semantics, one-call-per-instance refusal, and actual fuel/memory/deadline/cancel retirement behavior. + +### Pinned bytecode import-attribute compatibility + +The resolved rquickjs-sys0.12.2 `quickjs/quickjs.c` has SHA256 `CE9A95D3C011F255C1FACB7D4361C71209E71BA58D84ACD3281A755D0686A285`. `JS_WriteModule` at38038–38042 writes requested module names without attributes. `JS_ReadModule` at39124–39142 allocates zeroed entries and passes their attribute value to the import resolver without initializing it to JS_UNDEFINED. The public rquickjs wrapper therefore supplies `Some` with a non-object value for ordinary bytecode imports; enumerating it throws. The initial native test and reduced relative-import fixture both failed on this path. + +`load_bytecode` uses a runtime-local RAII `BytecodePhase` to ignore that placeholder only during synchronous loading of exact supervisor-owned bytecode. Nested loads restore the prior phase; evaluation occurs outside it. Source compilation rejects actual attributes for every module, including unused files, before producing a prepared result. Dynamic imports remain subject to the normal attribute check; regression tests prove attributed dynamic imports fail after nested loading while ordinary dynamic imports succeed. There is no engine modification, global flag, guest-visible toggle or acceptance of client-supplied bytecode. This workaround remains tied to the pinned dependency behavior and needs re-review on an engine upgrade. diff --git a/wasm-runtime/docs/license-inventory.json b/wasm-runtime/docs/license-inventory.json new file mode 100644 index 0000000..31baeb5 --- /dev/null +++ b/wasm-runtime/docs/license-inventory.json @@ -0,0 +1,2054 @@ +{ + "scope": "Cargo.lock inventory; selected metadata includes build/proc-macro/dev dependencies and is not a linked-binary SBOM", + "artifactSha256": "4b247288d6cdff6783f8087a2a9039935f50af9099451cc841a3639c9625a18c", + "cargoLockSha256": "f077884125e23e5989d158fe6fe86df9009f5842076609cedf2da6f4951021a1", + "metadataCommand": "cargo +1.97.1 metadata --offline --locked --filter-platform wasm32-wasip1 --format-version 1", + "metadataSha256": "416330d183fe4deea30f9f9894f82340a713a483a6d3491ec0807d1556fe8cde", + "packages": [ + { + "name": "aho-corasick", + "version": "1.1.5", + "declaredLicense": "Unlicense OR MIT", + "declaredLicenseFile": null, + "lockChecksum": "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba", + "cachedArchiveSha256": "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "COPYING", + "sha256": "01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f" + }, + { + "path": "LICENSE-MIT", + "sha256": "0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f" + }, + { + "path": "UNLICENSE", + "sha256": "7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c" + } + ], + "source": "https://crates.io/crates/aho-corasick/1.1.5" + }, + { + "name": "allocator-api2", + "version": "0.2.21", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923", + "cachedArchiveSha256": "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "20fe7b00e904ed690e3b9fd6073784d3fc428141dbd10b81c01fd143d0797f58" + }, + { + "path": "LICENSE-MIT", + "sha256": "36516aefdc84c5d5a1e7485425913a22dbda69eb1930c5e84d6ae4972b5194b9" + } + ], + "source": "https://crates.io/crates/allocator-api2/0.2.21" + }, + { + "name": "anyhow", + "version": "1.0.104", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470", + "cachedArchiveSha256": "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/anyhow/1.0.104" + }, + { + "name": "base64", + "version": "0.22.1", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6", + "cachedArchiveSha256": "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example", + "test", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "0dd882e53de11566d50f8e8e2d5a651bcf3fabee4987d70f306233cf39094ba7" + } + ], + "source": "https://crates.io/crates/base64/0.22.1" + }, + { + "name": "bindgen", + "version": "0.72.1", + "declaredLicense": "BSD-3-Clause", + "declaredLicenseFile": null, + "lockChecksum": "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895", + "cachedArchiveSha256": "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "c23953d9deb0a3312dbeaf6c128a657f3591acee45067612fa68405eaa4525db" + } + ], + "source": "https://crates.io/crates/bindgen/0.72.1" + }, + { + "name": "bitflags", + "version": "2.13.2", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06", + "cachedArchiveSha256": "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example", + "example", + "example", + "example", + "example", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb" + } + ], + "source": "https://crates.io/crates/bitflags/2.13.2" + }, + { + "name": "block-buffer", + "version": "0.10.4", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71", + "cachedArchiveSha256": "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a9040321c3712d8fd0b09cf52b17445de04a23a10165049ae187cd39e5c86be5" + }, + { + "path": "LICENSE-MIT", + "sha256": "d5c22aa3118d240e877ad41c5d9fa232f9c77d757d4aac0c2f943afc0a95e0ef" + } + ], + "source": "https://crates.io/crates/block-buffer/0.10.4" + }, + { + "name": "cc", + "version": "1.10.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0", + "cachedArchiveSha256": "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "378f5840b258e2779c39418f3f2d7b2ba96f1c7917dd6be0713f88305dbda397" + } + ], + "source": "https://crates.io/crates/cc/1.10.0" + }, + { + "name": "cexpr", + "version": "0.6.0", + "declaredLicense": "Apache-2.0/MIT", + "declaredLicenseFile": null, + "lockChecksum": "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766", + "cachedArchiveSha256": "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "d9771b8c6cf4426d3846de54c1febe20907f1eeadf7adfb5ade89a83bd9ea77f" + } + ], + "source": "https://crates.io/crates/cexpr/0.6.0" + }, + { + "name": "cfg-if", + "version": "1.0.5", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600", + "cachedArchiveSha256": "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "378f5840b258e2779c39418f3f2d7b2ba96f1c7917dd6be0713f88305dbda397" + } + ], + "source": "https://crates.io/crates/cfg-if/1.0.5" + }, + { + "name": "clang-sys", + "version": "1.9.1", + "declaredLicense": "Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a", + "cachedArchiveSha256": "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE.txt", + "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" + } + ], + "source": "https://crates.io/crates/clang-sys/1.9.1" + }, + { + "name": "convert_case", + "version": "0.11.0", + "declaredLicense": "MIT", + "declaredLicenseFile": null, + "lockChecksum": "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49", + "cachedArchiveSha256": "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "aed7b1758e35afa0cd0fde059d61950747ca11cd0e5e169cb21c11608daed772" + } + ], + "source": "https://crates.io/crates/convert_case/0.11.0" + }, + { + "name": "cpufeatures", + "version": "0.2.17", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280", + "cachedArchiveSha256": "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": false, + "targets": [ + null + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a9040321c3712d8fd0b09cf52b17445de04a23a10165049ae187cd39e5c86be5" + }, + { + "path": "LICENSE-MIT", + "sha256": "ae9baa7beea910273c2f384c2a6b721fb7bd02bda3436074a1072e4ee689f985" + } + ], + "source": "https://crates.io/crates/cpufeatures/0.2.17" + }, + { + "name": "crypto-common", + "version": "0.1.7", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a", + "cachedArchiveSha256": "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a9040321c3712d8fd0b09cf52b17445de04a23a10165049ae187cd39e5c86be5" + }, + { + "path": "LICENSE-MIT", + "sha256": "3521672491a3479422d5fe1aca6645dd2984090f85da6e5205abfb18fb7a6897" + } + ], + "source": "https://crates.io/crates/crypto-common/0.1.7" + }, + { + "name": "digest", + "version": "0.10.7", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292", + "cachedArchiveSha256": "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a9040321c3712d8fd0b09cf52b17445de04a23a10165049ae187cd39e5c86be5" + }, + { + "path": "LICENSE-MIT", + "sha256": "9e0dfd2dd4173a530e238cb6adb37aa78c34c6bc7444e0e10c1ab5d8881f63ba" + } + ], + "source": "https://crates.io/crates/digest/0.10.7" + }, + { + "name": "either", + "version": "1.19.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be", + "cachedArchiveSha256": "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "7576269ea71f767b99297934c0b2367532690f8c4badc695edf8e04ab6a1e545" + } + ], + "source": "https://crates.io/crates/either/1.19.0" + }, + { + "name": "env_filter", + "version": "2.0.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217", + "cachedArchiveSha256": "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "c6596eb7be8581c18be736c846fb9173b69eccf6ef94c5135893ec56bd92ba08" + }, + { + "path": "LICENSE-MIT", + "sha256": "6efb0476a1cc085077ed49357026d8c173bf33017278ef440f222fb9cbcb66e6" + } + ], + "source": "https://crates.io/crates/env_filter/2.0.0" + }, + { + "name": "env_logger", + "version": "0.11.11", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6", + "cachedArchiveSha256": "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example", + "example", + "example", + "example", + "example", + "example", + "example", + "example" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "c6596eb7be8581c18be736c846fb9173b69eccf6ef94c5135893ec56bd92ba08" + }, + { + "path": "LICENSE-MIT", + "sha256": "6efb0476a1cc085077ed49357026d8c173bf33017278ef440f222fb9cbcb66e6" + } + ], + "source": "https://crates.io/crates/env_logger/0.11.11" + }, + { + "name": "equivalent", + "version": "1.0.3", + "declaredLicense": "Apache-2.0 OR MIT", + "declaredLicenseFile": null, + "lockChecksum": "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826", + "cachedArchiveSha256": "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "7365cc8878a1d7ce155a58c4ca09c3d7a6be413efa5334a80ea842912b669349" + } + ], + "source": "https://crates.io/crates/equivalent/1.0.3" + }, + { + "name": "fastrand", + "version": "2.5.0", + "declaredLicense": "Apache-2.0 OR MIT", + "declaredLicenseFile": null, + "lockChecksum": "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223", + "cachedArchiveSha256": "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/fastrand/2.5.0" + }, + { + "name": "find-msvc-tools", + "version": "0.1.15", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48", + "cachedArchiveSha256": "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "378f5840b258e2779c39418f3f2d7b2ba96f1c7917dd6be0713f88305dbda397" + } + ], + "source": "https://crates.io/crates/find-msvc-tools/0.1.15" + }, + { + "name": "fnv", + "version": "1.0.7", + "declaredLicense": "Apache-2.0 / MIT", + "declaredLicenseFile": null, + "lockChecksum": "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1", + "cachedArchiveSha256": "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "65fdb6c76cd61612070c066eec9ecdb30ee74fb27859d0d9af58b9f499fd0c3e" + } + ], + "source": "https://crates.io/crates/fnv/1.0.7" + }, + { + "name": "foldhash", + "version": "0.2.0", + "declaredLicense": "Zlib", + "declaredLicenseFile": null, + "lockChecksum": "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb", + "cachedArchiveSha256": "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "b1181a40b2a7b25cf66fd01481713bc1005df082c53ef73e851e55071b102744" + } + ], + "source": "https://crates.io/crates/foldhash/0.2.0" + }, + { + "name": "generic-array", + "version": "0.14.7", + "declaredLicense": "MIT", + "declaredLicenseFile": null, + "lockChecksum": "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a", + "cachedArchiveSha256": "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "c09aae9d3c77b531f56351a9947bc7446511d6b025b3255312d3e3442a9a7583" + } + ], + "source": "https://crates.io/crates/generic-array/0.14.7" + }, + { + "name": "getrandom", + "version": "0.4.3", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099", + "cachedArchiveSha256": "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "bench", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "aaff376532ea30a0cd5330b9502ad4a4c8bf769c539c87ffe78819d188a18ebf" + }, + { + "path": "LICENSE-MIT", + "sha256": "523a42c25d245dde9c015f882cec7f4555aad883382a6cf19b4b7d9b2cd5419b" + } + ], + "source": "https://crates.io/crates/getrandom/0.4.3" + }, + { + "name": "glob", + "version": "0.3.4", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b", + "cachedArchiveSha256": "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb" + } + ], + "source": "https://crates.io/crates/glob/0.3.4" + }, + { + "name": "hashbrown", + "version": "0.17.1", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a", + "cachedArchiveSha256": "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test", + "test", + "test", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "ff8f68cb076caf8cefe7a6430d4ac086ce6af2ca8ce2c4e5a2004d4552ef52a2" + } + ], + "source": "https://crates.io/crates/hashbrown/0.17.1" + }, + { + "name": "ident_case", + "version": "1.0.1", + "declaredLicense": "MIT/Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39", + "cachedArchiveSha256": "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "508a77d2e7b51d98adeed32648ad124b7b30241a8e70b2e72c99f92d8e5874d1" + } + ], + "source": "https://crates.io/crates/ident_case/1.0.1" + }, + { + "name": "indexmap", + "version": "2.14.2", + "declaredLicense": "Apache-2.0 OR MIT", + "declaredLicenseFile": null, + "lockChecksum": "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855", + "cachedArchiveSha256": "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test", + "test", + "bench", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "ecc269ef87fd38a1d98e30bfac9ba964a9dbd9315c3770fed98d4d7cb5882055" + } + ], + "source": "https://crates.io/crates/indexmap/2.14.2" + }, + { + "name": "itertools", + "version": "0.13.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186", + "cachedArchiveSha256": "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "bench", + "bench", + "bench", + "bench", + "bench", + "bench", + "bench", + "bench", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "7576269ea71f767b99297934c0b2367532690f8c4badc695edf8e04ab6a1e545" + } + ], + "source": "https://crates.io/crates/itertools/0.13.0" + }, + { + "name": "itoa", + "version": "1.0.18", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682", + "cachedArchiveSha256": "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/itoa/1.0.18" + }, + { + "name": "javy", + "version": "8.1.0", + "declaredLicense": "Apache-2.0 WITH LLVM-exception", + "declaredLicenseFile": null, + "lockChecksum": "d5679b8c3d13df0aa91ed63adc87b98fa1a4e067634db394b694ea453c8291cb", + "cachedArchiveSha256": "d5679b8c3d13df0aa91ed63adc87b98fa1a4e067634db394b694ea453c8291cb", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test" + ], + "licenseFiles": [], + "source": "https://crates.io/crates/javy/8.1.0" + }, + { + "name": "libc", + "version": "0.2.191", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a", + "cachedArchiveSha256": "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "123a331b5dbf04c30097fa43b8f858bc85df671fe776de498d01f3d6b7c1f69e" + } + ], + "source": "https://crates.io/crates/libc/0.2.191" + }, + { + "name": "libloading", + "version": "0.8.9", + "declaredLicense": "ISC", + "declaredLicenseFile": null, + "lockChecksum": "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55", + "cachedArchiveSha256": "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test", + "test", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "b29f8b01452350c20dd1af16ef83b598fea3053578ccc1c7a0ef40e57be2620f" + } + ], + "source": "https://crates.io/crates/libloading/0.8.9" + }, + { + "name": "log", + "version": "0.4.35", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c", + "cachedArchiveSha256": "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb" + } + ], + "source": "https://crates.io/crates/log/0.4.35" + }, + { + "name": "memchr", + "version": "2.8.3", + "declaredLicense": "Unlicense OR MIT", + "declaredLicenseFile": null, + "lockChecksum": "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98", + "cachedArchiveSha256": "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "COPYING", + "sha256": "01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f" + }, + { + "path": "LICENSE-MIT", + "sha256": "0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f" + }, + { + "path": "UNLICENSE", + "sha256": "7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c" + } + ], + "source": "https://crates.io/crates/memchr/2.8.3" + }, + { + "name": "minimal-lexical", + "version": "0.2.1", + "declaredLicense": "MIT/Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a", + "cachedArchiveSha256": "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "8173d5c29b4f956d532781d2b86e4e30f83e6b7878dce18c919451d6ba707c90" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + }, + { + "path": "LICENSE.md", + "sha256": "dbe1fff0fb1314b6af94f161511406275cf01c5a32441fbf24528a57a051d599" + } + ], + "source": "https://crates.io/crates/minimal-lexical/0.2.1" + }, + { + "name": "nom", + "version": "7.1.3", + "declaredLicense": "MIT", + "declaredLicenseFile": null, + "lockChecksum": "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a", + "cachedArchiveSha256": "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example", + "example", + "example", + "example", + "example", + "example", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "4dbda04344456f09a7a588140455413a9ac59b6b26a1ef7cdf9c800c012d87f0" + } + ], + "source": "https://crates.io/crates/nom/7.1.3" + }, + { + "name": "prettyplease", + "version": "0.2.37", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b", + "cachedArchiveSha256": "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/prettyplease/0.2.37" + }, + { + "name": "proc-macro-crate", + "version": "3.5.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f", + "cachedArchiveSha256": "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "8ada45cd9f843acf64e4722ae262c622a2b3b3007c7310ef36ac1061a30f6adb" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/proc-macro-crate/3.5.0" + }, + { + "name": "proc-macro2", + "version": "1.0.107", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9", + "cachedArchiveSha256": "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test", + "test", + "test", + "test", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/proc-macro2/1.0.107" + }, + { + "name": "qqq-javy-runtime", + "version": "0.1.0", + "declaredLicense": "Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "", + "cachedArchiveSha256": null, + "archiveMatchesLock": null, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "cdylib", + "rlib", + "test" + ], + "licenseFiles": [], + "source": "local project" + }, + { + "name": "quickcheck", + "version": "1.1.0", + "declaredLicense": "Unlicense OR MIT", + "declaredLicenseFile": null, + "lockChecksum": "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b", + "cachedArchiveSha256": "95c589f335db0f6aaa168a7cd27b1fc6920f5e1470c804f814d9cd6e62a0f70b", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example", + "example", + "example", + "example", + "example", + "example" + ], + "licenseFiles": [ + { + "path": "COPYING", + "sha256": "01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f" + }, + { + "path": "LICENSE-MIT", + "sha256": "0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f" + }, + { + "path": "UNLICENSE", + "sha256": "7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c" + } + ], + "source": "https://crates.io/crates/quickcheck/1.1.0" + }, + { + "name": "quote", + "version": "1.0.47", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001", + "cachedArchiveSha256": "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/quote/1.0.47" + }, + { + "name": "r-efi", + "version": "6.0.0", + "declaredLicense": null, + "declaredLicenseFile": null, + "lockChecksum": "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf", + "cachedArchiveSha256": null, + "archiveMatchesLock": null, + "cachedSourcePresent": false, + "selectedByWasiMetadata": false, + "targets": [ + null + ], + "licenseFiles": [], + "source": "https://crates.io/crates/r-efi/6.0.0" + }, + { + "name": "rand", + "version": "0.10.3", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af", + "cachedArchiveSha256": "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "COPYRIGHT", + "sha256": "90eb64f0279b0d9432accfa6023ff803bc4965212383697eee27a0f426d5f8d5" + }, + { + "path": "LICENSE-APACHE", + "sha256": "35242e7a83f69875e6edeff02291e688c97caafe2f8902e4e19b49d3e78b4cab" + }, + { + "path": "LICENSE-MIT", + "sha256": "209fbbe0ad52d9235e37badf9cadfe4dbdc87203179c0899e738b39ade42177b" + } + ], + "source": "https://crates.io/crates/rand/0.10.3" + }, + { + "name": "rand_core", + "version": "0.10.1", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69", + "cachedArchiveSha256": "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test" + ], + "licenseFiles": [ + { + "path": "COPYRIGHT", + "sha256": "92b81db30f7ab6d693e0af5661a40d273ca1947e891123ed823924192c10cbc7" + }, + { + "path": "LICENSE-APACHE", + "sha256": "6df43f6f4b5d4587f3d8d71e45532c688fd168afa5fe89d571cb32fa09c4ef51" + }, + { + "path": "LICENSE-MIT", + "sha256": "8b6e9feec03e7c9a5facb26855cecd31662bf989b636bcfe79521bdf8ac863f0" + } + ], + "source": "https://crates.io/crates/rand_core/0.10.1" + }, + { + "name": "regex", + "version": "1.13.1", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d", + "cachedArchiveSha256": "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb" + } + ], + "source": "https://crates.io/crates/regex/1.13.1" + }, + { + "name": "regex-automata", + "version": "0.4.18", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2", + "cachedArchiveSha256": "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb" + } + ], + "source": "https://crates.io/crates/regex-automata/0.4.18" + }, + { + "name": "regex-syntax", + "version": "0.8.11", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4", + "cachedArchiveSha256": "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb" + }, + { + "path": "src/unicode_tables/LICENSE-UNICODE", + "sha256": "74db5baf44a41b1000312c673544b3374e4198af5605c7f9080a402cec42cfa3" + } + ], + "source": "https://crates.io/crates/regex-syntax/0.8.11" + }, + { + "name": "relative-path", + "version": "2.0.3", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "38540a682c6e9bca04a8bca9e5ec5a6ba168dfbb04c9998fe517a9befb92eb68", + "cachedArchiveSha256": "38540a682c6e9bca04a8bca9e5ec5a6ba168dfbb04c9998fe517a9befb92eb68", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "6485b8ed310d3f0340bf1ad1f47645069ce4069dcc6bb46c7d5c6faf41de1fdb" + } + ], + "source": "https://crates.io/crates/relative-path/2.0.3" + }, + { + "name": "rquickjs", + "version": "0.12.0", + "declaredLicense": "MIT", + "declaredLicenseFile": null, + "lockChecksum": "c0688f8b0192998cca685adefdfad3483da295fa40a0ec406b4c14ecd729e858", + "cachedArchiveSha256": "c0688f8b0192998cca685adefdfad3483da295fa40a0ec406b4c14ecd729e858", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "976ad3d07927343ab99b31510625acba89eac8e0e517c712925620ddeda91b70" + } + ], + "source": "https://crates.io/crates/rquickjs/0.12.0" + }, + { + "name": "rquickjs-core", + "version": "0.12.2", + "declaredLicense": "MIT", + "declaredLicenseFile": null, + "lockChecksum": "16e4f499ac5b943d97ee6dbc44f23c2c10426f420f7d2f1793d6318911b6608c", + "cachedArchiveSha256": "16e4f499ac5b943d97ee6dbc44f23c2c10426f420f7d2f1793d6318911b6608c", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [], + "source": "https://crates.io/crates/rquickjs-core/0.12.2" + }, + { + "name": "rquickjs-macro", + "version": "0.12.2", + "declaredLicense": "MIT", + "declaredLicenseFile": null, + "lockChecksum": "3cbcc8219b70ee2faa08d5339f47f15741cba2ce0cb9640e8495f2ab51293f50", + "cachedArchiveSha256": "3cbcc8219b70ee2faa08d5339f47f15741cba2ce0cb9640e8495f2ab51293f50", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "proc-macro" + ], + "licenseFiles": [], + "source": "https://crates.io/crates/rquickjs-macro/0.12.2" + }, + { + "name": "rquickjs-sys", + "version": "0.12.2", + "declaredLicense": "MIT", + "declaredLicenseFile": null, + "lockChecksum": "a13ac243b86a74120814ef7e9e30ad5a2c1199b7b9963b1cf7c84e4cdc1cad99", + "cachedArchiveSha256": "a13ac243b86a74120814ef7e9e30ad5a2c1199b7b9963b1cf7c84e4cdc1cad99", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "custom-build" + ], + "licenseFiles": [ + { + "path": "quickjs/LICENSE", + "sha256": "96f73f9d2a16c21a36b418f06073be26e7d6d5e7c1bc99756b21a4f2c74ef171" + } + ], + "source": "https://crates.io/crates/rquickjs-sys/0.12.2" + }, + { + "name": "rustc-hash", + "version": "2.1.3", + "declaredLicense": "Apache-2.0 OR MIT", + "declaredLicenseFile": null, + "lockChecksum": "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d", + "cachedArchiveSha256": "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "95bd3988beee069fa2848f648dab43cc6e0b2add2ad6bcb17360caf749802bcc" + }, + { + "path": "LICENSE-MIT", + "sha256": "30fefc3a7d6a0041541858293bcbea2dde4caa4c0a5802f996a7f7e8c0085652" + } + ], + "source": "https://crates.io/crates/rustc-hash/2.1.3" + }, + { + "name": "ryu", + "version": "1.0.23", + "declaredLicense": "Apache-2.0 OR BSL-1.0", + "declaredLicenseFile": null, + "lockChecksum": "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f", + "cachedArchiveSha256": "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-BOOST", + "sha256": "c9bff75738922193e67fa726fa225535870d2aa1059f91452c411736284ad566" + } + ], + "source": "https://crates.io/crates/ryu/1.0.23" + }, + { + "name": "serde", + "version": "1.0.228", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e", + "cachedArchiveSha256": "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/serde/1.0.228" + }, + { + "name": "serde_core", + "version": "1.0.228", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad", + "cachedArchiveSha256": "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/serde_core/1.0.228" + }, + { + "name": "serde_derive", + "version": "1.0.228", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79", + "cachedArchiveSha256": "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "proc-macro" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/serde_derive/1.0.228" + }, + { + "name": "serde_json", + "version": "1.0.145", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c", + "cachedArchiveSha256": "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "custom-build" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/serde_json/1.0.145" + }, + { + "name": "sha2", + "version": "0.10.9", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283", + "cachedArchiveSha256": "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a9040321c3712d8fd0b09cf52b17445de04a23a10165049ae187cd39e5c86be5" + }, + { + "path": "LICENSE-MIT", + "sha256": "b4eb00df6e2a4d22518fcaa6a2b4646f249b3a3c9814509b22bd2091f1392ff1" + } + ], + "source": "https://crates.io/crates/sha2/0.10.9" + }, + { + "name": "shlex", + "version": "1.3.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64", + "cachedArchiveSha256": "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "553fffcd9b1cb158bc3e9edc35da85ca5c3b3d7d2e61c883ebcfa8a65814b583" + }, + { + "path": "LICENSE-MIT", + "sha256": "4455bf75a91154108304cb283e0fea9948c14f13e20d60887cf2552449dea3b1" + } + ], + "source": "https://crates.io/crates/shlex/1.3.0" + }, + { + "name": "shlex", + "version": "2.0.1", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba", + "cachedArchiveSha256": "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "553fffcd9b1cb158bc3e9edc35da85ca5c3b3d7d2e61c883ebcfa8a65814b583" + }, + { + "path": "LICENSE-MIT", + "sha256": "4455bf75a91154108304cb283e0fea9948c14f13e20d60887cf2552449dea3b1" + } + ], + "source": "https://crates.io/crates/shlex/2.0.1" + }, + { + "name": "syn", + "version": "2.0.119", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297", + "cachedArchiveSha256": "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "test", + "bench", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + } + ], + "source": "https://crates.io/crates/syn/2.0.119" + }, + { + "name": "toml_datetime", + "version": "1.1.2+spec-1.1.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053", + "cachedArchiveSha256": "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "249fdb6694467dd615b259f7ddde07be3f8ae5a72cecebb0e24f6629af4215fe" + }, + { + "path": "LICENSE-MIT", + "sha256": "6efb0476a1cc085077ed49357026d8c173bf33017278ef440f222fb9cbcb66e6" + } + ], + "source": "https://crates.io/crates/toml_datetime/1.1.2+spec-1.1.0" + }, + { + "name": "toml_edit", + "version": "0.25.17+spec-1.1.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c", + "cachedArchiveSha256": "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "249fdb6694467dd615b259f7ddde07be3f8ae5a72cecebb0e24f6629af4215fe" + }, + { + "path": "LICENSE-MIT", + "sha256": "6efb0476a1cc085077ed49357026d8c173bf33017278ef440f222fb9cbcb66e6" + } + ], + "source": "https://crates.io/crates/toml_edit/0.25.17+spec-1.1.0" + }, + { + "name": "toml_parser", + "version": "1.1.5+spec-1.1.0", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c", + "cachedArchiveSha256": "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "249fdb6694467dd615b259f7ddde07be3f8ae5a72cecebb0e24f6629af4215fe" + }, + { + "path": "LICENSE-MIT", + "sha256": "6efb0476a1cc085077ed49357026d8c173bf33017278ef440f222fb9cbcb66e6" + } + ], + "source": "https://crates.io/crates/toml_parser/1.1.5+spec-1.1.0" + }, + { + "name": "typenum", + "version": "1.20.1", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20", + "cachedArchiveSha256": "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test" + ], + "licenseFiles": [ + { + "path": "LICENSE", + "sha256": "db11fec9946737df39ca3898d9cd8c10ec6f6c3a884a6802b0ad0b81b4e8f23a" + }, + { + "path": "LICENSE-APACHE", + "sha256": "516b24e051bf5630880ebbd55c40a25ce9552ebaf8970a53e8976eb70e522406" + }, + { + "path": "LICENSE-MIT", + "sha256": "a825bd853ab71619a4923d7b4311221427848070ff44d990da39b0b274c1683f" + } + ], + "source": "https://crates.io/crates/typenum/1.20.1" + }, + { + "name": "unicode-ident", + "version": "1.0.27", + "declaredLicense": "(MIT OR Apache-2.0) AND Unicode-3.0", + "declaredLicenseFile": null, + "lockChecksum": "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563", + "cachedArchiveSha256": "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "test", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "62c7a1e35f56406896d7aa7ca52d0cc0d272ac022b5d2796e7d6905db8a3636a" + }, + { + "path": "LICENSE-MIT", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + }, + { + "path": "LICENSE-UNICODE", + "sha256": "f7db81051789b729fea528a63ec4c938fdcb93d9d61d97dc8cc2e9df6d47f2a1" + } + ], + "source": "https://crates.io/crates/unicode-ident/1.0.27" + }, + { + "name": "unicode-segmentation", + "version": "1.13.3", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8", + "cachedArchiveSha256": "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "test", + "bench", + "bench", + "bench", + "bench" + ], + "licenseFiles": [ + { + "path": "COPYRIGHT", + "sha256": "23860c2a7b5d96b21569afedf033469bab9fe14a1b24a35068b8641c578ce24d" + }, + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "7b63ecd5f1902af1b63729947373683c32745c16a10e8e6292e2e2dcd7e90ae0" + } + ], + "source": "https://crates.io/crates/unicode-segmentation/1.13.3" + }, + { + "name": "version_check", + "version": "0.9.5", + "declaredLicense": "MIT/Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a", + "cachedArchiveSha256": "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib" + ], + "licenseFiles": [ + { + "path": "LICENSE-APACHE", + "sha256": "a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2" + }, + { + "path": "LICENSE-MIT", + "sha256": "b7e650f3fce5c53249d1cdc608b54df156a97edd636cf9d23498d0cfe7aec63e" + } + ], + "source": "https://crates.io/crates/version_check/0.9.5" + }, + { + "name": "windows-link", + "version": "0.2.1", + "declaredLicense": "MIT OR Apache-2.0", + "declaredLicenseFile": null, + "lockChecksum": "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5", + "cachedArchiveSha256": "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": false, + "targets": [ + null + ], + "licenseFiles": [ + { + "path": "license-apache-2.0", + "sha256": "c16f8dcf1a368b83be78d826ea23de4079fe1b4469a0ab9ee20563f37ff3d44b" + }, + { + "path": "license-mit", + "sha256": "c2cfccb812fe482101a8f04597dfc5a9991a6b2748266c47ac91b6a5aae15383" + } + ], + "source": "https://crates.io/crates/windows-link/0.2.1" + }, + { + "name": "winnow", + "version": "1.0.4", + "declaredLicense": "MIT", + "declaredLicenseFile": null, + "lockChecksum": "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81", + "cachedArchiveSha256": "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81", + "archiveMatchesLock": true, + "cachedSourcePresent": true, + "selectedByWasiMetadata": true, + "targets": [ + "lib", + "example", + "example", + "example", + "example", + "example", + "example", + "example", + "example", + "example", + "example", + "example", + "example", + "bench", + "bench", + "bench", + "bench", + "bench" + ], + "licenseFiles": [ + { + "path": "LICENSE-MIT", + "sha256": "cb5aedb296c5246d1f22e9099f925a65146f9f0d6b4eebba97fd27a6cdbbab2d" + } + ], + "source": "https://crates.io/crates/winnow/1.0.4" + } + ] +} diff --git a/wasm-runtime/docs/release-license-review.md b/wasm-runtime/docs/release-license-review.md new file mode 100644 index 0000000..62e1131 --- /dev/null +++ b/wasm-runtime/docs/release-license-review.md @@ -0,0 +1,62 @@ +# Fixed Wasm and standalone supervisor: license/provenance review + +2026-10-31. This is a bounded source/package inventory, not legal approval, a complete binary SBOM, or runtime qualification. No implementation, dependency, installed toolchain, or sealed audit artifact was changed. Only cached metadata inspection and primary public source reads were used. + +**Disposition:** no non-open license was identified in the inspected packages. Redistribution remains blocked on assembling and verifying the binary-specific notice/source evidence below. An open SPDX declaration alone does not establish that the final image includes its required notices. + +## Exact inputs + +| Input | SHA256 / identity | +|---|---| +| Fixed Wasm, 1,567,189 bytes | `4b247288d6cdff6783f8087a2a9039935f50af9099451cc841a3639c9625a18c` | + +Re-pin note (2026-11-02): the fixed Wasm hash above supersedes `ca72ac7d72a0876a2c602fe6b73c9ded7ead35fbba6f4034ee544345a38557af`. Both are built from the same code with the same locked toolchain; they differ only in 9 bytes of Rust panic-location line numbers (the license header added to each `src/*.rs` shifted lines by 3). The type, import, function, export, code and custom sections are byte-identical, so every license/import conclusion in this review applies unchanged. See `../PINNED-WASM.json` and `../Build-Wasm.ps1`. +| Cargo.lock | `f077884125e23e5989d158fe6fe86df9009f5842076609cedf2da6f4951021a1` | +| Rust1.97.1 wasm32-wasip1 std archive | `40ab01f39c0d99d4537e67463fae623d4ced762f2f1e344e87b01529e0864075` | +| WASI SDK33 Windows archive | `df14ca2a2127c2d6b6be07e6f5549b3af9c1b3c0112430c200a4749970c59f06` | +| Cached Extism1.41.0 `tools/libextism.so`, 30,811,880 bytes | `44eb7ed56c6b5610b97b9aa62a436e6ff7e3175730e487c42bb67105185d7e01` | +| JNA5.18.1 JAR | `260c4b1e22b1db9e110ee441c4f13ce115f841fa48c41d78750986214b395557` | +| org.json20251224 JAR | `2a3438fbf9293174ea82bdfa35e83d8b965cfe6bddc034a1ac5d60107174c209` | + +Root's isolated tool-download record identifies the dated [Rust distribution manifest](https://static.rust-lang.org/dist/channel-rust-1.97.1.toml) and [SDK33 release](https://github.com/WebAssembly/wasi-sdk/releases/tag/wasi-sdk-33). SDK and Rust tools are build inputs, not automatically final-image contents. The Extism entry above is the inspected local binary; the final image must bind its actual binary hash to this inventory rather than relying on a version string or archive filename. + +## Cargo evidence + +[license-inventory.json](license-inventory.json) records every lock entry: name/version, unmodified declared license expression, optional license-file declaration, lock checksum, independently hashed cached `.crate`, cached-source availability, target-filtered metadata membership, and relative license-file paths with SHA256. [inventory-cached-licenses.ps1](inventory-cached-licenses.ps1) reproduces that report without downloads. + +The lock has **74 entries: 73 external packages and this local crate**. Target-filtered metadata contains71 entries including the root. All72 available external crate archives match their lock checksums. `r-efi6.0.0` is not cached and is not selected for this WASI target; unrestricted offline metadata failed trying to resolve it. `cpufeatures0.2.17` and `windows-link0.2.1` are cached but outside the filtered set. The report deliberately leaves the uncached package's license unknown rather than inferring it. No install/download was attempted. + +The selected set includes build, procedural-macro and development graph entries. It is **not a claim that70 external crates are linked into the Wasm**. For the release SBOM, retain this full resolved build inventory and separately identify target-linked components from the exact build. Do not equate Cargo metadata membership or a lock entry with binary inclusion. + +Observed license families are MIT, Apache-2.0, Apache-2.0 WITH LLVM-exception, BSD-3-Clause, ISC, Zlib, BSL-1.0, Unlicense, and Unicode-3.0. Dual-license expressions remain alternatives; `unicode-ident` explicitly combines an MIT/Apache choice **AND Unicode-3.0**, so its Unicode notice must not be discarded. Legacy slash-separated expressions are preserved as declared rather than silently rewritten. + +The cached `javy8.1.0`, `rquickjs-core0.12.2`, and `rquickjs-macro0.12.2` packages contain no license-named files in the inspected tree. This local crate also has no module-local license/notice bundle. Those are concrete packaging gaps, not evidence of proprietary code. Javy's exact upstream [LICENSE.md](https://github.com/bytecodealliance/javy/blob/v9.1.0/LICENSE.md) is Apache-2.0 with LLVM exception. The rquickjs facade/source declares MIT; package provenance and the corresponding upstream copyright/permission text must accompany the distributed combination. + +## Vendored engine and toolchain code + +`rquickjs-sys0.12.2` embeds QuickJS sources. Its `quickjs/LICENSE` is present and hashed in the JSON inventory: MIT, including Bellard/Gordon/Noordhuis/Ibarra Corretgé attribution. Preserve that text as well as the Rust wrapper's notice. The bytecode-attribute workaround changes only our adapter, not upstream QuickJS. The engine source hash and exact workaround limitation are recorded in the sibling README. + +The Rust std archive includes LICENSE-MIT, LICENSE-APACHE and COPYRIGHT. COPYRIGHT explicitly points to detailed `COPYRIGHT-library.html` for library components; that file was **not present in the extracted std component**. Rust's top-level dual license does not replace its per-component notices. Obtain the matching1.97.1 library copyright/license metadata and map the code actually linked into the Wasm, including allocator/builtins/libc as applicable. Do not ship the entire compiler merely to satisfy runtime packaging; preserve build provenance separately. + +SDK33 pins LLVM at `4434dabb69916856b824f68a64b029c67175e532` and wasi-libc at `161b3195fc2558d2b1ba3eb9ffae3b2b47407623`. Its top-level Apache license does not cover every linked library by itself. The pinned [compiler-rt license](https://github.com/llvm/llvm-project/blob/4434dabb69916856b824f68a64b029c67175e532/compiler-rt/LICENSE.TXT) contains Apache/LLVM and legacy NCSA/MIT material. The pinned [wasi-libc license](https://github.com/WebAssembly/wasi-libc/blob/161b3195fc2558d2b1ba3eb9ffae3b2b47407623/LICENSE) identifies Apache/LLVM, Apache, MIT and component notices for allocators, cloudlibc, musl and musl-fts. No LICENSE/NOTICE/COPYRIGHT-named file was found in the extracted SDK tree. The build used Rust's self-contained final linker: the precise libc/builtins object origin must therefore be established from the actual link inputs, not assumed to be the SDK sysroot merely because SDK Clang compiled QuickJS. + +## Native supervisor dependencies + +| Component | Confirmed license and packaging limit | +|---|---| +| Extism1.41.0 | [BSD-3-Clause](https://github.com/extism/extism/blob/v1.41.0/LICENSE); preserve copyright, terms and disclaimer in binary distribution materials. | +| Extism static/transitive code | [Pinned runtime manifest](https://github.com/extism/extism/blob/v1.41.0/runtime/Cargo.toml) includes Wasmtime49/Cranelift/WASI and other dependencies. Our Wasm Cargo.lock does **not** inventory this separately built shared library. Need upstream release build feature/lock/SBOM or equivalent exact-source reconstruction, plus linked system-library inventory. Default HTTP/registration features in source must not be confused with our guest import grants. | +| JNA5.18.1 | JAR `META-INF/LICENSE` matches [Apache-2.0 OR LGPL-2.1-or-later](https://github.com/java-native-access/jna/blob/5.18.1/LICENSE). Record the distribution's chosen option and include its full applicable text. This is not LGPL-only. | +| JNA native payloads | JAR bundles native dispatch libraries; [vendored libffi](https://github.com/java-native-access/jna/blob/5.18.1/native/libffi/LICENSE) has its own MIT notice. Inventory retained native payloads, not merely Java Maven coordinates. The JAR's short license pointer does not contain the full Apache/libffi texts. | +| org.json20251224 | [Exact tag LICENSE](https://github.com/stleary/JSON-java/blob/20251224/LICENSE) says Public Domain. The inspected binary JAR had no LICENSE/NOTICE/COPYRIGHT-named entry; preserve the upstream declaration and provenance externally. Do not substitute the unrelated historical JSON “Good, not Evil” license. | + +No non-open license or unavoidable source-disclosure incompatibility was established by this scoped inspection. That conclusion does **not** clear uninspected Extism static dependencies or the eventual base/JRE image. Full corresponding-source obligations depend on the actual selected licenses and shipped components; source retention is also required here for reproducibility, independently of license requirements. + +## Remaining release gates + +1. Build an actual image notice directory containing the applicable complete texts/copyrights, including omitted crate texts, QuickJS, Unicode, Rust library components, libc/builtins, Extism, JNA/libffi and JSON provenance. Verify its files from the final image, not just the source tree. +2. Produce a target-linked Wasm SBOM plus separate native Extism/JNA/JRE/OS inventory. The73-package lock is the build graph baseline, not proof of all static code inside `libextism.so`. +3. Bind the exact Extism binary to upstream release source, build features/lock and transitives. Record source/archive checksums and license selections; inspect native dynamic dependencies when root assembles the image. +4. Preserve reproducible source/build recipes, Cargo.lock, tool archive hashes, adapter modification notice and exact final artifact hashes. Retain SDK/compiler evidence as build provenance even when those tools are omitted from the runtime image. + +These are precise unfinished distribution gates. There is no new security scan, blanket license clearance, legal approval, or authorization to execute untrusted scripts in this report. diff --git a/wasm-runtime/docs/wasm-imports.json b/wasm-runtime/docs/wasm-imports.json new file mode 100644 index 0000000..6ed1576 --- /dev/null +++ b/wasm-runtime/docs/wasm-imports.json @@ -0,0 +1,122 @@ +{ + "artifact": "target/wasm32-wasip1/release/qqq_javy_runtime.wasm", + "sha256": "4b247288d6cdff6783f8087a2a9039935f50af9099451cc841a3639c9625a18c", + "bytes": 1567189, + "imports": [ + { + "module": "extism:host/env", + "name": "alloc", + "kind": "function" + }, + { + "module": "extism:host/env", + "name": "store_u8", + "kind": "function" + }, + { + "module": "extism:host/user", + "name": "qqq_broker", + "kind": "function" + }, + { + "module": "extism:host/env", + "name": "length", + "kind": "function" + }, + { + "module": "extism:host/env", + "name": "load_u8", + "kind": "function" + }, + { + "module": "extism:host/env", + "name": "free", + "kind": "function" + }, + { + "module": "extism:host/env", + "name": "input_length", + "kind": "function" + }, + { + "module": "extism:host/env", + "name": "input_load_u8", + "kind": "function" + }, + { + "module": "extism:host/env", + "name": "output_set", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "random_get", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "environ_get", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "environ_sizes_get", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "clock_time_get", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "fd_close", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "fd_fdstat_get", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "fd_prestat_get", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "fd_prestat_dir_name", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "fd_read", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "fd_seek", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "fd_write", + "kind": "function" + }, + { + "module": "wasi_snapshot_preview1", + "name": "proc_exit", + "kind": "function" + } + ], + "exports": [ + { + "name": "memory", + "kind": "memory" + }, + { + "name": "run", + "kind": "function" + } + ] +} diff --git a/wasm-runtime/rust-toolchain.toml b/wasm-runtime/rust-toolchain.toml new file mode 100644 index 0000000..239957e --- /dev/null +++ b/wasm-runtime/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +channel = "1.97.1" +profile = "minimal" +components = ["rustfmt", "clippy"] +targets = ["wasm32-wasip1"] diff --git a/wasm-runtime/rustfmt.toml b/wasm-runtime/rustfmt.toml new file mode 100644 index 0000000..261e5bf --- /dev/null +++ b/wasm-runtime/rustfmt.toml @@ -0,0 +1,3 @@ +tab_spaces = 3 +max_width = 160 +newline_style = "Unix" diff --git a/wasm-runtime/src/abi.rs b/wasm-runtime/src/abi.rs new file mode 100644 index 0000000..5c74fdc --- /dev/null +++ b/wasm-runtime/src/abi.rs @@ -0,0 +1,94 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +//! Minimal Extism ABI. The only imported application capability is the broker. + +#[cfg(target_arch = "wasm32")] +mod wasm { + use crate::model::{Failure, MAX_FRAME}; + use std::cell::Cell; + + thread_local! { static USED: Cell = const { Cell::new(false) }; } + + #[link(wasm_import_module = "extism:host/env")] + unsafe extern "C" { + fn input_length() -> u64; + fn input_load_u8(offset: u64) -> u32; + fn alloc(length: u64) -> u64; + fn free(offset: u64); + fn length(offset: u64) -> u64; + fn load_u8(offset: u64) -> u32; + fn store_u8(offset: u64, value: u32); + fn output_set(offset: u64, length: u64); + } + + #[link(wasm_import_module = "extism:host/user")] + unsafe extern "C" { + fn qqq_broker(request: u64) -> u64; + } + + /// Read a single bounded host allocation; never turn a pointer into a Rust slice. + fn read(offset: u64) -> Result, Failure> { + // SAFETY: Extism performs the memory access; offsets are validated by its ABI. + let size = unsafe { length(offset) }; + if offset == 0 || size == 0 || size > MAX_FRAME as u64 { + return Err(Failure::Limit); + } + Ok((0..size).map(|index| unsafe { load_u8(offset + index) as u8 }).collect()) + } + + /// Allocate and copy bounded output through the public Extism memory ABI. + fn write(bytes: &[u8]) -> Result { + if bytes.len() > MAX_FRAME { + return Err(Failure::Limit); + } + // SAFETY: allocation and byte stores are implemented by Extism, not native pointers. + let offset = unsafe { alloc(bytes.len() as u64) }; + if offset == 0 { + return Err(Failure::Limit); + } + for (index, byte) in bytes.iter().enumerate() { + unsafe { + store_u8(offset + index as u64, u32::from(*byte)); + } + } + Ok(offset) + } + + /// Invoke the current call's broker synchronously; free only owned allocations. + fn broker(bytes: &[u8]) -> Result, Failure> { + let request = write(bytes)?; + // SAFETY: the supervisor binds this function to the current invocation only. + let response = unsafe { qqq_broker(request) }; + let result = if response == request { Err(Failure::Broker) } else { read(response) }; + unsafe { + free(request); + if response != 0 && response != request { + free(response); + } + } + result + } + + /// One operation per fresh Wasm instance. Any second call refuses unconditionally. + #[unsafe(no_mangle)] + pub extern "C" fn run() -> i32 { + if USED.with(|used| used.replace(true)) { + return -1; + } + // SAFETY: input access is provided and bounds-checked by Extism. + let size = unsafe { input_length() }; + if size == 0 || size > MAX_FRAME as u64 { + return -1; + } + let input: Vec = (0..size).map(|offset| unsafe { input_load_u8(offset) as u8 }).collect(); + let result = crate::process(&input, broker).and_then(|bytes| { + let offset = write(&bytes)?; + unsafe { + output_set(offset, bytes.len() as u64); + } + Ok(()) + }); + if result.is_ok() { 0 } else { -1 } + } +} diff --git a/wasm-runtime/src/lib.rs b/wasm-runtime/src/lib.rs new file mode 100644 index 0000000..ccb6630 --- /dev/null +++ b/wasm-runtime/src/lib.rs @@ -0,0 +1,38 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +//! Fixed Javy/Wasm runtime adapter. This module is not a qualified execution provider. +//! +//! Author code executes only inside this Wasm module. The supervisor must verify +//! its exact binary identity, enforce fuel/linear-memory/wall limits, and create +//! one fresh instance per operation. No actor state belongs in the code cache. + +mod abi; +mod model; +mod runtime; + +pub use model::Failure; + +/// Process one bounded private prepare/run envelope through public Javy APIs. +/// The broker callback is supplied by the trusted supervisor and has its own deadline. +pub fn process(input: &[u8], broker: impl Fn(&[u8]) -> Result, Failure> + 'static) -> Result, Failure> { + let operation = model::parse_operation(input)?; + let output = match operation { + model::Operation::Prepare { + runtime_sha256, + artifact_sha256, + bundle, + } => serde_json::to_value(runtime::prepare(&runtime_sha256, &artifact_sha256, &bundle)?).map_err(|_| Failure::Output)?, + model::Operation::Run { + runtime_sha256, + prepared_sha256, + prepared, + request, + } => runtime::execute(&runtime_sha256, &prepared_sha256, prepared, request, broker)?, + }; + let bytes = serde_json::to_vec(&output).map_err(|_| Failure::Output)?; + if bytes.len() > model::MAX_FRAME { + return Err(Failure::Limit); + } + Ok(bytes) +} diff --git a/wasm-runtime/src/model.rs b/wasm-runtime/src/model.rs new file mode 100644 index 0000000..32e923e --- /dev/null +++ b/wasm-runtime/src/model.rs @@ -0,0 +1,283 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +//! Bounded input, portable module paths and immutable prepared-code identities. + +use std::collections::BTreeSet; +use std::fmt::{Display, Formatter}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use serde::{Deserialize, Serialize, de::DeserializeOwned}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +/// Maximum private transport envelope and serialized result bytes. +pub const MAX_FRAME: usize = 4 * 1024 * 1024; +/// UTF-8 ceiling imposed by the existing execute transport envelope. +pub const MAX_SOURCE: usize = 2 * 1024 * 1024; +/// Existing Java bundle character ceiling, measured in UTF-16 code units. +pub const MAX_SOURCE_UTF16: usize = 1024 * 1024; +/// Maximum compiled publication bytecode before base64 encoding. +pub const MAX_BYTECODE: usize = 2 * 1024 * 1024; +/// Fixed compiler/runtime profile; exact binary SHA is additionally bound by the host. +pub const PROFILE: &str = "qqq-javy8.1-rquickjs0.12-esm/1"; +/// Existing QQQ worker protocol remains unchanged. +pub const PROTOCOL: &str = "qbit-scripting-worker/1"; + +/// Host-owned error codes never include author source or exception payloads. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Failure { + /// Invalid bounded protocol input. + Input, + /// Resource or collection bound exceeded. + Limit, + /// Artifact, runtime or prepared-code identity mismatch. + Identity, + /// Author compilation failure. + Compile, + /// Author execution, module or Promise failure. + Script, + /// Broker authority or response refusal. + Broker, + /// Invalid serialized result. + Output, +} + +impl Display for Failure { + fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::Input => "INVALID_INPUT", + Self::Limit => "RESOURCE_LIMIT", + Self::Identity => "ARTIFACT_IDENTITY", + Self::Compile => "COMPILE_FAILED", + Self::Script => "SCRIPT_FAILED", + Self::Broker => "BROKER_DENIED", + Self::Output => "INVALID_OUTPUT", + }) + } +} + +impl std::error::Error for Failure {} + +/// Private supervisor-to-Wasm envelope, distinct from the unchanged worker protocol. +#[derive(Deserialize)] +#[serde(tag = "operation", rename_all = "camelCase", deny_unknown_fields)] +pub enum Operation { + /// Cold preparation: never contains actor, token, event or record context. + #[serde(rename_all = "camelCase")] + Prepare { + runtime_sha256: String, + artifact_sha256: String, + bundle: String, + }, + /// Hot execution: immutable prepared bytecode plus a fresh worker request without bundle. + #[serde(rename_all = "camelCase")] + Run { + runtime_sha256: String, + prepared_sha256: String, + prepared: Prepared, + request: Value, + }, +} + +/// Existing emitted ESM artifact; its source text is hashed before parsing. +#[derive(Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Bundle { + /// Existing qqq-esm/1 discriminator. + pub format: String, + /// Exact canonical JavaScript entry path. + pub entry_point: String, + /// Closed module source collection. + pub files: Vec, +} + +/// One emitted JavaScript module. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceFile { + /// Canonical relative path. + pub path: String, + /// Emitted ES2022 source. + pub content: String, +} + +/// Immutable compilation result, not an authorization token. +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct Prepared { + /// Compiler profile identifier. + pub profile: String, + /// Exact externally verified runtime binary identity. + pub runtime_sha256: String, + /// Exact existing emitted artifact hash. + pub artifact_sha256: String, + /// Canonical entry path. + pub entry_point: String, + /// Sorted privately compiled modules. + pub modules: Vec, + /// Domain-separated identity of every preceding field/module. + pub prepared_sha256: String, +} + +/// One privately generated QuickJS bytecode module. +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CompiledModule { + /// Canonical module name used during compilation and loading. + pub path: String, + /// Standard canonical base64 bytecode. + pub bytecode: String, + /// Exact decoded bytecode digest. + pub sha256: String, +} + +/// Parse bounded JSON with the existing maximum nesting depth. +pub fn parse(input: &[u8], max: usize) -> Result { + parse_with_depth(input, max, 12) +} + +/// Private wrapper adds two levels around the unchanged worker request. +pub fn parse_operation(input: &[u8]) -> Result { + parse_with_depth(input, MAX_FRAME, 14) +} + +/// Scan depth before allocating a parsed JSON tree. +fn parse_with_depth(input: &[u8], max: usize, max_depth: u32) -> Result { + if input.len() > max { + return Err(Failure::Limit); + } + let mut depth = 0_u32; + let mut quoted = false; + let mut escaped = false; + for byte in input { + if quoted { + if escaped { + escaped = false; + } else if *byte == b'\\' { + escaped = true; + } else if *byte == b'"' { + quoted = false; + } + } else { + match byte { + b'"' => quoted = true, + b'{' | b'[' => { + depth += 1; + if depth > max_depth { + return Err(Failure::Limit); + } + } + b'}' | b']' => depth = depth.checked_sub(1).ok_or(Failure::Input)?, + _ => {} + } + } + } + serde_json::from_slice(input).map_err(|_| Failure::Input) +} + +/// Verify a canonical lowercase SHA-256 identity. +pub fn digest_valid(value: &str) -> bool { + value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Compute the existing UTF-8 SHA-256 artifact identifier. +pub fn hash(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +/// Match the existing worker's portable source path restrictions. +pub fn path_valid(path: &str) -> bool { + path.len() <= 200 + && path.ends_with(".js") + && !path.contains("..") + && path + .split('/') + .all(|part| !part.is_empty() && (part.as_bytes()[0].is_ascii_alphanumeric() || part.starts_with('_'))) + && path + .split('/') + .all(|part| part.bytes().all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte))) +} + +/// Resolve only closed relative ES module specifiers; never a filesystem or URL. +pub fn resolve(base: &str, name: &str) -> Result { + if !path_valid(base) || !(name.starts_with("./") || name.starts_with("../")) || !name.ends_with(".js") { + return Err(Failure::Input); + } + let mut parts: Vec<&str> = base.split('/').collect(); + parts.pop(); + for part in name.split('/') { + match part { + "." => {} + ".." => { + parts.pop().ok_or(Failure::Input)?; + } + "" => return Err(Failure::Input), + _ => parts.push(part), + } + } + let result = parts.join("/"); + if !path_valid(&result) { + return Err(Failure::Input); + } + Ok(result) +} + +impl Prepared { + /// Validate the trusted prepared code and return decoded modules. + pub fn validate(&self, runtime: &str, expected: &str) -> Result)>, Failure> { + if self.profile != PROFILE + || !digest_valid(runtime) + || self.runtime_sha256 != runtime + || !digest_valid(&self.artifact_sha256) + || !digest_valid(expected) + || self.prepared_sha256 != expected + || !path_valid(&self.entry_point) + || self.modules.is_empty() + || self.modules.len() > 200 + { + return Err(Failure::Identity); + } + let mut seen = BTreeSet::new(); + let mut total = 0_usize; + let mut decoded = Vec::new(); + let mut previous = ""; + for module in &self.modules { + if !path_valid(&module.path) + || !seen.insert(module.path.to_ascii_lowercase()) + || module.path.as_str() <= previous + || module.bytecode.len() > MAX_BYTECODE * 4 / 3 + 4 + { + return Err(Failure::Identity); + } + previous = &module.path; + let bytes = STANDARD.decode(&module.bytecode).map_err(|_| Failure::Identity)?; + total = total.checked_add(bytes.len()).ok_or(Failure::Limit)?; + if total > MAX_BYTECODE || bytes.is_empty() || STANDARD.encode(&bytes) != module.bytecode || hash(&bytes) != module.sha256 { + return Err(Failure::Identity); + } + decoded.push((module.path.clone(), bytes)); + } + if !decoded.iter().any(|(path, _)| path == &self.entry_point) || self.digest() != expected { + return Err(Failure::Identity); + } + Ok(decoded) + } + + /// Hash every immutable field with length-prefix framing and a versioned domain. + pub fn digest(&self) -> String { + let mut digest = Sha256::new(); + digest.update(b"qqq-javy-prepared-v1\0"); + for field in [&self.profile, &self.runtime_sha256, &self.artifact_sha256, &self.entry_point] { + digest.update((field.len() as u64).to_be_bytes()); + digest.update(field.as_bytes()); + } + for module in &self.modules { + for field in [&module.path, &module.sha256, &module.bytecode] { + digest.update((field.len() as u64).to_be_bytes()); + digest.update(field.as_bytes()); + } + } + format!("{:x}", digest.finalize()) + } +} diff --git a/wasm-runtime/src/runtime.rs b/wasm-runtime/src/runtime.rs new file mode 100644 index 0000000..1fc01ea --- /dev/null +++ b/wasm-runtime/src/runtime.rs @@ -0,0 +1,358 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +//! Public Javy/rquickjs adapter: compile without evaluation, then execute in a fresh runtime. + +use std::cell::Cell; +use std::collections::{BTreeMap, BTreeSet}; +use std::rc::Rc; +use std::time::{Duration, Instant}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use javy::{BytecodeStripping, Config, Runtime}; +use rquickjs::{ + Ctx, Function, Module, Object, Promise, Value as JsValue, + loader::{ImportAttributes, Loader, Resolver}, + module::Declared, +}; +use serde_json::{Value, json}; + +use crate::model::{self, Bundle, CompiledModule, Failure, MAX_BYTECODE, MAX_FRAME, MAX_SOURCE, MAX_SOURCE_UTF16, PROFILE, PROTOCOL, Prepared}; + +/// Fixed maximum number of pending-job turns in this resource profile. +const MAX_JOBS: usize = 10_000; +/// Trusted SDK adapter source, evaluated only inside Wasm QuickJS before author modules. +const API_FACTORY: &str = include_str!("sdk.js"); + +/// Construct a bounded new engine context; no author code or PRNG call runs here. +fn new_runtime(memory: usize, deadline: Instant) -> Result { + let mut config = Config::default(); + config + .memory_limit(memory) + .max_stack_size(256 * 1024) + .bytecode_stripping(BytecodeStripping::None) + .log_stream(Box::new(std::io::sink())) + .err_stream(Box::new(std::io::sink())); + let runtime = Runtime::new(config).map_err(|_| Failure::Limit)?; + runtime + .context() + .runtime() + .set_interrupt_handler(Some(Box::new(move || Instant::now() >= deadline))); + Ok(runtime) +} + +/// Compile a verified emitted artifact into privately cached bytecode without evaluation. +pub fn prepare(runtime_sha: &str, artifact_sha: &str, source: &str) -> Result { + if !model::digest_valid(runtime_sha) + || !model::digest_valid(artifact_sha) + || source.len() > MAX_SOURCE + || source.encode_utf16().count() > MAX_SOURCE_UTF16 + || model::hash(source.as_bytes()) != artifact_sha + { + return Err(Failure::Identity); + } + let bundle: Bundle = model::parse(source.as_bytes(), MAX_SOURCE)?; + if bundle.format != "qqq-esm/1" || !model::path_valid(&bundle.entry_point) || bundle.files.is_empty() || bundle.files.len() > 200 { + return Err(Failure::Input); + } + let mut files = BTreeMap::new(); + let mut names = BTreeSet::new(); + for file in bundle.files { + if !model::path_valid(&file.path) || !names.insert(file.path.to_ascii_lowercase()) { + return Err(Failure::Input); + } + files.insert(file.path, file.content); + } + if !files.contains_key(&bundle.entry_point) { + return Err(Failure::Input); + } + let runtime = new_runtime(64 * 1024 * 1024, Instant::now() + Duration::from_secs(10))?; + runtime + .context() + .runtime() + .set_loader(ClosedResolver(Rc::new(Cell::new(false))), SourceLoader(files.clone())); + let mut modules = Vec::new(); + let mut total = 0_usize; + for (path, content) in files { + let bytes = runtime.compile_to_bytecode(&path, &content).map_err(|_| Failure::Compile)?; + total = total.checked_add(bytes.len()).ok_or(Failure::Limit)?; + if total > MAX_BYTECODE { + return Err(Failure::Limit); + } + modules.push(CompiledModule { + path, + sha256: model::hash(&bytes), + bytecode: STANDARD.encode(bytes), + }); + } + let mut prepared = Prepared { + profile: PROFILE.into(), + runtime_sha256: runtime_sha.into(), + artifact_sha256: artifact_sha.into(), + entry_point: bundle.entry_point, + modules, + prepared_sha256: String::new(), + }; + prepared.prepared_sha256 = prepared.digest(); + Ok(prepared) +} + +/// Read a bounded unsigned execution limit from the unchanged worker request. +fn limit(request: &Value, name: &str, max: u64, zero_allowed: bool) -> Result { + let value = request + .get("limits") + .and_then(|limits| limits.get(name)) + .and_then(Value::as_u64) + .ok_or(Failure::Input)?; + if value > max || (!zero_allowed && value == 0) { + return Err(Failure::Limit); + } + Ok(value) +} + +/// Read bounded text without echoing it into an exception. +fn text<'a>(request: &'a Value, name: &str, max: usize) -> Result<&'a str, Failure> { + let value = request.get(name).and_then(Value::as_str).ok_or(Failure::Input)?; + if value.len() > max { + return Err(Failure::Limit); + } + Ok(value) +} + +/// Execute exactly one prepared publication with a fresh broker authority context. +pub fn execute( + runtime_sha: &str, + expected_prepared: &str, + prepared: Prepared, + request: Value, + broker: impl Fn(&[u8]) -> Result, Failure> + 'static, +) -> Result { + let started = Instant::now(); + let modules = prepared.validate(runtime_sha, expected_prepared)?; + let request_bytes = serde_json::to_vec(&request).map_err(|_| Failure::Input)?; + let _: Value = model::parse(&request_bytes, MAX_FRAME)?; + if request.get("bundle").is_some() + || text(&request, "protocol", 100)? != PROTOCOL + || text(&request, "type", 30)? != "execute" + || text(&request, "artifactSha256", 64)? != prepared.artifact_sha256 + { + return Err(Failure::Identity); + } + let entry = text(&request, "entryPoint", 200)?; + let entry_js = entry.strip_suffix(".ts").map(|base| format!("{base}.js")).ok_or(Failure::Input)?; + if entry_js != prepared.entry_point { + return Err(Failure::Identity); + } + let request_id = text(&request, "requestId", 128)?.to_owned(); + let token = text(&request, "capabilityToken", 200)?.to_owned(); + let event = text(&request, "event", 32)?.to_owned(); + if !matches!( + event.as_str(), + "CONTEXT" | "PRE_INSERT" | "PRE_UPDATE" | "PRE_DELETE" | "POST_INSERT" | "POST_UPDATE" | "POST_DELETE" | "AFTER_INSERT" | "AFTER_UPDATE" | "AFTER_DELETE" + ) { + return Err(Failure::Input); + } + let payload = request.get("payload").filter(|value| value.is_object()).ok_or(Failure::Input)?; + let wall = limit(&request, "wallMillis", 10_000, false)?; + let _cpu = limit(&request, "cpuMillis", 5_000, false)?; + let memory = limit(&request, "memoryBytes", 256 * 1024 * 1024, false)? as usize; + let output_limit = limit(&request, "outputBytes", 128 * 1024, false)? as usize; + let call_limit = limit(&request, "brokerCalls", 50, true)?; + let deadline = started + Duration::from_millis(wall); + let runtime = new_runtime(memory, deadline)?; + let entry_bytecode = modules + .iter() + .find(|(name, _)| name == &prepared.entry_point) + .ok_or(Failure::Identity)? + .1 + .clone(); + let loading_bytecode = Rc::new(Cell::new(false)); + runtime.context().runtime().set_loader( + ClosedResolver(Rc::clone(&loading_bytecode)), + BytecodeLoader(modules.into_iter().collect(), Rc::clone(&loading_bytecode)), + ); + let rejections = Rc::new(Cell::new(0_usize)); + let tracker = Rc::clone(&rejections); + runtime + .context() + .runtime() + .set_host_promise_rejection_tracker(Some(Box::new(move |_ctx, _promise, _reason, handled| { + tracker.set(if handled { + tracker.get().saturating_sub(1) + } else { + tracker.get().saturating_add(1) + }); + }))); + let broker_id = request_id.clone(); + let calls = Rc::new(Cell::new(0_u64)); + let native_calls = Rc::clone(&calls); + let context_json = serde_json::to_string(&json!({"event": event, "payload": payload})).map_err(|_| Failure::Input)?; + let outcome = runtime.context().with(|ctx| -> Result { + let native = Function::new(ctx.clone(), move |arguments: String| -> rquickjs::Result { + if Instant::now() >= deadline || native_calls.get() >= call_limit { + return Err(rquickjs::Error::new_from_js_message("query", "result", "BROKER_LIMIT")); + } + native_calls.set(native_calls.get() + 1); + let args: Value = model::parse(arguments.as_bytes(), MAX_FRAME).map_err(|_| rquickjs::Error::new_from_js("query", "request"))?; + let args = args + .as_array() + .filter(|args| args.len() == 4) + .ok_or_else(|| rquickjs::Error::new_from_js("query", "request"))?; + let message = json!({"protocol": PROTOCOL, "type": "broker", "requestId": broker_id, "operation": "query", "token": token, + "table": args[0], "fields": args[1], "criteria": args[2], "limit": args[3]}); + let bytes = serde_json::to_vec(&message).map_err(|_| rquickjs::Error::new_from_js("query", "request"))?; + if bytes.len() > MAX_FRAME { + return Err(rquickjs::Error::new_from_js("query", "request")); + } + let response = broker(&bytes).map_err(|_| rquickjs::Error::new_from_js_message("query", "result", "BROKER_DENIED"))?; + if Instant::now() >= deadline { + return Err(rquickjs::Error::new_from_js_message("query", "result", "DEADLINE")); + } + let parsed: Value = model::parse(&response, MAX_FRAME).map_err(|_| rquickjs::Error::new_from_js("query", "result"))?; + if parsed.get("protocol").and_then(Value::as_str) != Some(PROTOCOL) + || parsed.get("type").and_then(Value::as_str) != Some("brokerResult") + || parsed.get("ok").and_then(Value::as_bool) != Some(true) + || !parsed.get("rows").is_some_and(Value::is_array) + { + return Err(rquickjs::Error::new_from_js_message("query", "result", "BROKER_DENIED")); + } + String::from_utf8(response).map_err(|_| rquickjs::Error::new_from_js("query", "result")) + }) + .map_err(|_| Failure::Script)?; + let make_api: Function = ctx.eval(API_FACTORY).map_err(|_| Failure::Script)?; + let api: Object = make_api.call((native, call_limit)).map_err(|_| Failure::Script)?; + let context = ctx.json_parse(context_json).map_err(|_| Failure::Script)?; + let module = load_bytecode(ctx.clone(), &entry_bytecode, &loading_bytecode).map_err(|_| Failure::Script)?; + if module.name::().map_err(|_| Failure::Script)? != prepared.entry_point { + return Err(Failure::Identity); + } + let (module, evaluation) = module.eval().map_err(|_| Failure::Script)?; + settle(&ctx, &evaluation, deadline)?; + let main: Function = module.get("main").map_err(|_| Failure::Script)?; + let value: JsValue = main.call((context, api)).map_err(|_| Failure::Script)?; + let value = if let Some(promise) = value.as_promise() { + settle(&ctx, promise, deadline)? + } else { + value + }; + drain(&ctx, deadline)?; + if rejections.get() != 0 { + return Err(Failure::Script); + } + let encoded = ctx.json_stringify(value).map_err(|_| Failure::Output)?.ok_or(Failure::Output)?; + let encoded = encoded.to_string().map_err(|_| Failure::Output)?; + // Serialization can execute author toJSON/getters and enqueue Promise work. + // No author callback may run after the final job/rejection acceptance check. + drain(&ctx, deadline)?; + if rejections.get() != 0 { + return Err(Failure::Script); + } + let parsed: Value = model::parse(encoded.as_bytes(), output_limit)?; + if !parsed.is_object() { + return Err(Failure::Output); + } + Ok(parsed) + }); + let message = match outcome { + Ok(result) => json!({"protocol": PROTOCOL, "type": "executeResult", "requestId": request_id, "outcome": "OK", "result": result}), + Err(_) => json!({"protocol": PROTOCOL, "type": "executeResult", "requestId": request_id, "outcome": "SCRIPT_ERROR", + "error": {"code": "SCRIPT_FAILED", "message": "Script execution failed"}}), + }; + Ok(json!({"message": message, "guestWallMillis": started.elapsed().as_millis() as u64})) +} + +/// Await a real settled promise; an idle queue with a pending promise is a refusal. +fn settle<'js>(ctx: &Ctx<'js>, promise: &Promise<'js>, deadline: Instant) -> Result, Failure> { + for _ in 0..MAX_JOBS { + if Instant::now() >= deadline { + return Err(Failure::Limit); + } + if let Some(value) = promise.result::() { + return value.map_err(|_| Failure::Script); + } + if !ctx.execute_pending_job() || ctx.has_exception() { + return Err(Failure::Script); + } + } + Err(Failure::Limit) +} + +/// Drain remaining jobs and reject exceptions rather than accepting detached failures. +fn drain(ctx: &Ctx<'_>, deadline: Instant) -> Result<(), Failure> { + for _ in 0..MAX_JOBS { + if Instant::now() >= deadline { + return Err(Failure::Limit); + } + let pending = ctx.execute_pending_job(); + if ctx.has_exception() { + return Err(Failure::Script); + } + if !pending { + return Ok(()); + } + } + Err(Failure::Limit) +} + +/// Resolver restricted to the publication's canonical relative module graph. +struct ClosedResolver(Rc>); +impl Resolver for ClosedResolver { + fn resolve<'js>(&mut self, _ctx: &Ctx<'js>, base: &str, name: &str, attributes: Option>) -> rquickjs::Result { + if !self.0.get() && has_import_attributes(attributes) { + return Err(rquickjs::Error::new_resolving(base, name)); + } + model::resolve(base, name).map_err(|_| rquickjs::Error::new_resolving(base, name)) + } +} + +/// Compile-time loader returns only already validated emitted source. +struct SourceLoader(BTreeMap); +impl Loader for SourceLoader { + fn load<'js>(&mut self, ctx: &Ctx<'js>, name: &str, attributes: Option>) -> rquickjs::Result> { + if has_import_attributes(attributes) { + return Err(rquickjs::Error::new_loading(name)); + } + let source = self.0.get(name).ok_or_else(|| rquickjs::Error::new_loading(name))?; + Module::declare(ctx.clone(), name, source.as_bytes()) + } +} + +/// Invocation loader accepts only supervisor-owned, hash-validated compiler output. +struct BytecodeLoader(BTreeMap>, Rc>); +impl Loader for BytecodeLoader { + fn load<'js>(&mut self, ctx: &Ctx<'js>, name: &str, attributes: Option>) -> rquickjs::Result> { + if !self.1.get() && has_import_attributes(attributes) { + return Err(rquickjs::Error::new_loading(name)); + } + let bytes = self.0.get(name).ok_or_else(|| rquickjs::Error::new_loading(name))?; + let module = load_bytecode(ctx.clone(), bytes, &self.1)?; + if module.name::()? != name { + return Err(rquickjs::Error::new_loading(name)); + } + Ok(module) + } +} + +/// Refuse actual import attributes, including an attribute enumeration failure. +fn has_import_attributes(attributes: Option>) -> bool { + attributes.is_some_and(|attributes| attributes.keys().next().is_some()) +} + +/// Restore the exact prior loader phase, including nested loads and error unwinding. +struct BytecodePhase<'a>(&'a Cell, bool); +impl Drop for BytecodePhase<'_> { + fn drop(&mut self) { + self.0.set(self.1); + } +} + +/// Read only trusted compiler output. QuickJS bundled by rquickjs-sys 0.12.2 supplies a zeroed, +/// non-object attribute placeholder; its writer does not serialize attributes. +/// Source compilation rejects attributes before this cache is created. Evaluation +/// and dynamic imports run after this scope and retain the normal strict check. +fn load_bytecode<'js>(ctx: Ctx<'js>, bytes: &[u8], loading: &Cell) -> rquickjs::Result> { + let _phase = BytecodePhase(loading, loading.replace(true)); + // SAFETY: only exact privately compiled bytes retained by the trusted supervisor + // reach this path, after runtime/publication/module identity verification. + unsafe { Module::load(ctx, bytes) } +} diff --git a/wasm-runtime/src/sdk.js b/wasm-runtime/src/sdk.js new file mode 100644 index 0000000..70adb27 --- /dev/null +++ b/wasm-runtime/src/sdk.js @@ -0,0 +1,25 @@ +// Trusted SDK factory evaluated inside Wasm before any author module. +(() => { + const stringify = JSON.stringify; + const parse = JSON.parse; + const resolved = Promise.resolve.bind(Promise); + const freeze = Object.freeze; + return (native, callLimit) => { + let pending = false; + let calls = 0; + return freeze({ + query: async (table, fields, criteria = [], limit = 100) => { + if (pending || ++calls > callLimit) throw new Error("BROKER_LIMIT"); + pending = true; + try { + const frame = stringify([table, fields, criteria, limit]); + await resolved(); + const response = parse(native(frame)); + return response.rows; + } finally { + pending = false; + } + } + }); + }; +})() diff --git a/wasm-runtime/tests/trusted_contract.rs b/wasm-runtime/tests/trusted_contract.rs new file mode 100644 index 0000000..4955570 --- /dev/null +++ b/wasm-runtime/tests/trusted_contract.rs @@ -0,0 +1,250 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +//! Fixed trusted compatibility cases. These are not hostile-runtime qualification. +#![allow(clippy::expect_used)] + +use std::cell::Cell; +use std::rc::Rc; +use std::thread; +use std::time::Duration; + +use qqq_javy_runtime::{Failure, process}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +/// Synthetic exact runtime identity; an actual supervisor binds its loaded binary digest. +fn runtime_id() -> String { + "ab".repeat(32) +} + +/// Prepare only fixed test module sources; this is not TypeScript compilation. +fn prepare(files: Vec<(&str, &str)>) -> Value { + prepare_result(files).expect("trusted preparation") +} + +/// Keep compiler refusals observable without relaxing the trusted cache boundary. +fn prepare_result(files: Vec<(&str, &str)>) -> Result { + let bundle = json!({"format":"qqq-esm/1","entryPoint":"main.js","files":files.into_iter().map(|(path,content)|json!({"path":path,"content":content})).collect::>()}).to_string(); + let input = json!({"operation":"prepare","runtimeSha256":runtime_id(),"artifactSha256":format!("{:x}",Sha256::digest(bundle.as_bytes())),"bundle":bundle}); + let output = process(input.to_string().as_bytes(), |_| panic!("prepare must not broker"))?; + Ok(serde_json::from_slice(&output).expect("prepared JSON")) +} + +/// Create the exact hot request shape used by the persistent transport, without bundle. +fn request(prepared: &Value) -> Value { + json!({"protocol":"qbit-scripting-worker/1","type":"execute","requestId":"trusted-test","artifactId":"fixed", + "artifactSha256":prepared["artifactSha256"],"entryPoint":"main.ts","event":"CONTEXT","payload":{"actor":"alpha"},"capabilityToken":"synthetic-only", + "limits":{"wallMillis":2000,"cpuMillis":1000,"memoryBytes":64*1024*1024,"outputBytes":65536,"brokerCalls":2}}) +} + +/// Invoke through the public private-envelope boundary. +fn run(prepared: &Value, request: Value, broker: impl Fn(&[u8]) -> Result, Failure> + 'static) -> Result { + let input = json!({"operation":"run","runtimeSha256":runtime_id(),"preparedSha256":prepared["preparedSha256"],"prepared":prepared,"request":request}); + let output = process(input.to_string().as_bytes(), broker)?; + Ok(serde_json::from_slice(&output).expect("result JSON")) +} + +/// Exact existing broker response, containing only synthetic data. +fn allow(bytes: &[u8]) -> Result, Failure> { + let request: Value = serde_json::from_slice(bytes).expect("broker request"); + assert_eq!(request["operation"], "query"); + assert_eq!(request["token"], "synthetic-only"); + Ok( + json!({"protocol":"qbit-scripting-worker/1","type":"brokerResult","ok":true,"rows":[{"id":7}],"truncated":false}) + .to_string() + .into_bytes(), + ) +} + +/// Preparation must not execute module top-level effects or throw statements. +#[test] +fn preparation_does_not_evaluate_author_module() { + let prepared = prepare(vec![( + "main.js", + "throw new Error('fixed top level'); export function main(){return {ok:true}}", + )]); + let result = run(&prepared, request(&prepared), |_| panic!("no broker")).expect("bounded result"); + assert_eq!(result["message"]["outcome"], "SCRIPT_ERROR"); +} + +/// Java's artifact character limit is not an ASCII-only byte limit. +#[test] +fn unicode_artifact_above_one_megabyte_remains_compatible() { + let source = format!( + "const marker='{}'; export function main(){{return {{length:marker.length}}}}", + "界".repeat(400_000) + ); + assert!(source.len() > 1024 * 1024); + assert!(source.encode_utf16().count() < 1024 * 1024); + let prepared = prepare(vec![("main.js", source.as_str())]); + let result = run(&prepared, request(&prepared), |_| panic!("no broker")).expect("bounded Unicode result"); + assert_eq!(result["message"]["outcome"], "OK"); + assert_eq!(result["message"]["result"]["length"], 400_000); +} + +/// Separate fixed module, await and broker cases localize compatibility failures. +#[test] +fn fixed_compatibility_boundaries() { + for (name, source) in [ + ("relative", "import {id} from './state.js'; export function main(){return {id}}"), + ("tla", "await Promise.resolve(); export function main(){return {id:7}}"), + ( + "broker", + "export async function main(c,q){const rows=await q.query('purchaseOrder',['id']);return {id:rows[0].id}}", + ), + ] { + let prepared = prepare(vec![("main.js", source), ("state.js", "export const id=7;")]); + let result = run(&prepared, request(&prepared), allow).expect("fixed boundary result"); + assert_eq!(result["message"]["outcome"], "OK", "boundary: {name}"); + assert_eq!(result["message"]["result"]["id"], 7); + } +} + +/// Bytecode loading restores strict runtime import checks after nested module reads. +#[test] +fn nested_bytecode_and_dynamic_imports_preserve_attribute_policy() { + for attributes in ["", ", {with:{type:'json'}}"] { + let source = format!( + "import {{id}} from './nested/state.js'; export async function main(){{const extra=await import('./extra.js'{attributes});return {{id,extra:extra.id}}}}" + ); + let prepared = prepare(vec![ + ("main.js", &source), + ("nested/state.js", "export {id} from '../value.js';"), + ("value.js", "export const id=7;"), + ("extra.js", "export const id=8;"), + ]); + let result = run(&prepared, request(&prepared), |_| panic!("no broker")).expect("bounded dynamic result"); + assert_eq!(result["message"]["outcome"], if attributes.is_empty() { "OK" } else { "SCRIPT_ERROR" }); + if attributes.is_empty() { + assert_eq!(result["message"]["result"], json!({"id":7,"extra":8})); + } + } +} + +/// Attributed source imports never enter the private compiled-bytecode cache. +#[test] +fn attributed_static_import_is_refused_at_preparation() { + for attributed_path in ["main.js", "unused.js"] { + let mut files = vec![ + ( + attributed_path, + "import {id} from './state.js' with {type:'json'}; export function main(){return {id}}", + ), + ("state.js", "export const id=7;"), + ]; + if attributed_path != "main.js" { + files.push(("main.js", "export function main(){return {ok:true}}")); + } + assert_eq!(prepare_result(files).expect_err("unsupported import attributes"), Failure::Compile); + } +} + +/// Relative modules, live bindings, TLA, async main and Promise broker remain observable. +#[test] +fn fresh_modules_time_random_and_async_broker() { + let prepared = prepare(vec![ + ( + "state.js", + "export let count=0; export const stamp=Date.now(); export const random=Math.random(); export function bump(){count++}", + ), + ( + "main.js", + "import {count,stamp,random,bump} from './state.js'; await Promise.resolve(); export async function main(context,qqq){const rows=await qqq.query('purchaseOrder',['id']); bump(); return {count,stamp,random,id:rows[0].id,actor:context.payload.actor}}", + ), + ]); + let first = run(&prepared, request(&prepared), allow).expect("first invocation"); + thread::sleep(Duration::from_millis(5)); + let mut second_request = request(&prepared); + second_request["payload"]["actor"] = json!("beta"); + let second = run(&prepared, second_request, allow).expect("second invocation"); + assert_eq!(first["message"]["outcome"], "OK"); + assert_eq!(second["message"]["outcome"], "OK"); + let a = &first["message"]["result"]; + let b = &second["message"]["result"]; + assert_eq!(a["count"], 1); + assert_eq!(b["count"], 1); + assert_eq!(a["id"], 7); + assert_eq!(b["actor"], "beta"); + assert_ne!(a["stamp"], b["stamp"]); + assert_ne!(a["random"], b["random"]); +} + +/// Pending promises cannot be treated as successful completion. +#[test] +fn idle_pending_main_is_refused() { + let prepared = prepare(vec![("main.js", "export function main(){return new Promise(()=>{})}")]); + let result = run(&prepared, request(&prepared), |_| panic!("no broker")).expect("bounded result"); + assert_eq!(result["message"]["outcome"], "SCRIPT_ERROR"); +} + +/// A second unawaited SDK call is refused by the existing one-pending-call rule. +#[test] +fn simultaneous_broker_calls_are_refused() { + let prepared = prepare(vec![( + "main.js", + "export async function main(c,q){await Promise.all([q.query('purchaseOrder',['id']),q.query('purchaseOrder',['id'])]);return {ok:true}}", + )]); + let calls = Rc::new(Cell::new(0)); + let observed = Rc::clone(&calls); + let result = run(&prepared, request(&prepared), move |bytes| { + observed.set(observed.get() + 1); + allow(bytes) + }) + .expect("bounded result"); + assert_eq!(result["message"]["outcome"], "SCRIPT_ERROR"); + assert!(calls.get() <= 1); +} + +/// Current broker refusal is not converted into an authorized row result. +#[test] +fn broker_denial_is_generic() { + let prepared = prepare(vec![( + "main.js", + "export async function main(c,q){return {rows:await q.query('purchaseOrder',['id'])}}", + )]); + let result = run(&prepared, request(&prepared), |_| Err(Failure::Broker)).expect("bounded result"); + assert_eq!(result["message"]["outcome"], "SCRIPT_ERROR"); + assert_eq!(result["message"]["error"]["message"], "Script execution failed"); +} + +/// Cached code is bound to both publication bytes and runtime identity before execution. +#[test] +fn altered_bytecode_or_runtime_never_brokers() { + let prepared = prepare(vec![("main.js", "export function main(){return {ok:true}}")]); + let mut changed = prepared.clone(); + changed["modules"][0]["bytecode"] = json!("AA=="); + assert_eq!( + run(&changed, request(&prepared), |_| panic!("must not broker")).expect_err("changed bytes"), + Failure::Identity + ); + changed = prepared.clone(); + changed["runtimeSha256"] = json!("cd".repeat(32)); + assert_eq!( + run(&changed, request(&prepared), |_| panic!("must not broker")).expect_err("changed runtime"), + Failure::Identity + ); +} + +/// Author serialization callbacks cannot create an unchecked detached rejection. +#[test] +fn detached_rejection_during_result_serialization_is_refused() { + let prepared = prepare(vec![( + "main.js", + "export function main(){return {toJSON(){Promise.reject(new Error('fixed trusted rejection'));return {ok:true}}}}", + )]); + let result = run(&prepared, request(&prepared), |_| panic!("no broker")).expect("bounded serialization result"); + assert_eq!(result["message"]["outcome"], "SCRIPT_ERROR"); +} + +/// Return values are bounded plain objects and preserve declared output limits. +#[test] +fn output_limit_and_array_results_refused() { + for source in ["export function main(){return {text:'x'.repeat(200)}}", "export function main(){return [1,2]}"] { + let prepared = prepare(vec![("main.js", source)]); + let mut input = request(&prepared); + input["limits"]["outputBytes"] = json!(64); + let result = run(&prepared, input, |_| panic!("no broker")).expect("bounded result"); + assert_eq!(result["message"]["outcome"], "SCRIPT_ERROR"); + } +} diff --git a/worker/.dockerignore b/worker/.dockerignore new file mode 100644 index 0000000..7717b0c --- /dev/null +++ b/worker/.dockerignore @@ -0,0 +1,5 @@ +node_modules +dist +coverage +*.log +.git diff --git a/worker/.gitignore b/worker/.gitignore new file mode 100644 index 0000000..e6d5efa --- /dev/null +++ b/worker/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +coverage/ +*.log diff --git a/worker/Dockerfile b/worker/Dockerfile new file mode 100644 index 0000000..4e6e058 --- /dev/null +++ b/worker/Dockerfile @@ -0,0 +1,25 @@ +# Supply approved immutable Node 24.22.2 trixie-slim and distroless nonroot +# runtime references. Neither stage has a default floating image tag. +ARG NODE_BUILD_IMAGE +ARG NODE_RUNTIME_IMAGE +FROM ${NODE_BUILD_IMAGE} AS build +WORKDIR /worker +COPY package.json package-lock.json ./ +COPY LICENSE NOTICE ./ +RUN test "$(node --version)" = "v24.22.2" && test "$(npm --version)" = "11.19.1" +RUN npm ci --ignore-scripts --no-audit --no-fund +COPY tsconfig.json ./ +COPY src ./src +RUN npm run build && npm prune --omit=dev --ignore-scripts + +FROM ${NODE_RUNTIME_IMAGE} +# Distroless has no shell or package manager. Keep the complete TypeScript +# package below so its shipped standard library declarations remain available. +USER 65532:65532 +RUN ["/nodejs/bin/node", "-e", "if(process.versions.node!=='24.22.2'||process.getuid()!==65532)process.exit(1)"] +WORKDIR /worker +COPY --from=build /worker/dist ./dist +COPY --from=build /worker/node_modules ./node_modules +COPY --from=build /worker/package.json ./ +COPY --from=build /worker/LICENSE /worker/NOTICE ./ +ENTRYPOINT ["/nodejs/bin/node", "--max-old-space-size=128", "/worker/dist/entrypoint.js"] diff --git a/worker/LICENSE b/worker/LICENSE new file mode 100644 index 0000000..4bb040e --- /dev/null +++ b/worker/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to the Licensor for inclusion in the Work by the copyright + owner or by an individual or Legal Entity authorized to submit on + behalf of the copyright owner. For the purposes of this definition, + "submitted" means any form of electronic, verbal, or written + communication sent to the Licensor or its representatives, including + but not limited to communication on electronic mailing lists, source + code control systems, and issue tracking systems that are managed by, + or on behalf of, the Licensor for the purpose of discussing and + improving the Work, but excluding communication that is conspicuously + marked or otherwise designated in writing by the copyright owner as + "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2024 Kingsrook, LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/worker/NOTICE b/worker/NOTICE new file mode 100644 index 0000000..6fbe46b --- /dev/null +++ b/worker/NOTICE @@ -0,0 +1,6 @@ +QBit Scripting +Copyright 2026 Kingsrook, LLC + +This product includes software developed at Kingsrook, LLC (https://kingsrook.com/). + +Licensed under the Apache License, Version 2.0. diff --git a/worker/QUALIFICATION.md b/worker/QUALIFICATION.md new file mode 100644 index 0000000..fa0d448 --- /dev/null +++ b/worker/QUALIFICATION.md @@ -0,0 +1,45 @@ +# Runtime qualification status + +Status: **not qualified for untrusted production workloads**. There is no production-approved image digest, runsc attestation, deployment evidence ID, or hostile containment test result from this session. Docker Desktop Linux availability is insufficient. The launcher has no process/runc fallback and refuses an unconfigured production invocation. + +Implementation verification on 2026-10-30 uses pinned Node 24.22.0, TypeScript 6.0.3, strict compiler checks, and ESLint. Tests exercise deterministic compilation, type errors, import policy, manifest hashes, path bounds, byte framing, configuration refusal, broker round trips, context results, and host transport timeout/output/cancellation/token checks. Synthetic host processes run only fixed repository-owned harmless fixtures. They do not test container escape resistance, and no adversarial user code was executed on the host. Exact latest test counts belong in the implementation report. + +Verification update 2026-11-02: the worker enforces Node 24.22.2 exactly (an older note above says 24.22.0). Current source passes `npm test` (18/18) and `npm run lint` (0 warnings) under the pinned Node 24.22.2 binary (sha256 `1C331340FA6CAC5E4294725959070A143FA0507195A78AD77EED7943F719AAC9`); under any other Node the strict pin refuses, which is intended. See `../deploy/BUILD-CLOSURE.md`. This does not change the not-qualified status below. + +Before a deployment may assign an accepted qualification identifier: + +1. Pin and review the Node base and final image digests, dependency lock, SBOM, licensing, and current vulnerability findings. Build reproducibly and retain image provenance. The Dockerfile comment alone does not enforce an immutable base argument. +2. Attest the actual Linux host, gVisor binary/version/configuration, Docker runtime mapping, default seccomp policy, cgroup support, and isolation configuration. A runtime merely named `runsc` is insufficient. +3. Deploy and qualify the implemented independent `reaper-cli.js` and durable attempt journal for abandoned containers. Service/timer examples are supplied. Synthetic tests verify exact immutable ID removal, ownership mismatch refusal, lease expiry, create-response loss, retry after removal failure, and fail-closed durability errors. Test real coordinator SIGKILL, host restart, Docker disconnect, stuck removal, and ambiguous create timeouts. Alert on cleanup failures and prevent accumulated or orphaned workers. +4. Verify wall time, total CPU accounting across child processes, RSS/memory/OOM, tmpfs, PID/file descriptor, output, source and diagnostic bounds. Compilation is hostile work too. The current CPU policy combines cgroup throttling with per-process RLIMIT_CPU; it does not prove an exact invocation-wide CPU budget. +5. In a disposable qualified test environment, exercise filesystem, network, syscall, process creation, device, environment/secret, and cross-invocation isolation boundaries. No host application data, sockets, secrets, or filesystem mounts may be supplied to a worker. Native Node access inside its own container is expected, so tests must target the real OS boundary. +6. Verify the concrete Java `WorkerProvider` adapter, cancellation through container removal, caller-thread broker callbacks, source/artifact signatures, current caller/tenant authorization, bounded query returns, published context output checks, audited failures, and output acceptance only after host cleanup. +7. Load test compile and execution under the proposed deployment's concurrency limits. Account for Docker setup/cleanup latency and TypeScript baseline memory. Demonstrate recovery under capacity exhaustion and establish service limits from measurements. + +Qualification evidence must be versioned against the image, runtime, host policy, Java adapter, and protocol. Changing them requires review. This file records missing evidence, not an approval or a claim of complete security. + +An operator-only harness is implemented: `node dist/qualification-cli.js --run-qualified-fixtures /absolute/new-evidence.jsonl`. It is never invoked by build/test. Run it only in an explicitly authorized disposable Linux/runsc laboratory with all server-owned environment settings present. It compiles a fixed fixture inside the sandbox, verifies a broker/context round trip, checks termination of a fixed endless-loop fixture, and records fsynced evidence. The loop is never executed on the host. The harness deliberately records `completeQualification: false`; these checks do not replace the remaining host-failure, resource, and escape-boundary assessment. + +## Primary documentation consulted + +- [TypeScript noEmitOnError](https://www.typescriptlang.org/tsconfig/noEmitOnError.html) and [compiler options](https://www.typescriptlang.org/tsconfig/): strict checking and emit behavior are compiler settings, not an execution sandbox. +- [Node child_process](https://nodejs.org/api/child_process.html): separate processes, bounded buffers/timeouts, and argument arrays without a shell. +- [Node TypeScript support](https://nodejs.org/api/typescript.html): native type stripping does not replace TypeScript type checking and does not use tsconfig as this compiler does. +- [gVisor Docker quick start](https://gvisor.dev/docs/user_guide/quick_start/docker/) and [production guidance](https://gvisor.dev/docs/user_guide/production/): configuring runsc and qualifying a deployment is separate from building a container image. +- [Docker container run reference](https://docs.docker.com/reference/cli/docker/container/run): runtime, namespaces, read-only filesystem, capabilities, memory/CPU/PID controls, and security options. Actual support must be verified on the deployment host. + +## Security baseline correction (2026-10-31) + +The historical24.22.0 verification above does not qualify the corrected source. Current source requires Node24.22.1 after the official October21 security release. The original image and scan reports remain immutable; a new candidate, trusted regression run and complete scan must establish the updated evidence. Removing unused package managers is separate image minimization and cannot substitute for patching Node's embedded dependencies. Hostile-runtime isolation qualification remains outstanding. + +## Corrected implementation evidence (2026-10-31) + +Node24.22.1 compile/lint/17 trusted tests and new OCI build passed. Candidate qbit-documents/artifacts/worker-image-20261031-0059 records exact identities and new231-occurrence OS scan, with0 Node-package findings. Package-manager pruning removes npm/Yarn; Corepack remains inventoried. Runtime/escape qualification remains outstanding; no hostile work or launcher fallback occurred. Original0020 evidence is unchanged. + +## 2026-10-31 Node24.22.2 Distroless candidate + +All exact worker guards now require24.22.2. Compile/lint and17 trusted tests pass. Separate immutable Trixie builder and Distroless runtime produce candidate worker-image-20261031-0240; final-image trusted compilation and fixed nonroot/runtime/library/license checks pass. Independent local scan reports32 OS occurrences (23medium,8low,1unknown),0high/critical and0 Node-package findings. Base index signature verifies; upstream SPDX predicate verification remains unresolved and is not called verified. No hostile code ran, no global runtime changed, and no runsc/deployment qualification is inferred. Historical evidence remains unchanged. + +### 02:56 SPDX verification follow-up + +The exact upstream legacy SPDX attestation now verifies using unmodified patched Cosign2.6.5 with the same official identity, issuer and digest. Wrong signer and wrong subject digest are refused. Original3.1.4 predicate failures remain preserved. See ../../qbit-documents/docs/reviews/worker-spdx-verification-20261031-0256.md. This closes only attestation verification; image/source bytes are unchanged and hostile-runtime, vulnerability, license and deployment gates remain open. diff --git a/worker/README.md b/worker/README.md new file mode 100644 index 0000000..cff95ab --- /dev/null +++ b/worker/README.md @@ -0,0 +1,79 @@ +# Isolated TypeScript worker + +This directory implements strict TypeScript compilation, an ESM execution worker, and a host transport that creates one disposable gVisor container per compilation or execution. It is not a qualified production sandbox yet. See [QUALIFICATION.md](QUALIFICATION.md) before enabling untrusted projects. + +The Java QBit owns authentication, record authorization, published artifact signatures, capability grants, quotas, hook semantics, and audit persistence. This worker owns compilation and isolated execution. There are no Java QQQ objects, application tokens, database credentials, network connections, or application filesystem mounts inside the worker. + +## Build and verification + +Use Node 24.22.2 and npm 11.19.1: + +```text +npm ci --ignore-scripts +npm run lint +npm test +``` + +TypeScript 6.0.3 and all development dependencies are pinned in `package-lock.json`. No user project packages, lifecycle scripts, compiler configuration, plugins, or build hooks are installed or loaded. The Dockerfile requires separate operator-supplied `NODE_BUILD_IMAGE` and `NODE_RUNTIME_IMAGE` arguments. Release tooling must verify the approved immutable Node 24.22.2 Trixie builder and Distroless nonroot runtime digests. Neither stage has a default image tag. The final worker image must also be supplied by immutable digest and already exist on the Docker host (`--pull=never`). The current local candidate and verification evidence are in `../../qbit-documents/artifacts/worker-image-20261031-0240/README.md`. Historical 0020 and 0059 candidates remain immutable evidence. None is qualified for hostile execution. + +## Host integration + +`node dist/coordinator-cli.js` reads one JSON line on stdin, emits any broker requests as JSON lines on stdout, reads their JSON-line answers from stdin, and emits one final response. It exits after that invocation. stderr contains only a generic failure message. The host process never imports or evaluates the submitted compiler output. `src/entrypoint.ts` is the separate container entry point; its environment marker is a misuse guard, not security isolation. + +The deployment supplies these server-owned environment values: + +| Variable | Meaning | +| --- | --- | +| `QQQ_WORKER_IMAGE` | Approved final worker image reference ending in `@sha256:<64 hex digits>` | +| `QQQ_WORKER_QUALIFICATION_ID` | Identifier of external qualification evidence for this deployment | +| `QQQ_REQUIRED_QUALIFICATION_ID` | Independently configured identifier required by the application | +| `QQQ_WORKER_OWNER_ID` | Stable unique deployment identity for container ownership labels | +| `QQQ_WORKER_STATE_DIRECTORY` | Existing absolute directory owned exclusively by the coordinator/reaper account, on durable local POSIX storage supporting directory fsync | + +Missing values, mismatched identifiers, non-Linux Docker, or missing `runsc` reject the invocation. The identifier comparison checks configuration; it cannot establish that the operator actually performed qualification. A runtime named `runsc` alone does not prove its binary/configuration is authentic. The reviewed deployment must control that configuration and evidence. + +The Java `WorkerProvider` adapter must start this CLI with fixed executable/arguments/environment, bound its overall lifetime and protocol buffers, call `BrokerChannel` synchronously on the initiating authenticated QQQ thread, and send its reply to stdin. It must never expose the Docker socket to scripts. Node's broker promise remains asynchronous inside the worker. The sibling Java QBit now supplies CoordinatorWorkerProvider implementing this bridge; its transport tests use trusted fixed subprocess fixtures. This directory does not register itself in QQQ, and neither test suite constitutes sandbox qualification. + +## Protocol version 1 + +Every request has `protocol: "qbit-scripting-worker/1"`, `type`, and `requestId`. Frames are newline-delimited UTF-8 JSON objects. Input is bounded at 4 MiB before parsing; nesting is bounded. No arbitrary command arguments cross the protocol. + +Compile requests contain `strict: true`, `toolchainId: "typescript-6.0.3-node-24"`, `entryPoint`, `files: [{path, content}]`, and `sourceSha256`. The canonical source hash sorts ASCII paths, then hashes each UTF-8 path and content with two unsigned big-endian 32-bit length prefixes. Sources are limited to 200 files, 2 MiB aggregate UTF-8, 262,144 characters per file, and relative `.ts` paths. The compiler uses server-owned strict options with `noEmitOnError`, only shipped ES2022 libraries, and approved relative static imports or `import type` from `@qqq/sdk`. It emits no artifact on diagnostics. + +The successful `compileResult` contains `success`, `strict`, `toolchainId`, `bundle`, `bundleSha256`, and `sourceMap`. The bundle is an opaque deterministic JSON module archive (`format: "qqq-esm/1"`), not a concatenated JavaScript program. It contains `entryPoint` and `files` with relative `.js` paths. Source maps are returned separately, including original sources, and must receive the same access protection as the source. Java must verify and sign the resulting immutable published artifact. The worker SHA-256 is integrity metadata, not proof of trusted origin. + +Execute requests contain the published `bundle`, `artifactSha256`, source `entryPoint`, `event`, `payload`, `capabilityToken`, and integer `limits`. The worker verifies the bundle hash, writes modules to its disposable `/tmp` filesystem, and invokes `main({event, payload}, qqq)`. Successful responses have `type: "executeResult"`, `outcome: "OK"`, `result`, and informational `usage`; failures expose no user stack traces. Worker results and usage are untrusted and require Java validation. + +For record hooks, Java validates `ALLOW`, `PATCH`, or `REJECT`. For published context scripts, `event` is `CONTEXT`, `payload` is `{context: }`, and the script's result is an arbitrary bounded JSON object, without hook coercion. Example: + +```typescript +import type { Context, Qqq } from "@qqq/sdk"; + +export async function main(context: Context, qqq: Qqq) +{ + const products = await qqq.query("product", ["id", "name"], [], 20); + return { record: context.payload.context, products }; +} +``` + +`qqq.query(table, fields, criteria, limit)` emits a broker frame with `operation: "query"`, request ID, token, and arguments. The host checks token, operation, and call count before forwarding. Java separately authorizes table, fields, criteria, record scope, tenant, and result budgets using its server-created grant. The reply has `type: "brokerResult"`, protocol, `ok`, and `rows`; denial causes script failure. Only one broker call can be outstanding through the SDK. This SDK intentionally has no generic URL fetch, arbitrary QQQ operation, or mutation method. + +## Bounds and termination + +Compilation has a 10-second attached execution wall deadline, 5 CPU seconds, 256 MiB container memory, 32 PIDs, 64 file descriptors, and 4 MiB aggregate output. Execution accepts wall deadlines up to 10 seconds, whole-second CPU budgets from 1 to 5 seconds, memory from 64 to 256 MiB, output up to 128 KiB, and up to 50 broker calls. Unsupported budgets are rejected instead of rounded upward. Output counts stdout and stderr, including broker protocol traffic. Broker replies are at most 64 KiB per frame. These are initial bounds, not measured throughput or a performance SLA. + +The launcher sets `runsc`, no network, read-only root filesystem, nonroot UID/GID 65532, all capabilities dropped, no new privileges, no IPC namespace sharing, no ports or mounts, bounded memory/swap/CPU/PIDs, and a 64 MiB noexec/nosuid/nodev `/tmp`. Cgroup CPU rate and whole-second per-process RLIMIT_CPU are additional constraints, not a proven aggregate CPU accounting implementation. In particular, child processes can affect total CPU usage; qualification must validate the deployment's aggregate resource policy. + +An external host timer, AbortSignal, output limit, or malformed protocol terminates the attached client. Cleanup inspects the exact prepared container name, verifies deployment and attempt labels, then uses `docker rm --force` with its immutable 64-character container ID. Removal failure prevents success and must alert operations. Docker info/create/inspect/remove have separate 10-second timeouts, so total API latency may exceed the script's attached wall limit. + +Before creation, the launcher durably writes attempt evidence and a 60-second lease. `node dist/reaper-cli.js` recovers expired leases after coordinator death, including a crash between Docker creation and receipt of its ID. It never enumerates or removes unrelated Docker containers. Successful removal/absence is durably recorded before deleting the active lease; failed cleanup remains retryable. Evidence contains attempt IDs and lifecycle state, not source, records, or capability tokens. The evidence directory is append-only by convention and must be protected/retained by the deployment; this journal does not replace the Java QBit's user-visible audit. + +Deploy the independent reaper using the supplied service/timer examples in [deploy](deploy/README.md). Its live crash-recovery and durable-storage behavior still require qualification. Windows synthetic tests inject the directory-sync operation only; production never skips an unsupported fsync and fails closed. No host-crash cleanup guarantee is claimed while the reaper is not deployed or the Docker host is unavailable. + +Strict TypeScript is a correctness feature. Native module execution deliberately treats the entire worker process as compromised by hostile script code. Type declarations, import restrictions, object freezing, environment markers, and worker-generated telemetry do not provide hostile-code containment. The OS sandbox, trusted host validation, and independently qualified deployment do. + +## Node security baseline (2026-10-31) + +The historical 24.22.1 correction followed [the official security release](https://nodejs.org/en/blog/release/v24.22.1), which fixes the bundled Brotli decoder and updates built-in Undici to7.30.1. The original Node24.22.0 binary contained built-in Undici7.29.1, while the image scanner separately found npm's bundled Undici6.28.0 under `/usr/local/lib/node_modules/npm/node_modules/undici`; removing npm does not patch Node itself. Current exact host coordinator, compiler and container entry checks require Node24.22.2, whose [official release](https://nodejs.org/en/blog/release/v24.22.2) includes V8 backports and a stream async-context correction. No global runtime installation or existing image replacement is implied. + +Current review candidate: ../../qbit-documents/artifacts/worker-image-20261031-0240 (Node24.22.2, Distroless runtime,17 trusted tests plus actual final-image compilation passed). The runtime uses the absolute `/nodejs/bin/node` entrypoint and retains the complete TypeScript library and license files. Local scan reports32 OS findings, none high/critical, and no Node-package findings; this is not a blanket security or license clearance. The base index signature verifies. Its legacy SPDX attestation also verifies with unmodified patched Cosign2.6.5, including wrong-signer/digest refusal controls; Cosign3.1.4's earlier predicate failures remain preserved. See [the verification follow-up](../../qbit-documents/docs/reviews/worker-spdx-verification-20261031-0256.md). Hostile runtime qualification remains false. diff --git a/worker/deploy/README.md b/worker/deploy/README.md new file mode 100644 index 0000000..817f170 --- /dev/null +++ b/worker/deploy/README.md @@ -0,0 +1,23 @@ +# Deployment examples — not installed automatically + +The supplied systemd service/timer runs the exact-owned-container reaper independently every ten seconds and after boot. Review paths, service account, permissions, daemon access, timeout, monitoring, and host policy before installation. These units have not been installed/enabled, and Docker runtime configuration has not been changed. Local image candidates and fixed trusted compatibility fixtures exist; the hostile qualification harness has not run. The current SDK candidate and scoped evidence are in [worker0447](../../../qbit-documents/artifacts/worker-sdk-image-20261031-0447/README.md). It remains unqualified for hostile workloads. + +Provision `/opt/qqq-worker` with the verified build, `/opt/node/bin/node` with the pinned runtime, and an existing `/var/lib/qqq-worker` owned only by the trusted `qqq-worker` account (mode 0700). State storage must support durable file and directory fsync. The account needs Docker daemon access; that is privileged host authority and belongs exclusively to this trusted coordinator/reaper, never the sandbox. The units do not grant group membership or alter the host's Docker configuration. + +Create a protected `/etc/qqq-worker/runtime.env` containing deployment-specific, server-owned settings: + +```text +QQQ_WORKER_IMAGE=registry.example/qqq-worker@sha256: +QQQ_WORKER_QUALIFICATION_ID= +QQQ_REQUIRED_QUALIFICATION_ID= +QQQ_WORKER_OWNER_ID= +QQQ_WORKER_STATE_DIRECTORY=/var/lib/qqq-worker +``` + +Placeholders intentionally cannot pass the launch gate. Supply the same owner/state settings to the Java coordinator adapter. Restrict image registry contents and runtime configuration to deployment administrators. Do not put application credentials in this environment. + +Active lease files contain an immutable attempt UUID, exact container name, owner and timestamps. The launcher adds matching Docker labels. The reaper selects only expired leases for its configured owner, inspects each exact name, verifies both labels, and removes its immutable ID. It never issues `docker system prune`, wildcard kills, or name-prefix enumeration. Ownership mismatch or daemon failure retains the lease and records failure. A later run retries. Successful/absent cleanup records evidence before removing the lease, so a crash can repeat a safe inspection. Up to 1,000 expired leases are handled per invocation; alert and increase deployment capacity when cleanup backlogs grow. + +The 60-second lease exceeds the launcher's ordinary create, execution, inspection and removal budget. Running the reaper every ten seconds normally gives an abandoned worker a roughly 70-second maximum lifetime after preparation, plus Docker response time. This is conditional on a healthy timer, durable filesystem, and daemon; it is not a guarantee under host failure. A daemon/host outage requires restoring the service and running recovery before accepting new work. Monitor nonzero exit codes, expired active-lease count, evidence storage capacity, timer freshness and Docker availability. Retain lifecycle evidence according to the application's audit policy. + +For an authorized sandbox lab, run the qualification harness described in [QUALIFICATION.md](../QUALIFICATION.md). Also test kill/restart scenarios against the actual service deployment. Synthetic tests validate ownership and recovery logic but do not validate host filesystem durability or gVisor containment. diff --git a/worker/deploy/qqq-script-reaper.service b/worker/deploy/qqq-script-reaper.service new file mode 100644 index 0000000..6506bc7 --- /dev/null +++ b/worker/deploy/qqq-script-reaper.service @@ -0,0 +1,20 @@ +[Unit] +Description=Reap expired QQQ script containers with verified ownership +After=docker.service + +[Service] +Type=oneshot +User=qqq-worker +Group=qqq-worker +EnvironmentFile=/etc/qqq-worker/runtime.env +WorkingDirectory=/opt/qqq-worker +ExecStart=/opt/node/bin/node /opt/qqq-worker/dist/reaper-cli.js +TimeoutStartSec=120 +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +PrivateTmp=true +ReadWritePaths=/var/lib/qqq-worker + +[Install] +WantedBy=multi-user.target diff --git a/worker/deploy/qqq-script-reaper.timer b/worker/deploy/qqq-script-reaper.timer new file mode 100644 index 0000000..65a7a86 --- /dev/null +++ b/worker/deploy/qqq-script-reaper.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Check QQQ script container leases every ten seconds + +[Timer] +OnBootSec=5s +OnUnitInactiveSec=10s +AccuracySec=1s +Persistent=true +Unit=qqq-script-reaper.service + +[Install] +WantedBy=timers.target diff --git a/worker/eslint.config.mjs b/worker/eslint.config.mjs new file mode 100644 index 0000000..ec61b8c --- /dev/null +++ b/worker/eslint.config.mjs @@ -0,0 +1,77 @@ +// Copyright (C) 2026 Kingsrook, LLC. +// SPDX-License-Identifier: Apache-2.0 +// KofTwentyTwo TypeScript / JavaScript lint configuration (K22-CODE-TS). +// The Kingsrook layout (3-space indent, Allman braces, if(...) with no space, no more than +// one blank line inside code) is enforced by ESLint Stylistic rather than Prettier, which +// cannot express Allman braces. Type-aware strict rules come from typescript-eslint. +// Install: npm i -D eslint @eslint/js typescript-eslint @stylistic/eslint-plugin typescript +import js from "@eslint/js"; +import stylistic from "@stylistic/eslint-plugin"; +import tseslint from "typescript-eslint"; + +export default tseslint.config( + { + ignores: ["dist/**", "build/**", "coverage/**", "node_modules/**", "**/*.generated.*"], + }, + js.configs.recommended, + tseslint.configs.strictTypeChecked, + tseslint.configs.stylisticTypeChecked, + { + languageOptions: { + parserOptions: { + projectService: true, + tsconfigRootDir: import.meta.dirname, + }, + }, + plugins: { "@stylistic": stylistic }, + rules: { + // Kingsrook layout + "@stylistic/indent": ["error", 3, { SwitchCase: 1 }], + "@stylistic/brace-style": ["error", "allman", { allowSingleLine: false }], + "@stylistic/keyword-spacing": ["error", { + before: true, + after: true, + overrides: { + if: { after: false }, + for: { after: false }, + while: { after: false }, + switch: { after: false }, + catch: { after: false }, + }, + }], + // 1 blank line inside code; up to 3 so methods can be separated by 3. + "@stylistic/no-multiple-empty-lines": ["error", { max: 3, maxBOF: 0, maxEOF: 0 }], + "@stylistic/lines-between-class-members": ["error", "always", { exceptAfterSingleLine: true }], + "@stylistic/semi": ["error", "always"], + "@stylistic/quotes": ["error", "double", { avoidEscape: true }], + "@stylistic/comma-dangle": ["error", "always-multiline"], + "@stylistic/eol-last": ["error", "always"], + "@stylistic/no-tabs": "error", + "@stylistic/no-trailing-spaces": "error", + "@stylistic/operator-linebreak": ["error", "before"], + "@stylistic/dot-location": ["error", "property"], + "@stylistic/max-len": "off", + + // Kingsrook non-layout rules + "curly": ["error", "all"], + "eqeqeq": ["error", "always"], + "no-console": "error", + "no-warning-comments": ["error", { terms: ["todo", "fixme"], location: "start" }], + "@typescript-eslint/explicit-function-return-type": "error", + "@typescript-eslint/naming-convention": ["error", + { selector: "default", format: ["camelCase"] }, + { selector: "variable", modifiers: ["const", "global"], format: ["camelCase", "UPPER_CASE"] }, + { selector: "typeLike", format: ["PascalCase"] }, + { selector: "enumMember", format: ["PascalCase", "UPPER_CASE"] }, + { selector: "import", format: null }, + { selector: "objectLiteralProperty", format: null }, + ], + }, + }, + { + files: ["**/*.js", "**/*.mjs", "**/*.cjs"], + extends: [tseslint.configs.disableTypeChecked], + // Plain-JavaScript test fixtures cannot carry TypeScript return annotations. + rules: { "@typescript-eslint/explicit-function-return-type": "off" }, + }, +); diff --git a/worker/package-lock.json b/worker/package-lock.json new file mode 100644 index 0000000..98aafd5 --- /dev/null +++ b/worker/package-lock.json @@ -0,0 +1,1425 @@ +{ + "name": "@qrun/qqq-isolated-script-worker", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@qrun/qqq-isolated-script-worker", + "version": "0.1.0", + "dependencies": { + "typescript": "6.0.3" + }, + "devDependencies": { + "@eslint/js": "10.0.1", + "@stylistic/eslint-plugin": "5.10.0", + "@types/node": "24.19.2", + "eslint": "10.14.0", + "typescript-eslint": "8.72.1" + }, + "engines": { + "node": "24.22.2" + } + }, + "node_modules/@cacheable/memory": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz", + "integrity": "sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.5.0", + "@keyv/bigmap": "^1.3.1", + "hookified": "^1.15.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.5.0.tgz", + "integrity": "sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.5.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^3.0.5", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.7.0.tgz", + "integrity": "sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/js": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.3.tgz", + "integrity": "sha512-IkO+/KEUvwbVpiURZg+P7zF74z5Jxe0UgJxVni+RtoHQ6IZieXaO02kmadomap/q+l6bc/jdPGGqTjhuZnuz1Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@keyv/bigmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz", + "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.4.0", + "hookified": "^1.15.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "keyv": "^5.6.0" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@stylistic/eslint-plugin": { + "version": "5.10.0", + "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-5.10.0.tgz", + "integrity": "sha512-nPK52ZHvot8Ju/0A4ucSX1dcPV2/1clx0kLcH5wDmrE4naKso7TUC/voUyU1O9OTKTrR6MYip6LP0ogEMQ9jPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/types": "^8.56.0", + "eslint-visitor-keys": "^4.2.1", + "espree": "^10.4.0", + "estraverse": "^5.3.0", + "picomatch": "^4.0.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "peerDependencies": { + "eslint": "^9.0.0 || ^10.0.0" + } + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.10.tgz", + "integrity": "sha512-lTyOMnOcmRBzdCv6/4zUZCrEhV0xrqQiUZBZtVcbmLHBXKLnN8q89v/wKRF/Q0HbwVsqbIG3o5HDoFl9sduM+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.2.tgz", + "integrity": "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@typescript-eslint/eslint-plugin": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.72.1.tgz", + "integrity": "sha512-5CokTWXpmbylfFabsbvHC0CAw3XLaGnoa+nFJX5FR592h/LoZsENUE6p/rg5xvkKrFEXz6pYmgI29AgXc2N3BQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/regexpp": "^4.12.2", + "@typescript-eslint/scope-manager": "8.72.1", + "@typescript-eslint/type-utils": "8.72.1", + "@typescript-eslint/utils": "8.72.1", + "@typescript-eslint/visitor-keys": "8.72.1", + "ignore": "^7.0.5", + "natural-compare": "^1.4.0", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "@typescript-eslint/parser": "^8.72.1", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/eslint-plugin/node_modules/ignore": { + "version": "7.0.14", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-7.0.14.tgz", + "integrity": "sha512-wCBiZ77u1tuWde6G/nfOtLaCTYcFDU5yCzARzK10y+rLhzYoL7a7yEemcopcc2Mzow7gZOIjnV2ym1NYmdiJOQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/@typescript-eslint/parser": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.72.1.tgz", + "integrity": "sha512-ZGwZP4OCfyA0AQTpEAMWz+rtX01DRhQWW5hB0ucKDEIX5U7jdmPSaMx94rQfaPAQzIYz2BimOgKOJtjfpRFQ9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/scope-manager": "8.72.1", + "@typescript-eslint/types": "8.72.1", + "@typescript-eslint/typescript-estree": "8.72.1", + "@typescript-eslint/visitor-keys": "8.72.1", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/project-service": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.72.1.tgz", + "integrity": "sha512-fAuVMkwdO/huI8nyo/WKnNKf6Gag93S56Rh3Ts5mGcavwSdaN8tRg1gCeIGX8PF+nzp1BjGfO4gHhGzW1J6jTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/tsconfig-utils": "^8.72.1", + "@typescript-eslint/types": "^8.72.1", + "debug": "^4.4.3" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/scope-manager": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.72.1.tgz", + "integrity": "sha512-lC/t6eMCZZQAAKZEYGL6RDVSuFehpdmMT6h+nJIjIa6oylN/PwDgDVtEIbB7O9xFfZYtZAYLoLlDNWKJhtF+uQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.72.1", + "@typescript-eslint/visitor-keys": "8.72.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/tsconfig-utils": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.72.1.tgz", + "integrity": "sha512-e4D5R5vpeAEPBxT2BAzsFQL3sbwyeeeo5+aczgNSRK1V734jE7pTqaGFU+fXapWEQwnZbpFaTn32+Z8DwS6OTg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/type-utils": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.72.1.tgz", + "integrity": "sha512-iY2wlmyrXJmx4yxUGP3aYCFF3cA/6ShddOKmLU2dx7LnEykbAM8WThlxZEfd1NddqceF6eJmZUayxx9oKIYNlg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.72.1", + "@typescript-eslint/typescript-estree": "8.72.1", + "@typescript-eslint/utils": "8.72.1", + "debug": "^4.4.3", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/types": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.72.1.tgz", + "integrity": "sha512-LmE5IhQet1BWOWEXqBrVtBPsMAOQ6Nzk35d+WLh3mk4EUF33sqlfxjio1MdYeSvKSG5skactKKuixBIKXhL4Rw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.72.1.tgz", + "integrity": "sha512-DAlUoUV93hIvrtIQLwCbmZQYVZwOMhiHJrOVUUIWicFZaIYjTRqlY3I/NZevfkLmSqkNHRu2qYDcq9lrn16oOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/project-service": "8.72.1", + "@typescript-eslint/tsconfig-utils": "8.72.1", + "@typescript-eslint/types": "8.72.1", + "@typescript-eslint/visitor-keys": "8.72.1", + "debug": "^4.4.3", + "minimatch": "^10.2.2", + "semver": "^7.7.3", + "tinyglobby": "^0.2.15", + "ts-api-utils": "^2.5.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/utils": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.72.1.tgz", + "integrity": "sha512-5v5liypurJBFHCBNqh/RjsymAD+VWcE0djuGY/BUqp5fC3Qn+luWDEksjuTtWjPn6i5zQlUtkZCRG/CYG0yCkA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.9.1", + "@typescript-eslint/scope-manager": "8.72.1", + "@typescript-eslint/types": "8.72.1", + "@typescript-eslint/typescript-estree": "8.72.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/@typescript-eslint/visitor-keys": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.72.1.tgz", + "integrity": "sha512-SB8xqipSzZHUePCCLNeYxqnxq83X61/qzNhjQGbTfjxGZIeOEP7tBgWYX05Tcd6CgE02WdTJnh5b8kMZJ/+bnQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/types": "8.72.1", + "eslint-visitor-keys": "^5.0.0" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + } + }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/cacheable": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.5.0.tgz", + "integrity": "sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/memory": "^2.2.0", + "@cacheable/utils": "^2.5.0", + "hookified": "^1.15.0", + "keyv": "^5.6.0", + "qified": "^0.10.1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "10.14.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.14.0.tgz", + "integrity": "sha512-T0pNsxkb9IKmtQjhpK3d7nWOpFv+YHuMbTJnSVMoQblF0onvXfErrtp4V1jHtsQPan7dKaGVD0q1WCXnzMF31A==", + "dev": true, + "license": "MIT", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.7.0", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.3", + "@humanfs/node": "^0.16.8", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "11.1.5 || >11.1.6 <12", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.6", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-4.2.1.tgz", + "integrity": "sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "10.4.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-10.4.0.tgz", + "integrity": "sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.15.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^4.2.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/file-entry-cache": { + "version": "11.1.5", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-11.1.5.tgz", + "integrity": "sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^6.1.23" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "6.1.23", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.23.tgz", + "integrity": "sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cacheable": "^2.5.0", + "flatted": "^3.4.2", + "hookified": "^1.15.0" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/hashery": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz", + "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^1.15.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/qified": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", + "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^2.1.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/qified/node_modules/hookified": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz", + "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==", + "dev": true, + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/ts-api-utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.5.0.tgz", + "integrity": "sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18.12" + }, + "peerDependencies": { + "typescript": ">=4.8.4" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/typescript-eslint": { + "version": "8.72.1", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.72.1.tgz", + "integrity": "sha512-3Li9FfPmQp45nxEtFD1ZCrYFI3cYK5kpyNtmm1H7lMagm8u9sy6PTjy3ytwQZs7KNukOyKFHAZ6z6GjYFeaeXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@typescript-eslint/eslint-plugin": "8.72.1", + "@typescript-eslint/parser": "8.72.1", + "@typescript-eslint/typescript-estree": "8.72.1", + "@typescript-eslint/utils": "8.72.1" + }, + "engines": { + "node": "^18.18.0 || ^20.9.0 || >=21.1.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/typescript-eslint" + }, + "peerDependencies": { + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", + "typescript": ">=4.8.4 <6.1.0" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/worker/package.json b/worker/package.json new file mode 100644 index 0000000..5301dba --- /dev/null +++ b/worker/package.json @@ -0,0 +1,21 @@ +{ + "name": "@qrun/qqq-isolated-script-worker", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { "node": "24.22.2" }, + "packageManager": "npm@11.19.1", + "scripts": { + "build": "tsc", + "test": "npm run build && node --test test/*.test.mjs", + "lint": "eslint src test --max-warnings 0" + }, + "dependencies": { "typescript": "6.0.3" }, + "devDependencies": { + "@types/node": "24.19.2", + "@eslint/js": "10.0.1", + "eslint": "10.14.0", + "typescript-eslint": "8.72.1", + "@stylistic/eslint-plugin": "5.10.0" + } +} diff --git a/worker/src/compiler-entry.ts b/worker/src/compiler-entry.ts new file mode 100644 index 0000000..9119dc7 --- /dev/null +++ b/worker/src/compiler-entry.ts @@ -0,0 +1,67 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import ts from "typescript"; +import { compile } from "./compiler.js"; +import { encode, frames } from "./frames.js"; +import { TOOLCHAIN } from "./protocol.js"; +import type { JsonObject } from "./protocol.js"; + +const CHILD_PROTOCOL = "qqq-typescript-child/1"; + +/******************************************************************************* + ** Flush one bounded frame; no author diagnostic is written to stderr. + *******************************************************************************/ +async function send(value: JsonObject): Promise +{ + await new Promise((resolve, reject) => + { + process.stdout.write(encode(value), error => + { + if(error) + { + reject(error); + } + else + { + resolve(); + } + }); + }); +} + +/******************************************************************************* + ** Compile-only disposable child. Emitted author JavaScript is never evaluated. + ** Stay alive after RESULT so the parent can sample this actual child CPU before + ** requesting retirement and awaiting its real process exit. + *******************************************************************************/ +async function main(): Promise +{ + if(process.versions.node !== "24.22.2" || ts.version !== "6.0.3") + { + throw new Error("COMPILER_TOOLCHAIN"); + } + const input = frames(process.stdin); + await send({ protocol: CHILD_PROTOCOL, type: "READY", nodeVersion: process.versions.node, typescriptVersion: ts.version, toolchainId: TOOLCHAIN }); + const request = await input.next(); + if(request.done) + { + throw new Error("COMPILE_REQUEST_REQUIRED"); + } + await send(compile(request.value)); + const retirement = await input.next(); + if(retirement.done || Object.keys(retirement.value).sort().join(",") !== "protocol,type" + || retirement.value.protocol !== CHILD_PROTOCOL || retirement.value.type !== "RETIRE") + { + throw new Error("COMPILER_RETIREMENT"); + } + await send({ protocol: CHILD_PROTOCOL, type: "EXITING" }); + process.stdin.destroy(); +} + +void main().catch(() => +{ + process.stderr.write("COMPILER_FAILED\n"); + process.exitCode = 1; + process.stdin.destroy(); +}); diff --git a/worker/src/compiler.ts b/worker/src/compiler.ts new file mode 100644 index 0000000..526d35b --- /dev/null +++ b/worker/src/compiler.ts @@ -0,0 +1,129 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import path from "node:path"; +import ts from "typescript"; +import { hash, PROTOCOL, sourceHash, sourcePath, sources, text, TOOLCHAIN } from "./protocol.js"; +import type { JsonObject } from "./protocol.js"; + +const SDK = `declare module "@qqq/sdk" { + export type Scalar = string | number | boolean | null; + export interface Criterion { field: string; operator: "EQUALS" | "NOT_EQUALS" | "IN" | "LESS_THAN" | "LESS_THAN_OR_EQUALS" | "GREATER_THAN" | "GREATER_THAN_OR_EQUALS"; values: Scalar[] } + export interface Qqq { query(table: string, fields: string[], criteria?: Criterion[], limit?: number): Promise>>> } + export type HookEvent = "PRE_INSERT" | "PRE_UPDATE" | "PRE_DELETE" | "POST_INSERT" | "POST_UPDATE" | "POST_DELETE" | "AFTER_INSERT" | "AFTER_UPDATE" | "AFTER_DELETE"; + export type CommittedEvent = Readonly<{ id: string; tableName: string; recordId: Exclude; dryRun?: boolean }> & (Readonly<{ dataState: "CURRENT"; record: Readonly> }> | Readonly<{ dataState: "DELETED_METADATA"; record?: never }>); + export interface Context { readonly event: HookEvent | "CONTEXT"; readonly payload: Readonly> & { readonly event?: CommittedEvent } } + export type HookResult = { action: "ALLOW" } | { action: "REJECT"; message: string } | { action: "PATCH"; patch: Record }; +}`; + +/******************************************************************************* + ** Compile only virtual project files and shipped standard libraries. User + ** config/plugins/packages/build hooks are never read. Run this inside sandbox. + *******************************************************************************/ +export function compile(request: JsonObject): JsonObject +{ + if(ts.version !== "6.0.3" || process.versions.node !== "24.22.2") + { + throw new Error("TOOLCHAIN_VERSION_MISMATCH"); + } + if(request.protocol !== PROTOCOL || request.type !== "compile" || request.strict !== true || request.toolchainId !== TOOLCHAIN) + { + throw new Error("INVALID_COMPILE_POLICY"); + } + const requestId = text(request.requestId, 128); + const files = sources(request.files); + const entryPoint = sourcePath(request.entryPoint); + if(sourceHash(files) !== request.sourceSha256 || !files.some(file => file.path === entryPoint)) + { + throw new Error("SOURCE_IDENTITY"); + } + const virtual = new Map(files.map(file => ["/src/" + file.path, file.content])); + virtual.set("/sdk/qqq.d.ts", SDK); + const options: ts.CompilerOptions = { + strict: true, noEmitOnError: true, noUncheckedIndexedAccess: true, exactOptionalPropertyTypes: true, + target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.ES2022, moduleResolution: ts.ModuleResolutionKind.Bundler, + types: [], lib: ["lib.es2022.d.ts"], sourceMap: true, inlineSources: true, rootDir: "/src", outDir: "/out", + skipLibCheck: false, allowJs: false, noEmitHelpers: false, + }; + const host = ts.createCompilerHost(options); + const libraryDirectory = path.dirname(ts.getDefaultLibFilePath(options)); + const originalRead = host.readFile.bind(host); + const originalExists = host.fileExists.bind(host); + const allowedLibrary = (name: string): boolean => path.dirname(name) === libraryDirectory && /^lib\.[a-z0-9.]+\.d\.ts$/.test(path.basename(name)); + host.readFile = (name): string | undefined => virtual.get(name) ?? (allowedLibrary(name) ? originalRead(name) : undefined); + host.fileExists = (name): boolean => virtual.has(name) || (allowedLibrary(name) && originalExists(name)); + host.getSourceFile = (name, languageVersion): ts.SourceFile | undefined => + { + const source = host.readFile(name); + return source === undefined ? undefined : ts.createSourceFile(name, source, languageVersion, true); + }; + host.getCurrentDirectory = (): string => "/src"; + host.resolveModuleNameLiterals = (names, containing): ts.ResolvedModuleWithFailedLookupLocations[] => names.map(literal => + { + const name = literal.text; + if(name === "@qqq/sdk") + { + return { resolvedModule: { resolvedFileName: "/sdk/qqq.d.ts", extension: ts.Extension.Dts } }; + } + if(!name.startsWith("./") && !name.startsWith("../")) + { + return { resolvedModule: undefined }; + } + if(!name.endsWith(".js")) + { + return { resolvedModule: undefined }; + } + const resolved = path.posix.normalize(path.posix.join(path.posix.dirname(containing), name)).replace(/\.js$/, ".ts"); + return { resolvedModule: virtual.has(resolved) && resolved.startsWith("/src/") ? { resolvedFileName: resolved, extension: ts.Extension.Ts } : undefined }; + }); + const emitted = new Map(); + host.writeFile = (name, content): void => + { + emitted.set(name.replace(/^\/out\//, ""), content); + }; + const program = ts.createProgram([...virtual.keys()], options, host); + const policyErrors: string[] = []; + for(const source of program.getSourceFiles().filter(file => file.fileName.startsWith("/src/"))) + { + if(source.referencedFiles.length || source.typeReferenceDirectives.length || source.libReferenceDirectives.length) + { + policyErrors.push("Reference directives are not supported"); + } + const visit = (node: ts.Node): void => + { + if(ts.isImportEqualsDeclaration(node) || (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword)) + { + policyErrors.push("Dynamic imports and import-equals are not supported"); + } + if(ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier) && node.moduleSpecifier.text === "@qqq/sdk" && node.importClause?.phaseModifier !== ts.SyntaxKind.TypeKeyword) + { + policyErrors.push("SDK imports must use import type"); + } + ts.forEachChild(node, visit); + }; + visit(source); + } + const diagnostics = ts.getPreEmitDiagnostics(program); + const result: JsonObject = { protocol: PROTOCOL, type: "compileResult", requestId, strict: true, toolchainId: TOOLCHAIN, success: false, diagnostics: [] }; + if(policyErrors.length || diagnostics.length) + { + result.diagnostics = [...policyErrors.slice(0, 50).map(message => ({ file: entryPoint, line: 0, column: 0, code: "POLICY", message })), + ...diagnostics.slice(0, 50).map(item => + { + const position = item.file?.getLineAndCharacterOfPosition(item.start ?? 0); + return { file: item.file?.fileName.replace(/^\/src\//, "").slice(0, 200) ?? "", line: (position?.line ?? -1) + 1, column: (position?.character ?? -1) + 1, code: String(item.code), message: ts.flattenDiagnosticMessageText(item.messageText, " ").slice(0, 500) }; + })]; + return result; + } + if(program.emit().emitSkipped) + { + throw new Error("EMIT_FAILED"); + } + const bundle = JSON.stringify({ format: "qqq-esm/1", entryPoint: entryPoint.replace(/\.ts$/, ".js"), files: [...emitted].filter(([name]) => name.endsWith(".js")).sort().map(([name, content]) => ({ path: name, content })) }); + const sourceMap = JSON.stringify(Object.fromEntries([...emitted].filter(([name]) => name.endsWith(".map")).sort())); + if(bundle.length > 1024 * 1024 || sourceMap.length > 1024 * 1024) + { + throw new Error("BUILD_OUTPUT_LIMIT"); + } + return { ...result, success: true, bundle, bundleSha256: hash(bundle), sourceMap }; +} diff --git a/worker/src/coordinator-cli.ts b/worker/src/coordinator-cli.ts new file mode 100644 index 0000000..da8a2ca --- /dev/null +++ b/worker/src/coordinator-cli.ts @@ -0,0 +1,57 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { frames, encode } from "./frames.js"; +import { runIsolated } from "./launcher.js"; +import type { JsonObject } from "./protocol.js"; + +/******************************************************************************* + ** Trusted host transport bridge for Java WorkerProvider. Java handles each + ** forwarded broker frame synchronously on its authenticated calling thread. + *******************************************************************************/ +async function main(): Promise +{ + if(process.versions.node !== "24.22.2") + { + throw new Error("HOST_NODE_VERSION_MISMATCH"); + } + const input = frames(process.stdin); + const first = await input.next(); + if(first.done) + { + throw new Error("MISSING_REQUEST"); + } + const abort = new AbortController(); + process.once("SIGTERM", () => + { + abort.abort(); + }); + process.once("SIGINT", () => + { + abort.abort(); + }); + const result = await runIsolated(first.value, { + image: process.env.QQQ_WORKER_IMAGE ?? "", + qualificationId: process.env.QQQ_WORKER_QUALIFICATION_ID ?? "", + requiredQualificationId: process.env.QQQ_REQUIRED_QUALIFICATION_ID ?? "", + ownerId: process.env.QQQ_WORKER_OWNER_ID ?? "", + stateDirectory: process.env.QQQ_WORKER_STATE_DIRECTORY ?? "", + }, async (message: JsonObject): Promise => + { + process.stdout.write(encode(message, 64 * 1024)); + const response = await input.next(); + if(response.done) + { + throw new Error("BROKER_CLOSED"); + } + return response.value; + }, abort.signal); + process.stdout.write(encode(result), () => process.exit(0)); +} + +void main().catch(() => +{ + process.stderr.write("ISOLATED_WORKER_UNAVAILABLE\n"); + process.exitCode = 1; + process.stdin.destroy(); +}); diff --git a/worker/src/docker-control.ts b/worker/src/docker-control.ts new file mode 100644 index 0000000..ef06fff --- /dev/null +++ b/worker/src/docker-control.ts @@ -0,0 +1,46 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { object, text } from "./protocol.js"; +import type { ContainerControl, ContainerIdentity } from "./lifecycle.js"; + +const command = promisify(execFile); + +export const dockerControl: ContainerControl = { + async inspect(name: string): Promise + { + let output: string; + try + { + output = (await command("docker", ["container", "inspect", "--format", "{{json .}}", name], { timeout: 10000, maxBuffer: 65536, windowsHide: true })).stdout; + } + catch(error) + { + // Only the daemon's explicit missing-container response proves absence. + const stderr: unknown = (error as { stderr?: unknown }).stderr; + if(typeof stderr === "string" && stderr.trim() === "Error: No such container: " + name) + { + return undefined; + } + throw new Error("CONTAINER_INSPECTION_FAILED", { cause: error }); + } + const data = object(JSON.parse(output) as unknown); + const labels = object(object(data.Config).Labels); + const checked: Record = {}; + for(const [key, value] of Object.entries(labels)) + { + checked[key] = text(value, 256); + } + return { id: text(data.Id, 64), name: text(data.Name, 100), labels: checked }; + }, + async remove(id: string): Promise + { + if(!/^[a-f0-9]{64}$/.test(id)) + { + throw new Error("INVALID_CONTAINER_ID"); + } + await command("docker", ["rm", "--force", id], { timeout: 10000, maxBuffer: 4096, windowsHide: true }); + }, +}; diff --git a/worker/src/entrypoint.ts b/worker/src/entrypoint.ts new file mode 100644 index 0000000..c243e03 --- /dev/null +++ b/worker/src/entrypoint.ts @@ -0,0 +1,49 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { execute } from "./execute.js"; +import { encode, frames } from "./frames.js"; +import type { JsonObject } from "./protocol.js"; + +/******************************************************************************* + ** Container-only protocol entry. The marker is a misconfiguration guard, not + ** evidence of isolation; only the host launcher establishes the OS boundary. + *******************************************************************************/ +async function main(): Promise +{ + if(process.versions.node !== "24.22.2") + { + throw new Error("WORKER_NODE_VERSION_MISMATCH"); + } + if(process.env.QQQ_WORKER_MODE !== "container") + { + throw new Error("CONTAINER_REQUIRED"); + } + const input = frames(process.stdin); + const first = await input.next(); + if(first.done) + { + throw new Error("MISSING_REQUEST"); + } + const request = first.value; + const broker = async (message: JsonObject): Promise => + { + process.stdout.write(encode(message, 64 * 1024)); + const response = await input.next(); + if(response.done) + { + throw new Error("BROKER_CLOSED"); + } + return response.value; + }; + const result = request.type === "compile" ? (await import("./compiler.js")).compile(request) : await execute(request, broker); + const limit = request.type === "compile" ? 4 * 1024 * 1024 : 128 * 1024; + process.stdout.write(encode(result, limit), () => process.exit(0)); +} + +void main().catch(() => +{ + process.stderr.write("WORKER_FAILED\n"); + process.exitCode = 1; + process.stdin.destroy(); +}); diff --git a/worker/src/execute.ts b/worker/src/execute.ts new file mode 100644 index 0000000..c4aefbf --- /dev/null +++ b/worker/src/execute.ts @@ -0,0 +1,109 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { hash, object, parse, PROTOCOL, sourcePath, text } from "./protocol.js"; +import type { JsonObject } from "./protocol.js"; + +export type Broker = (request: JsonObject) => Promise; + +/******************************************************************************* + ** Execute only within the disposable OS sandbox. Native module import is never + ** used by the host coordinator. Script code can control its worker process; + ** its output, usage and broker frames remain untrusted at the parent boundary. + *******************************************************************************/ +export async function execute(request: JsonObject, broker: Broker): Promise +{ + if(request.protocol !== PROTOCOL || request.type !== "execute") + { + throw new Error("INVALID_EXECUTE"); + } + const requestId = text(request.requestId, 128); + const bundleText = text(request.bundle, 1024 * 1024); + if(hash(bundleText) !== request.artifactSha256) + { + throw new Error("ARTIFACT_IDENTITY"); + } + const bundle = parse(bundleText, 2 * 1024 * 1024); + const entryPoint = sourcePath(request.entryPoint).replace(/\.ts$/, ".js"); + if(bundle.format !== "qqq-esm/1" || bundle.entryPoint !== entryPoint || !Array.isArray(bundle.files) || bundle.files.length > 200) + { + throw new Error("INVALID_BUNDLE"); + } + const limits = object(request.limits); + const callLimit = Number(limits.brokerCalls); + if(!Number.isInteger(callLimit) || callLimit < 0 || callLimit > 50) + { + throw new Error("INVALID_LIMIT"); + } + const token = text(request.capabilityToken, 200); + let calls = 0; + let pending = false; + const qqq = Object.freeze({ + query: async (table: string, fields: string[], criteria: unknown[] = [], limit = 100): Promise => + { + if(pending || ++calls > callLimit) + { + throw new Error("BROKER_LIMIT"); + } + pending = true; + try + { + const response = await broker({ protocol: PROTOCOL, type: "broker", requestId, operation: "query", token, table, fields, criteria, limit }); + if(response.protocol !== PROTOCOL || response.type !== "brokerResult" || response.ok !== true) + { + throw new Error("BROKER_DENIED"); + } + return response.rows; + } + finally + { + pending = false; + } + }, + }); + const directory = await mkdtemp(path.join(tmpdir(), "qqq-script-")); + const started = performance.now(); + const cpu = process.cpuUsage(); + try + { + await writeFile(path.join(directory, "package.json"), "{\"type\":\"module\"}", { flag: "wx", mode: 0o600 }); + const seen = new Set(); + for(const item of bundle.files) + { + const file = object(item); + const name = sourcePath(file.path, ".js"); + if(seen.has(name.toLowerCase())) + { + throw new Error("DUPLICATE_MODULE"); + } + seen.add(name.toLowerCase()); + const target = path.join(directory, ...name.split("/")); + await mkdir(path.dirname(target), { recursive: true, mode: 0o700 }); + await writeFile(target, text(file.content, 1024 * 1024), { flag: "wx", mode: 0o600 }); + } + if(!seen.has(entryPoint.toLowerCase())) + { + throw new Error("MISSING_ENTRY"); + } + const module = object(await import(pathToFileURL(path.join(directory, entryPoint)).href) as unknown); + if(typeof module.main !== "function") + { + throw new Error("MISSING_MAIN"); + } + const result: unknown = await (module.main as (context: JsonObject, api: typeof qqq) => Promise)({ event: request.event, payload: request.payload }, qqq); + const usage = process.cpuUsage(cpu); + return { protocol: PROTOCOL, type: "executeResult", requestId, outcome: "OK", result: object(result), usage: { cpuMillis: Math.ceil((usage.user + usage.system) / 1000), wallMillis: Math.ceil(performance.now() - started) } }; + } + catch + { + return { protocol: PROTOCOL, type: "executeResult", requestId, outcome: "SCRIPT_ERROR", error: { code: "SCRIPT_FAILED", message: "Script execution failed" } }; + } + finally + { + await rm(directory, { recursive: true, force: true }); + } +} diff --git a/worker/src/frames.ts b/worker/src/frames.ts new file mode 100644 index 0000000..773d9a6 --- /dev/null +++ b/worker/src/frames.ts @@ -0,0 +1,57 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import type { Readable } from "node:stream"; +import { MAX_FRAME, parse } from "./protocol.js"; +import type { JsonObject } from "./protocol.js"; + +/******************************************************************************* + ** Newline framing with byte bounds before parsing, including incomplete lines. + *******************************************************************************/ +export async function* frames(stream: Readable, maximum = MAX_FRAME): AsyncGenerator +{ + let pending = Buffer.alloc(0); + for await (const value of stream) + { + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(String(value)); + let start = 0; + for(let end = 0; end < chunk.length; end++) + { + if(chunk[end] !== 10) + { + continue; + } + const piece = chunk.subarray(start, end); + if(pending.length + piece.length > maximum) + { + throw new Error("FRAME_LIMIT"); + } + yield parse(Buffer.concat([pending, piece]).toString("utf8"), maximum); + pending = Buffer.alloc(0); + start = end + 1; + } + const remainder = chunk.subarray(start); + if(pending.length + remainder.length > maximum) + { + throw new Error("FRAME_LIMIT"); + } + pending = Buffer.concat([pending, remainder]); + } + if(pending.length !== 0) + { + throw new Error("INCOMPLETE_FRAME"); + } +} + +/******************************************************************************* + ** Bound outgoing frames before writing any bytes. + *******************************************************************************/ +export function encode(value: JsonObject, maximum = MAX_FRAME): string +{ + const json = JSON.stringify(value); + if(Buffer.byteLength(json) > maximum) + { + throw new Error("FRAME_LIMIT"); + } + return json + "\n"; +} diff --git a/worker/src/launcher.ts b/worker/src/launcher.ts new file mode 100644 index 0000000..7d838da --- /dev/null +++ b/worker/src/launcher.ts @@ -0,0 +1,251 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { execFile, spawn } from "node:child_process"; +import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import { promisify } from "node:util"; +import path from "node:path"; +import { encode, frames } from "./frames.js"; +import { object, PROTOCOL, text } from "./protocol.js"; +import type { JsonObject } from "./protocol.js"; +import type { Broker } from "./execute.js"; +import { ATTEMPT_LABEL, AttemptJournal, cleanAttempt, OWNER_LABEL } from "./lifecycle.js"; +import { dockerControl } from "./docker-control.js"; + +const command = promisify(execFile); +export interface SandboxConfig +{ + image: string; + qualificationId: string; + requiredQualificationId: string; + ownerId: string; + stateDirectory: string; +} +interface Limits { wallMillis: number; cpuMillis: number; memoryBytes: number; outputBytes: number; brokerCalls: number } + +/******************************************************************************* + ** Validate operator evidence and Docker capabilities. No runc/process fallback. + ** The evidence id must represent external qualification, not self-attestation. + *******************************************************************************/ +export function qualify(config: SandboxConfig, info: JsonObject): void +{ + if(!/^[a-z0-9./:_-]+@sha256:[a-f0-9]{64}$/.test(config.image) + || !/^[A-Za-z0-9._:-]{1,128}$/.test(config.qualificationId) + || config.qualificationId !== config.requiredQualificationId + || !/^[A-Za-z0-9._-]{1,64}$/.test(config.ownerId) || !path.isAbsolute(config.stateDirectory) + || info.OSType !== "linux" || !Object.hasOwn(object(info.Runtimes), "runsc")) + { + throw new Error("SANDBOX_UNQUALIFIED"); + } +} + +/******************************************************************************* + ** Clamp every resource requested by a host. Unsupported subsecond CPU budgets + ** are rejected because RLIMIT_CPU is expressed in whole seconds. + *******************************************************************************/ +function limitsFor(request: JsonObject): Limits +{ + if(request.type === "compile") + { + return { wallMillis: 10000, cpuMillis: 5000, memoryBytes: 256 * 1024 * 1024, outputBytes: 4 * 1024 * 1024, brokerCalls: 0 }; + } + const values = object(request.limits); + for(const key of ["wallMillis", "cpuMillis", "memoryBytes", "outputBytes", "brokerCalls"]) + { + if(typeof values[key] !== "number") + { + throw new Error("UNSUPPORTED_LIMITS"); + } + } + const limits = { wallMillis: Number(values.wallMillis), cpuMillis: Number(values.cpuMillis), memoryBytes: Number(values.memoryBytes), outputBytes: Number(values.outputBytes), brokerCalls: Number(values.brokerCalls) }; + if(Object.values(limits).some(value => !Number.isSafeInteger(value)) + || limits.wallMillis < 1 || limits.wallMillis > 10000 || limits.cpuMillis < 1000 || limits.cpuMillis > 5000 || limits.cpuMillis % 1000 !== 0 + || limits.memoryBytes < 64 * 1024 * 1024 || limits.memoryBytes > 256 * 1024 * 1024 + || limits.outputBytes < 1 || limits.outputBytes > 128 * 1024 || limits.brokerCalls < 0 || limits.brokerCalls > 50) + { + throw new Error("UNSUPPORTED_LIMITS"); + } + return limits; +} + +/******************************************************************************* + ** Fixed argument array: no shell, ports, credentials, volumes or caller flags. + *******************************************************************************/ +export function createArguments(request: JsonObject, config: SandboxConfig, name: string): string[] +{ + const limits = limitsFor(request); + return ["create", "--name", name, "--label", OWNER_LABEL + "=" + config.ownerId, + "--label", ATTEMPT_LABEL + "=" + name.replace(/^qqq-script-/, ""), + "--pull=never", "--runtime=runsc", "--network=none", "--read-only", "--user=65532:65532", + "--cap-drop=ALL", "--security-opt=no-new-privileges:true", "--ipc=none", "--pids-limit=32", + "--memory=" + String(limits.memoryBytes), "--memory-swap=" + String(limits.memoryBytes), + "--cpus=" + Math.min(1, limits.cpuMillis / limits.wallMillis).toFixed(3), + "--ulimit", "cpu=" + String(limits.cpuMillis / 1000) + ":" + String(limits.cpuMillis / 1000), + "--ulimit", "nofile=64:64", "--tmpfs=/tmp:rw,noexec,nosuid,nodev,size=67108864,mode=1777", + "--env=QQQ_WORKER_MODE=container", "--log-driver=none", "-i", config.image]; +} + +/******************************************************************************* + ** Start one sandbox for one invocation. External wall/output/call limits remain + ** enforceable even when user code takes over the entire worker Node process. + ** Removal failure is an operational failure, not silently ignored success. + *******************************************************************************/ +export async function runIsolated(request: JsonObject, config: SandboxConfig, broker: Broker, signal?: AbortSignal): Promise +{ + if(request.protocol !== PROTOCOL || !["compile", "execute"].includes(String(request.type))) + { + throw new Error("INVALID_REQUEST"); + } + const limits = limitsFor(request); + encode(request); + if(signal?.aborted) + { + throw new Error("CANCELLED"); + } + const info = await command("docker", ["info", "--format", "{{json .}}"], { timeout: 10000, maxBuffer: 1024 * 1024, windowsHide: true }); + qualify(config, object(JSON.parse(info.stdout) as unknown)); + const journal = new AttemptJournal(config.stateDirectory); + const attempt = await journal.prepare(config.ownerId, Date.now(), text(request.requestId, 128)); + const name = attempt.name; + let completed: JsonObject | undefined; + let invocationFailure: Error | undefined; + try + { + const created = await command("docker", createArguments(request, config, name), { timeout: 10000, maxBuffer: 4096, windowsHide: true }); + const containerId = created.stdout.trim(); + if(!/^[a-f0-9]{64}$/.test(containerId)) + { + throw new Error("INVALID_CREATE_RESPONSE"); + } + await journal.record(attempt, "CREATED", { containerId }); + if(signal?.aborted) + { + throw new Error("CANCELLED"); + } + const child = spawn("docker", ["start", "--attach", "--interactive", name], { stdio: ["pipe", "pipe", "pipe"], shell: false, windowsHide: true }); + completed = await relay(child, request, limits, broker, signal); + } + catch(error) + { + invocationFailure = error instanceof Error ? error : new Error("WORKER_FAILED"); + } + // Always remove the sandbox before returning a result or surfacing a failure. + try + { + await cleanAttempt(journal, attempt, dockerControl); + } + catch + { + throw new Error("SANDBOX_CLEANUP_FAILED"); + } + await journal.record(attempt, invocationFailure === undefined ? "INVOCATION_FINISHED" : "INVOCATION_FAILED"); + if(invocationFailure !== undefined) + { + throw invocationFailure; + } + if(!completed) + { + throw new Error("WORKER_FAILED"); + } + return completed; +} + +/******************************************************************************* + ** Bound an attached worker transport outside the worker's trust boundary. + ** Factored for deterministic synthetic-process termination tests. + *******************************************************************************/ +export async function relay(child: ChildProcessWithoutNullStreams, request: JsonObject, limits: Limits, broker: Broker, signal?: AbortSignal): Promise +{ + const initial = encode(request); + let failure: Error | undefined; + let totalOutput = 0; + let calls = 0; + let final: JsonObject | undefined; + const kill = (reason: string): void => + { + failure ??= new Error(reason); + child.kill(); + }; + const onAbort = (): void => + { + kill("CANCELLED"); + }; + const timer = setTimeout(() => + { + kill("WALL_LIMIT"); + }, limits.wallMillis); + signal?.addEventListener("abort", onAbort, { once: true }); + const exited = new Promise((resolve, reject) => + { + child.once("error", reject); + child.once("close", resolve); + }); + // Attach a rejection handler immediately, before reading asynchronous frames. + void exited.catch(() => undefined); + const count = (chunk: Buffer): void => + { + totalOutput += chunk.length; + if(totalOutput > limits.outputBytes) + { + kill("OUTPUT_LIMIT"); + } + }; + child.stdout.on("data", count); + child.stderr.on("data", count); + child.stdin.on("error", () => + { + kill("INPUT_CLOSED"); + }); + child.stdin.write(initial); + try + { + for await (const message of frames(child.stdout, limits.outputBytes)) + { + if(failure) + { + throw failure; + } + if(message.protocol !== PROTOCOL || message.requestId !== request.requestId || final) + { + throw new Error("INVALID_RESPONSE"); + } + if(message.type === "broker") + { + if(++calls > limits.brokerCalls || message.token !== request.capabilityToken || message.operation !== "query") + { + throw new Error("BROKER_LIMIT"); + } + const answer = await Promise.race([broker(message), exited.then(() => + { + throw new Error("WORKER_EXITED"); + })]); + child.stdin.write(encode(answer, 64 * 1024)); + } + else if(message.type === (request.type === "compile" ? "compileResult" : "executeResult")) + { + final = message; + child.stdin.end(); + } + else + { + throw new Error("INVALID_RESPONSE"); + } + } + const status = await exited; + if(failure) + { + throw failure; + } + if(status !== 0 || !final) + { + throw new Error("WORKER_FAILED"); + } + return final; + } + finally + { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + child.kill(); + } +} diff --git a/worker/src/lifecycle.ts b/worker/src/lifecycle.ts new file mode 100644 index 0000000..c444376 --- /dev/null +++ b/worker/src/lifecycle.ts @@ -0,0 +1,235 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { mkdir, open, rename, unlink, opendir, lstat } from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { object, text } from "./protocol.js"; +import type { JsonObject } from "./protocol.js"; + +export interface Attempt +{ + id: string; + owner: string; + name: string; + createdAt: number; + expiresAt: number; + requestId: string; +} +export interface ContainerIdentity { id: string; name: string; labels: Record } +export interface ContainerControl +{ + inspect(name: string): Promise; + remove(id: string): Promise; +} +export const OWNER_LABEL = "io.qrun.scripting.owner"; +export const ATTEMPT_LABEL = "io.qrun.scripting.attempt"; +const ATTEMPT_PATTERN = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/; + +function validateOwner(owner: string): void +{ + if(!/^[A-Za-z0-9._-]{1,64}$/.test(owner)) + { + throw new Error("INVALID_OWNER"); + } +} + +async function syncDirectory(directory: string): Promise +{ + const handle = await open(directory, "r"); + try + { + await handle.sync(); + } + finally + { + await handle.close(); + } +} + +/******************************************************************************* + ** Deployment-owned local durable storage, never a path supplied by scripts. + ** Production requires directory fsync support; failures stop launch/cleanup. + *******************************************************************************/ +export class AttemptJournal +{ + public constructor(private readonly directory: string, private readonly sync: (directory: string) => Promise = syncDirectory) + { + if(!path.isAbsolute(directory)) + { + throw new Error("ABSOLUTE_STATE_DIRECTORY_REQUIRED"); + } + } + + private async initialize(): Promise + { + if((await lstat(this.directory)).isSymbolicLink()) + { + throw new Error("STATE_DIRECTORY_SYMLINK"); + } + await mkdir(path.join(this.directory, "active"), { mode: 0o700, recursive: true }); + await mkdir(path.join(this.directory, "evidence"), { mode: 0o700, recursive: true }); + await this.sync(this.directory); + } + + private async save(directory: string, filename: string, value: unknown): Promise + { + const temporary = path.join(directory, ".pending-" + randomUUID()); + const handle = await open(temporary, "wx", 0o600); + try + { + await handle.writeFile(JSON.stringify(value) + "\n", "utf8"); + await handle.sync(); + } + finally + { + await handle.close(); + } + await rename(temporary, path.join(directory, filename)); + await this.sync(directory); + } + + public async prepare(owner: string, now = Date.now(), requestId = "unspecified"): Promise + { + validateOwner(owner); + text(requestId, 128); + await this.initialize(); + const id = randomUUID(); + const attempt = { id, owner, name: "qqq-script-" + id, createdAt: now, expiresAt: now + 60000, requestId }; + // Evidence precedes the active lease; neither includes user source or tokens. + await this.record(attempt, "PREPARED"); + await this.save(path.join(this.directory, "active"), id + ".json", attempt); + return attempt; + } + + public async record(attempt: Attempt, event: string, details: JsonObject = {}): Promise + { + await this.save(path.join(this.directory, "evidence"), attempt.id + "-" + randomUUID() + ".json", { ...attempt, event, observedAt: Date.now(), details }); + } + + public async finish(attempt: Attempt, event: "REMOVED" | "ALREADY_ABSENT", containerId?: string): Promise + { + await this.record(attempt, event, containerId === undefined ? {} : { containerId }); + try + { + await unlink(path.join(this.directory, "active", attempt.id + ".json")); + await this.sync(path.join(this.directory, "active")); + } + catch(error) + { + if((error as NodeJS.ErrnoException).code !== "ENOENT") + { + throw error; + } + } + } + + public async expired(owner: string, now = Date.now()): Promise + { + validateOwner(owner); + await this.initialize(); + const result: Attempt[] = []; + const entries = await opendir(path.join(this.directory, "active")); + for await (const entry of entries) + { + if(!entry.isFile() || !entry.name.endsWith(".json")) + { + continue; + } + let raw: string; + try + { + const handle = await open(path.join(this.directory, "active", entry.name), "r"); + try + { + const bytes = Buffer.alloc(4097); + const read = await handle.read(bytes); + if(read.bytesRead > 4096) + { + throw new Error("INVALID_ATTEMPT_RECORD"); + } + raw = bytes.subarray(0, read.bytesRead).toString("utf8"); + } + finally + { + await handle.close(); + } + } + catch(error) + { + if((error as NodeJS.ErrnoException).code === "ENOENT") + { + continue; + } + throw error; + } + const value = object(JSON.parse(raw) as unknown); + const id = text(value.id, 36); + const requestId = text(value.requestId, 128); + if(!ATTEMPT_PATTERN.test(id) || entry.name !== id + ".json" || value.name !== "qqq-script-" + id + || typeof value.createdAt !== "number" || typeof value.expiresAt !== "number" + || !Number.isSafeInteger(value.createdAt) || value.expiresAt !== value.createdAt + 60000) + { + throw new Error("INVALID_ATTEMPT_RECORD"); + } + if(value.owner === owner && value.expiresAt <= now) + { + result.push({ id, owner, name: "qqq-script-" + id, createdAt: value.createdAt, expiresAt: value.expiresAt, requestId }); + if(result.length === 1000) + { + break; + } + } + } + return result; + } +} + +/******************************************************************************* + ** Inspect the exact prepared name and remove only its verified immutable ID. + ** Supports a crash after create but before its result was recorded locally. + *******************************************************************************/ +export async function cleanAttempt(journal: AttemptJournal, attempt: Attempt, containers: ContainerControl): Promise +{ + try + { + const container = await containers.inspect(attempt.name); + if(container === undefined) + { + await journal.finish(attempt, "ALREADY_ABSENT"); + return; + } + if(!/^[a-f0-9]{64}$/.test(container.id) || container.name !== "/" + attempt.name + || container.labels[OWNER_LABEL] !== attempt.owner || container.labels[ATTEMPT_LABEL] !== attempt.id) + { + throw new Error("CONTAINER_OWNERSHIP_MISMATCH"); + } + await journal.record(attempt, "REMOVAL_REQUESTED", { containerId: container.id }); + await containers.remove(container.id); + await journal.finish(attempt, "REMOVED", container.id); + } + catch(error) + { + await journal.record(attempt, "CLEANUP_FAILED"); + throw error; + } +} + +export async function reapExpired(journal: AttemptJournal, owner: string, containers: ContainerControl, now = Date.now()): Promise<{ cleaned: number; failed: number }> +{ + let cleaned = 0; + let failed = 0; + for(const attempt of await journal.expired(owner, now)) + { + try + { + await cleanAttempt(journal, attempt, containers); + cleaned++; + } + catch + { + failed++; + } + } + return { cleaned, failed }; +} diff --git a/worker/src/protocol.ts b/worker/src/protocol.ts new file mode 100644 index 0000000..5cd1b4a --- /dev/null +++ b/worker/src/protocol.ts @@ -0,0 +1,148 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { createHash } from "node:crypto"; + +export const PROTOCOL = "qbit-scripting-worker/1"; +export const TOOLCHAIN = "typescript-6.0.3-node-24"; +export const MAX_FRAME = 4 * 1024 * 1024; +export type JsonObject = Record; +export interface SourceFile { path: string; content: string } +export interface Bundle { format: string; entryPoint: string; files: SourceFile[] } + +/******************************************************************************* + ** Reject malformed protocol values without echoing potentially sensitive data. + *******************************************************************************/ +export function object(value: unknown): JsonObject +{ + if(value === null || typeof value !== "object" || Array.isArray(value)) + { + throw new Error("INVALID_OBJECT"); + } + return value as JsonObject; +} + +/******************************************************************************* + ** Bound text before it reaches path handling, compiler or protocol output. + *******************************************************************************/ +export function text(value: unknown, max: number): string +{ + if(typeof value !== "string" || value.length > max) + { + throw new Error("INVALID_TEXT"); + } + return value; +} + +/******************************************************************************* + ** Parse size/depth-bounded plain JSON; no revivers or prototype merging. + *******************************************************************************/ +export function parse(value: string, max = MAX_FRAME): JsonObject +{ + if(Buffer.byteLength(value) > max) + { + throw new Error("FRAME_LIMIT"); + } + let depth = 0; + let quoted = false; + let escaped = false; + for(const char of value) + { + if(quoted) + { + if(escaped) + { + escaped = false; + } + else if(char === "\\") + { + escaped = true; + } + else if(char === "\"") + { + quoted = false; + } + } + else if(char === "\"") + { + quoted = true; + } + else if(char === "{" || char === "[") + { + if(++depth > 12) + { + throw new Error("DEPTH_LIMIT"); + } + } + else if(char === "}" || char === "]") + { + depth--; + } + } + return object(JSON.parse(value) as unknown); +} + +/******************************************************************************* + ** Source names are portable relative paths, never filesystem capabilities. + *******************************************************************************/ +export function sourcePath(value: unknown, suffix = ".ts"): string +{ + const name = text(value, 200); + if(!name.endsWith(suffix) || name.includes("..") || !name.split("/").every(part => /^[A-Za-z0-9_][A-Za-z0-9_.-]*$/.test(part))) + { + throw new Error("INVALID_PATH"); + } + return name; +} + +/******************************************************************************* + ** SHA-256 is an integrity identifier; host publication supplies authorization. + *******************************************************************************/ +export function hash(value: string): string +{ + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +/******************************************************************************* + ** Match Java SourceSnapshot's length-prefixed UTF-8 canonical encoding. + *******************************************************************************/ +export function sourceHash(files: SourceFile[]): string +{ + const digest = createHash("sha256"); + for(const file of [...files].sort((left, right) => left.path < right.path ? -1 : left.path > right.path ? 1 : 0)) + { + const path = Buffer.from(file.path); + const content = Buffer.from(file.content); + const lengths = Buffer.alloc(8); + lengths.writeInt32BE(path.length, 0); + lengths.writeInt32BE(content.length, 4); + digest.update(lengths).update(path).update(content); + } + return digest.digest("hex"); +} + +/******************************************************************************* + ** Validate source manifests, including case collisions and aggregate bytes. + *******************************************************************************/ +export function sources(value: unknown): SourceFile[] +{ + if(!Array.isArray(value) || value.length < 1 || value.length > 200) + { + throw new Error("FILE_LIMIT"); + } + const seen = new Set(); + let bytes = 0; + return value.map((entry: unknown) => + { + const item = object(entry); + const path = sourcePath(item.path); + const content = text(item.content, 262144); + bytes += Buffer.byteLength(content); + if(bytes > 2 * 1024 * 1024 || seen.has(path.toLowerCase())) + { + throw new Error("SOURCE_LIMIT"); + } + seen.add(path.toLowerCase()); + return { path, content }; + }); +} diff --git a/worker/src/qualification-cli.ts b/worker/src/qualification-cli.ts new file mode 100644 index 0000000..3a4132b --- /dev/null +++ b/worker/src/qualification-cli.ts @@ -0,0 +1,95 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { open } from "node:fs/promises"; +import { runIsolated } from "./launcher.js"; +import { PROTOCOL, sourceHash, text, TOOLCHAIN } from "./protocol.js"; +import type { JsonObject } from "./protocol.js"; + +/******************************************************************************* + ** Operator-invoked disposable-lab checks only. Never run by npm test or build. + ** This is useful evidence, not a substitute for a complete sandbox assessment. + *******************************************************************************/ +async function main(): Promise +{ + if(process.argv[2] !== "--run-qualified-fixtures" || !process.argv[3]) + { + throw new Error("EXPLICIT_LAB_INVOCATION_REQUIRED"); + } + const config = { + image: process.env.QQQ_WORKER_IMAGE ?? "", + qualificationId: process.env.QQQ_WORKER_QUALIFICATION_ID ?? "", + requiredQualificationId: process.env.QQQ_REQUIRED_QUALIFICATION_ID ?? "", + ownerId: process.env.QQQ_WORKER_OWNER_ID ?? "", + stateDirectory: process.env.QQQ_WORKER_STATE_DIRECTORY ?? "", + }; + const evidence = await open(process.argv[3], "wx", 0o600); + const record = async (value: JsonObject): Promise => + { + await evidence.writeFile(JSON.stringify({ at: new Date().toISOString(), ...value }) + "\n"); + await evidence.sync(); + }; + const broker = (request: JsonObject): Promise => + { + if(request.table !== "fixture" || request.token !== "qualification-fixture") + { + throw new Error("UNEXPECTED_BROKER_REQUEST"); + } + return Promise.resolve({ protocol: PROTOCOL, type: "brokerResult", ok: true, rows: [{ id: 17 }], truncated: false }); + }; + try + { + await record({ event: "START", image: config.image, qualificationId: config.qualificationId, owner: config.ownerId }); + const source = 'import type {Context,Qqq} from "@qqq/sdk"; export async function main(context:Context, qqq:Qqq) { if(context.event === "TIMEOUT") { while(true) {} } return {rows:await qqq.query("fixture",["id"],[],1)}; }'; + const files = [{ path: "main.ts", content: source }]; + const compiled = await runIsolated({ protocol: PROTOCOL, type: "compile", requestId: "qualification-compile", strict: true, toolchainId: TOOLCHAIN, entryPoint: "main.ts", files, sourceSha256: sourceHash(files) }, config, broker); + if(compiled.success !== true) + { + throw new Error("FIXTURE_COMPILE_FAILED"); + } + await record({ event: "COMPILE_PASSED", bundleSha256: compiled.bundleSha256 }); + const execute: JsonObject = { protocol: PROTOCOL, type: "execute", requestId: "qualification-execute", artifactSha256: compiled.bundleSha256, bundle: text(compiled.bundle, 1024 * 1024), entryPoint: "main.ts", event: "CONTEXT", payload: { context: {} }, capabilityToken: "qualification-fixture", limits: { wallMillis: 5000, cpuMillis: 1000, memoryBytes: 134217728, outputBytes: 131072, brokerCalls: 1 } }; + const result = await runIsolated(execute, config, broker); + if(result.outcome !== "OK" || JSON.stringify(result.result) !== '{"rows":[{"id":17}]}') + { + throw new Error("FIXTURE_RESULT_FAILED"); + } + await record({ event: "BROKER_CONTEXT_PASSED" }); + let terminated = false; + try + { + await runIsolated({ ...execute, requestId: "qualification-timeout", event: "TIMEOUT" }, config, broker); + } + catch(error) + { + if(error instanceof Error && ["WALL_LIMIT", "WORKER_FAILED"].includes(error.message)) + { + terminated = true; + } + else + { + throw error; + } + } + if(!terminated) + { + throw new Error("TIMEOUT_NOT_ENFORCED"); + } + await record({ event: "TERMINATION_AND_CLEANUP_PASSED", completeQualification: false }); + } + catch(error) + { + await record({ event: "FAILED", message: error instanceof Error ? error.message.slice(0, 128) : "QUALIFICATION_FAILED" }); + throw error; + } + finally + { + await evidence.close(); + } +} + +void main().catch(() => +{ + process.stderr.write("QUALIFICATION_INCOMPLETE\n"); + process.exitCode = 1; +}); diff --git a/worker/src/reaper-cli.ts b/worker/src/reaper-cli.ts new file mode 100644 index 0000000..0e9f108 --- /dev/null +++ b/worker/src/reaper-cli.ts @@ -0,0 +1,19 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import { AttemptJournal, reapExpired } from "./lifecycle.js"; +import { dockerControl } from "./docker-control.js"; + +async function main(): Promise +{ + const journal = new AttemptJournal(process.env.QQQ_WORKER_STATE_DIRECTORY ?? ""); + const result = await reapExpired(journal, process.env.QQQ_WORKER_OWNER_ID ?? "", dockerControl); + process.stdout.write(JSON.stringify(result) + "\n"); + process.exitCode = result.failed === 0 ? 0 : 1; +} + +void main().catch(() => +{ + process.stderr.write("REAPER_FAILED\n"); + process.exitCode = 1; +}); diff --git a/worker/test/lifecycle.test.mjs b/worker/test/lifecycle.test.mjs new file mode 100644 index 0000000..16b59e7 --- /dev/null +++ b/worker/test/lifecycle.test.mjs @@ -0,0 +1,106 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import assert from "node:assert/strict"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { AttemptJournal, ATTEMPT_LABEL, OWNER_LABEL, cleanAttempt, reapExpired } from "../dist/lifecycle.js"; + +async function fixture(t) +{ + const directory = await mkdtemp(path.join(tmpdir(), "qqq-journal-test-")); + t.after(() => rm(directory, { recursive: true, force: true })); + // Windows test host lacks POSIX directory fsync; inject only that operation. + // File writes still use fsync. Production's default never skips directory fsync. + const journal = new AttemptJournal(directory, async () => undefined); + return { directory, journal }; +} + +function identity(attempt) +{ + return { id: "a".repeat(64), name: "/" + attempt.name, labels: { [OWNER_LABEL]: attempt.owner, [ATTEMPT_LABEL]: attempt.id } }; +} + +test("restart recovers a create whose ID response was lost and removes exact owned ID", async t => +{ + const { directory, journal } = await fixture(t); + const attempt = await journal.prepare("deployment-a", 1000); + const restarted = new AttemptJournal(directory, async () => undefined); + const removed = []; + const control = { + inspect: async name => + { + assert.equal(name, attempt.name); + return identity(attempt); + }, + remove: async id => removed.push(id), + }; + assert.deepEqual(await reapExpired(restarted, "deployment-a", control, 61000), { cleaned: 1, failed: 0 }); + assert.deepEqual(removed, ["a".repeat(64)]); + assert.deepEqual(await readdir(path.join(directory, "active")), []); + const events = await Promise.all((await readdir(path.join(directory, "evidence"))).map(async filename => JSON.parse(await readFile(path.join(directory, "evidence", filename), "utf8")))); + assert.deepEqual(new Set(events.map(event => event.event)), new Set(["PREPARED", "REMOVAL_REQUESTED", "REMOVED"])); + assert.ok(events.every(event => event.id === attempt.id && event.owner === "deployment-a")); +}); + +test("reaper never inspects live or another deployment's leases", async t => +{ + const { journal } = await fixture(t); + await journal.prepare("other", 0); + await journal.prepare("deployment-a", 60000); + const unexpected = async () => + { + throw new Error("must not inspect/remove"); + }; + assert.deepEqual(await reapExpired(journal, "deployment-a", { inspect: unexpected, remove: unexpected }, 61000), { cleaned: 0, failed: 0 }); +}); + +test("ownership mismatch never removes a reused name and retains retry evidence", async t => +{ + const { journal } = await fixture(t); + const attempt = await journal.prepare("deployment-a", 0); + let removals = 0; + const control = { + inspect: async () => ({ ...identity(attempt), labels: { [OWNER_LABEL]: "another-deployment", [ATTEMPT_LABEL]: attempt.id } }), + remove: async () => removals++, + }; + assert.deepEqual(await reapExpired(journal, "deployment-a", control, 60000), { cleaned: 0, failed: 1 }); + assert.equal(removals, 0); + assert.equal((await journal.expired("deployment-a", 60000)).length, 1); +}); + +test("failed removal remains retryable after restart, missing container closes lease", async t => +{ + const { directory, journal } = await fixture(t); + const attempt = await journal.prepare("deployment-a", 0); + await assert.rejects(cleanAttempt(journal, attempt, { + inspect: async () => identity(attempt), + remove: async () => + { + throw new Error("daemon disconnected"); + }, + }), /disconnected/); + const restarted = new AttemptJournal(directory, async () => undefined); + const result = await reapExpired(restarted, "deployment-a", { + inspect: async () => undefined, + remove: async () => + { + throw new Error("must not remove missing container"); + }, + }, 60000); + assert.deepEqual(result, { cleaned: 1, failed: 0 }); + assert.deepEqual(await restarted.expired("deployment-a", 60000), []); +}); + +test("durability failure prevents preparation of an active lease", async t => +{ + const { directory } = await fixture(t); + const failing = new AttemptJournal(directory, async () => + { + throw new Error("fsync unsupported"); + }); + await assert.rejects(failing.prepare("deployment-a"), /fsync unsupported/); + assert.deepEqual(await readdir(path.join(directory, "active")), []); +}); diff --git a/worker/test/transport-fixture.mjs b/worker/test/transport-fixture.mjs new file mode 100644 index 0000000..52ab40a --- /dev/null +++ b/worker/test/transport-fixture.mjs @@ -0,0 +1,24 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import process from "node:process"; +import { setTimeout } from "node:timers"; + +// Synthetic protocol peer only. No user source is loaded by this fixture. +process.stdin.once("data", () => +{ + if(process.argv[2] === "output") + { + process.stdout.write("x".repeat(2048)); + } + else if(process.argv[2] === "broker") + { + process.stdout.write(JSON.stringify({ protocol: "qbit-scripting-worker/1", type: "broker", requestId: "fixture-1", token: "wrong", operation: "query" }) + "\n"); + } + else if(process.argv[2] === "duplicate") + { + const frame = JSON.stringify({ protocol: "qbit-scripting-worker/1", type: "executeResult", requestId: "fixture-1", outcome: "OK", result: { action: "ALLOW" } }) + "\n"; + process.stdout.write(frame + frame); + } +}); +setTimeout(() => process.exit(0), 10000); diff --git a/worker/test/worker.test.mjs b/worker/test/worker.test.mjs new file mode 100644 index 0000000..1fa1555 --- /dev/null +++ b/worker/test/worker.test.mjs @@ -0,0 +1,197 @@ +/* Copyright (C) 2026 Kingsrook, LLC. + * SPDX-License-Identifier: Apache-2.0 + */ +import assert from "node:assert/strict"; +import { Buffer } from "node:buffer"; +import process from "node:process"; +import { tmpdir } from "node:os"; +import { spawn } from "node:child_process"; +import { Readable } from "node:stream"; +import test from "node:test"; +import { compile } from "../dist/compiler.js"; +import { sourceHash, PROTOCOL, TOOLCHAIN, hash } from "../dist/protocol.js"; +import { frames } from "../dist/frames.js"; +import { createArguments, qualify, relay } from "../dist/launcher.js"; + +/******************************************************************************* + ** Synthetic compile fixtures only; user-supplied code is never run in test host. + *******************************************************************************/ +function request(content, extra = []) +{ + const files = [{ path: "main.ts", content }, ...extra]; + return { protocol: PROTOCOL, type: "compile", requestId: "fixture-1", strict: true, toolchainId: TOOLCHAIN, entryPoint: "main.ts", files, sourceSha256: sourceHash(files) }; +} + +test("strict checking returns no executable on a type error", () => +{ + const result = compile(request("export function main(): string { return 12; }")); + assert.equal(result.success, false); + assert.equal(result.bundle, undefined); + assert.ok(result.diagnostics.length > 0); +}); + +test("local ESM imports compile with stable bundles and source maps", () => +{ + const source = request('import { value } from "./helper.js"; export function main() { return { answer: value }; }', [{ path: "helper.ts", content: "export const value: number = 42;" }]); + const first = compile(source); + assert.equal(first.success, true, JSON.stringify(first.diagnostics)); + assert.equal(first.bundle, compile(source).bundle); + assert.equal(first.bundleSha256, hash(first.bundle)); + assert.equal(JSON.parse(first.bundle).files.length, 2); + assert.match(first.sourceMap, /sourcesContent/); +}); + +test("SDK type imports support async queries without Node types", () => +{ + const result = compile(request('import type {Context,Qqq,HookResult} from "@qqq/sdk"; export async function main(context: Context, qqq: Qqq): Promise { await qqq.query("orders", ["id"]); return {action:"ALLOW"}; }')); + assert.equal(result.success, true, JSON.stringify(result.diagnostics)); + assert.equal(compile(request('import fs from "node:fs"; export const value = fs;')).success, false); + assert.equal(compile(request("export const value = process.env;")).success, false); + assert.equal(compile(request('export async function main() { return import("./helper.js"); }', [{ path: "helper.ts", content: "export const value = 1;" }])).success, false); +}); + +test("SDK exposes nine phases and refuses deleted-value assumptions", () => +{ + const result = compile(request('import type {HookEvent,Context,CommittedEvent} from "@qqq/sdk"; const phases: HookEvent[] = ["PRE_INSERT","PRE_UPDATE","PRE_DELETE","POST_INSERT","POST_UPDATE","POST_DELETE","AFTER_INSERT","AFTER_UPDATE","AFTER_DELETE"]; export function main(context: Context) { const event: CommittedEvent | undefined = context.payload.event; if(event?.dataState === "CURRENT") { return {action:"ALLOW", value:event.record.id, phases}; } return {action:"ALLOW"}; }')); + assert.equal(result.success, true, JSON.stringify(result.diagnostics)); + assert.equal(compile(request('import type {HookEvent} from "@qqq/sdk"; export const phase: HookEvent = "AFTER_ARBITRARY";')).success, false); + assert.equal(compile(request('import type {CommittedEvent} from "@qqq/sdk"; export function main(event: CommittedEvent) { if(event.dataState === "DELETED_METADATA") { return event.record.id; } return null; }')).success, false); +}); + +test("source identities and paths cannot be substituted", () => +{ + assert.throws(() => compile({ ...request("export const value = 1;"), sourceSha256: "0".repeat(64) })); + assert.throws(() => compile(request("export const value = 1;", [{ path: "../escape.ts", content: "" }]))); + assert.throws(() => compile(request("export const value = 1;", [{ path: "MAIN.ts", content: "" }]))); +}); + +test("frames enforce limits before parsing and reject missing delimiters", async () => +{ + const result = []; + for await (const frame of frames(Readable.from([Buffer.from('{"a":'), Buffer.from("1}\n")]), 20)) + { + result.push(frame); + } + assert.deepEqual(result, [{ a: 1 }]); + await assert.rejects(async () => + { + for await (const unused of frames(Readable.from([Buffer.alloc(21, 97)]), 20)) + { + void unused; + } + }); + await assert.rejects(async () => + { + for await (const unused of frames(Readable.from([Buffer.from("{}")]), 20)) + { + void unused; + } + }); +}); + +test("production requires runsc, matching evidence and an immutable image", () => +{ + const config = { image: "qqq-worker@sha256:" + "a".repeat(64), qualificationId: "review-1", requiredQualificationId: "review-1", ownerId: "fixture", stateDirectory: tmpdir() }; + qualify(config, { OSType: "linux", Runtimes: { runsc: {} } }); + assert.throws(() => qualify(config, { OSType: "linux", Runtimes: { runc: {} } })); + assert.throws(() => qualify({ ...config, qualificationId: "" }, { OSType: "linux", Runtimes: { runsc: {} } })); + assert.throws(() => qualify({ ...config, image: "node:latest" }, { OSType: "linux", Runtimes: { runsc: {} } })); + const args = createArguments(request("export const value=1;"), config, "fixture"); + for(const flag of ["--network=none", "--read-only", "--runtime=runsc", "--cap-drop=ALL", "--pids-limit=32", "--pull=never"]) + { + assert.ok(args.includes(flag)); + } + assert.equal(args.some(arg => arg.startsWith("--volume") || arg === "--privileged"), false); +}); + +test("container entry refuses direct host execution without the marker", async () => +{ + const child = spawn(process.execPath, ["dist/entrypoint.js"], { env: { PATH: process.env.PATH }, stdio: ["pipe", "pipe", "pipe"] }); + child.stdin.end(); + const code = await new Promise(resolve => child.once("close", resolve)); + assert.equal(code, 1); +}); + +test("trusted synthetic compile and async broker fixture run in a separate process", async () => +{ + // This fixture tests protocol mechanics only, never hostile-code containment. + const built = compile(request('import type {Context,Qqq} from "@qqq/sdk"; export async function main(context:Context, qqq:Qqq) { const rows=await qqq.query("orders",["id"],[],1); return {action:"PATCH",patch:{count:rows.length}}; }')); + assert.equal(built.success, true, JSON.stringify(built.diagnostics)); + const child = spawn(process.execPath, ["dist/entrypoint.js"], { env: { PATH: process.env.PATH, QQQ_WORKER_MODE: "container" }, stdio: ["pipe", "pipe", "pipe"] }); + const exit = new Promise(resolve => child.once("close", resolve)); + child.stdin.write(JSON.stringify({ protocol: PROTOCOL, type: "execute", requestId: "fixture-1", artifactId: "fixture", artifactSha256: built.bundleSha256, bundle: built.bundle, entryPoint: "main.ts", event: "PRE_INSERT", payload: {}, capabilityToken: "fixture-capability", limits: { brokerCalls: 1 } }) + "\n"); + let final; + for await (const message of frames(child.stdout)) + { + if(message.type === "broker") + { + assert.equal(message.operation, "query"); + assert.equal(message.token, "fixture-capability"); + child.stdin.write(JSON.stringify({ protocol: PROTOCOL, type: "brokerResult", ok: true, rows: [{ id: 1 }], truncated: false }) + "\n"); + } + else + { + final = message; + } + } + assert.equal(await exit, 0); + assert.deepEqual(final.result, { action: "PATCH", patch: { count: 1 } }); +}); + +/******************************************************************************* + ** Exercise the external transport supervisor with a harmless synthetic peer. + *******************************************************************************/ +function peer(mode) +{ + return spawn(process.execPath, ["test/transport-fixture.mjs", mode], { env: { PATH: process.env.PATH }, stdio: ["pipe", "pipe", "pipe"] }); +} + +test("supervisor terminates stalled, oversized and cancelled transports", async () => +{ + const run = { protocol: PROTOCOL, type: "execute", requestId: "fixture-1", capabilityToken: "expected" }; + const limits = { wallMillis: 150, cpuMillis: 1000, memoryBytes: 67108864, outputBytes: 1024, brokerCalls: 1 }; + const deny = async () => + { + throw new Error("Unexpected broker invocation"); + }; + await assert.rejects(relay(peer("idle"), run, limits, deny), /WALL_LIMIT/); + await assert.rejects(relay(peer("output"), run, { ...limits, wallMillis: 2000 }, deny), /LIMIT/); + const controller = new globalThis.AbortController(); + const pending = relay(peer("idle"), run, { ...limits, wallMillis: 2000 }, deny, controller.signal); + controller.abort(); + await assert.rejects(pending, /CANCELLED/); +}); + +test("supervisor rejects forged capability tokens and duplicate final frames", async () => +{ + const run = { protocol: PROTOCOL, type: "execute", requestId: "fixture-1", capabilityToken: "expected" }; + const limits = { wallMillis: 2000, cpuMillis: 1000, memoryBytes: 67108864, outputBytes: 1024, brokerCalls: 1 }; + const deny = async () => + { + throw new Error("Unexpected broker invocation"); + }; + await assert.rejects(relay(peer("broker"), run, limits, deny), /BROKER_LIMIT/); + await assert.rejects(relay(peer("duplicate"), run, limits, deny), /INVALID_RESPONSE/); +}); + +test("published CONTEXT fixture returns its JSON object without hook coercion", async () => +{ + const built = compile(request('import type {Context} from "@qqq/sdk"; export function main(context:Context) { return {document:context.payload.context}; }')); + assert.equal(built.success, true, JSON.stringify(built.diagnostics)); + const child = spawn(process.execPath, ["dist/entrypoint.js"], { env: { PATH: process.env.PATH, QQQ_WORKER_MODE: "container" }, stdio: ["pipe", "pipe", "pipe"] }); + const limits = { wallMillis: 2000, cpuMillis: 1000, memoryBytes: 67108864, outputBytes: 131072, brokerCalls: 0 }; + const run = { protocol: PROTOCOL, type: "execute", requestId: "fixture-1", artifactSha256: built.bundleSha256, bundle: built.bundle, entryPoint: "main.ts", event: "CONTEXT", payload: { context: { order: { id: 17 } } }, capabilityToken: "fixture-capability", limits }; + const result = await relay(child, run, limits, async () => + { + throw new Error("Unexpected broker invocation"); + }); + assert.deepEqual(result.result, { document: { order: { id: 17 } } }); +}); + +test("launcher rejects unsupported and coerced resource limits before Docker", () => +{ + const limits = { wallMillis: 2000, cpuMillis: 1000, memoryBytes: 67108864, outputBytes: 131072, brokerCalls: 0 }; + const config = { image: "qqq-worker@sha256:" + "a".repeat(64), qualificationId: "review-1", requiredQualificationId: "review-1", ownerId: "fixture", stateDirectory: tmpdir() }; + assert.throws(() => createArguments({ type: "execute", limits: { ...limits, cpuMillis: 500 } }, config, "fixture"), /UNSUPPORTED_LIMITS/); + assert.throws(() => createArguments({ type: "execute", limits: { ...limits, wallMillis: "2000" } }, config, "fixture"), /UNSUPPORTED_LIMITS/); +}); diff --git a/worker/tsconfig.json b/worker/tsconfig.json new file mode 100644 index 0000000..6fb1771 --- /dev/null +++ b/worker/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ES2023", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitOverride": true, + "verbatimModuleSyntax": true, + "noEmitOnError": true, + "outDir": "dist", + "rootDir": "src", + "types": ["node"], + "skipLibCheck": false, + "sourceMap": true + }, + "include": ["src/**/*.ts"] +}