Author SHA1 Message Date
james.maesandClaude Opus 5.5 48722efb1b ci: trigger CircleCI now that the project is followed
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 08:27:50 -05:00
28 changed files with 807 additions and 1021 deletions

No files matched your search

+57 -11
View File
@@ -1,14 +1,60 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.8
qqq-orb: kingsrook/qqq-orb@0.6.8
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
workflows:
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/^v\d+\.\d+\.\d+$/]
-7
View File
@@ -1,7 +0,0 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
-23
View File
@@ -1,23 +0,0 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+13 -62
View File
@@ -1,74 +1,25 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
'on':
on:
workflow_dispatch:
inputs:
dry_run:
description: Plan a cut without changing release state
description: Print the plan without merging or creating branches
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
types: [closed]
branches: [develop]
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
-16
View File
@@ -1,16 +0,0 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
+22 -13
View File
@@ -6,7 +6,7 @@ This file provides guidance to Codex when working with this repository.
QBit Jobs is the typed Jobs dashboard, record panels, alert declarations and retention
process for QQQ durable jobs. Read `docs/DESIGN.md` and `README.md` first. Core owns
all seven `jobs_*` tables and control actions. This qbit adds no tables and no client UI.
all six `jobs_*` tables and control actions. This qbit adds no tables and no client UI.
## Build commands
@@ -27,17 +27,17 @@ and the memory backend. No Docker or external services. Do not commit in this sa
## Architecture by package
- `com.kingsrook.qbits.jobs`: `JobsQBitProducer` registers the app and metadata directly;
`JobsQBitConfig` supplies validated host settings; `JobsCoreContract` references core identifiers
and field names; core `DurableJobsSecurity` derives owner/admin session keys; `JobsIntegrations`
`JobsQBitConfig` supplies validated host settings; `JobsCoreContract` holds temporary
core identifiers; `JobsSecurity` derives owner/admin session keys; `JobsIntegrations`
supplies opt-in ESB, Quick Search and Data Analysis table lists.
- `widgets`: `JobsWidgetRenderer` dispatches the eight home/record panels;
`JobsChartData` provides typed stacked bars; `JobsChildRecordListRenderer` ensures
session keys before delegating to core's child-list renderer.
- `processes`: `JobsPurgeFinishedStep` performs retention with 200-row keyset batches;
`JobsStorage` uses core storage actions and durable jobs metadata.
- Tests: `BaseJobsTest` uses core `DurableJobsMetaDataProvider` and `DurableJobsMetaData`,
plus producer, renderer, security and retention suites. Memory storage verifies real
object deletion; a recording storage seam exercises failures and retries.
`JobsStorage` isolates reflective core storage/metadata compatibility.
- Tests: `BaseJobsTest`, `JobsTablesStandIn`, `JobsAlertsValidationStandIn`, plus producer,
renderer, security and retention suites. Test storage emulates the future deletion API
without filesystem access; the real RC.5 adapter's skip behavior is tested separately.
## Invariants
@@ -64,16 +64,25 @@ and the memory backend. No Docker or external services. Do not commit in this sa
every class/method, fluent `withX` setters and the Kingsrook Apache license header.
- Enforce the repository's `checkstyle/config.xml`; do not weaken gates to pass builds.
## Dependencies
## Dependencies and stand-ins
Parent `com.kingsrook:qbit-build-parent:2.0.0`; QQQ 4.1.0-RC.7 or later is required.
The default `qqq-snapshot` profile imports released core BOM `4.1.0-RC.7`.
The opt-in `demo` profile uses the same release, adds `src/demo/java` and test
RDBMS/Javalin/H2 dependencies for the demo server.
Parent `com.kingsrook:qbit-build-parent:2.0.0`; default `qqq-snapshot` profile imports
released core BOM `4.1.0-RC.5`. Main code and default tests must not import `core.jobs`.
The opt-in `demo` profile selects locally installed `4.1.0-SNAPSHOT`, adds `src/demo/java`
and test RDBMS/Javalin/H2 dependencies. No demo server is provided yet.
Remove stand-ins when RC.6 supplies the contracts:
- `JobsCoreContract`, `JobsSecurity`, test `JobsTablesStandIn`: core #1178.
- `JobsStorage`: #1173 deleteObject method and #1178 durableJobs metadata getter.
- Test `JobsAlertsValidationStandIn`: #1178 kernel identifiers use underscores rejected
by the released alert-name validator. Only those two errors are waived in tests;
production validation remains strict. A core validator fix is required for startup.
## Release flow
CircleCI uses `kingsrook/qqq-orb@0.6.8`: feature branches test; `develop` publishes
SNAPSHOTs; `release/*` publishes RCs; `v*` tags publish stable releases. The GitHub
release-candidate workflow follows the repository standard. Publishing requires explicit
owner authorization. Update CHANGELOG before a release; do not change another repository from this workspace.
owner authorization. Update CHANGELOG and resolve the RC.6 compatibility requirements
before a release; do not change another repository from this workspace.
+4 -29
View File
@@ -5,50 +5,25 @@ and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.1.0] - Unreleased
### Security
- The Jobs app and its five non-admin home widgets now require `<appName>.hasAccess`
(default `jobs.hasAccess`) and are hidden otherwise. Behaviour change: hosts must grant
this permission to the users who should see the Jobs app. Workers still needs `jobs_admin`.
### Changed
- Built on QQQ 4.1.0-RC.7; stand-ins removed.
### Added
- RC.8 optional job progress fields: phase labels, explicit percent with legacy fallback,
message, binary byte counts, ETA and dropped-log notice in typed widgets.
- Full-width owner-scoped Log child list after Live Progress, ascending by log ID.
- System-session retention of log lines, including 64-bit cursors and 200-row batches.
- Host `parentAppName` configuration for Admin > Operations > Jobs placement.
- Progress/log, owner isolation, real core log permission and navigation tests.
### Changed
- Default QQQ BOM from 4.1.0-RC.5 to 4.1.0-RC.8; existing `qqq.snapshot.version`
override verifies against locally installed 4.1.0-SNAPSHOT without the demo profile.
- Storage adapter test now verifies failure/reference preservation with the current API.
### Earlier additions
- Optional effects outbox child list, viewer-scoped attention review counts and typed
Live Progress effect statistics; Data Analysis includes `jobs_effect`.
- `jobs.effect-unresolved` warning alert for effects requiring outcome review.
- Retention deletes settled effects and preserves jobs and inputs with outstanding
effects, reporting the number kept in the process summary.
- Outbox Postgres table and indexes, and coverage for effect counts, owner isolation
and retention.
- RC.6 outbox contract/test stand-ins, Postgres table and indexes, and coverage for
effect counts, owner isolation, retention and cores without the effects table.
- Jobs app with six typed home widgets for live progress, throughput, attention,
worker health and 30-day history, scoped to each viewer's record permissions.
- Job progress, ordered chunk/error/event child lists, and filtered audit links.
- Core-compatible failure/stall alert declarations and a reaper heartbeat monitor.
- Configurable, hidden, scheduled retention with system-session isolation, paged
deletion, object cleanup and standard process summary lines.
deletion, object cleanup compatibility and standard process summary lines.
- Opt-in ESB, Quick Search and Data Analysis registration lists.
- Postgres DDL for the core tables and indexes.
- Memory-backed tests using core durable jobs metadata and storage actions.
- Released RC.5 contract/security/storage stand-ins and memory-backed tests.
- Offline Maven wrapper build with checkstyle and JaCoCo coverage enforcement.
- Readable home widgets:
- block cells with record links and progress bars;
+25 -34
View File
@@ -8,7 +8,7 @@ from QRun-IO/qqq `docs/specs/DURABLE-JOBS-AND-ASSISTED-BULK-LOAD.md`.
## Requirements
- Java 21.
- QQQ 4.1.0-RC.7 or later with durable jobs enabled: `DurableJobsMetaDataProvider.defineAll(...)` and
- QQQ 4.1 with durable jobs enabled: `DurableJobsMetaDataProvider.defineAll(...)` and
`DurableJobsMetaData` on the instance.
- A storage table for chunk objects and uploads, which should be object storage (S3).
@@ -35,16 +35,12 @@ The tables are core's, prefixed `jobs_`. This qbit adds no tables.
| `jobs_chunk` | The leased units of work, with attempts, checkpoints and per-chunk counts. Child list on the job. |
| `jobs_row_error` | Which rows failed and why. Child list on the job; reportable. |
| `jobs_effect` | Side effects outside the database (outbox): email, HTTP, S3, other databases; child list on the job. |
| `jobs_log` | Append-only, owner-scoped log lines (level, time, phase, message); optional child list on the job. |
| `jobs_event` | The job's timeline (launched, reclaimed, paused, completed and so on). Child list on the job. |
| `jobs_worker` | The pods that run jobs. |
| `jobs_upload` | Uploaded files (refs, owners, expiry). |
## Permissions
- Users see their own jobs: core's record lock on `jobs_job.userId` through the `jobs_user` key.
- The Jobs app and its home widgets require `<appName>.hasAccess` (default `jobs.hasAccess`, from
`JobsQBitConfig.getHasAccessPermissionName()`); without it they are hidden. Hosts must grant it to the
users who should see the Jobs app. The header Jobs tray and record locks are unaffected.
- `jobs_admin` sees and controls every job, and sees the Workers widget.
- Apply the keys in your auth customizer with `DurableJobsSecurity.applyToSession(session)`.
@@ -56,17 +52,7 @@ The tables are core's, prefixed `jobs_`. This qbit adds no tables.
- `schedulerName` (null; set to a host scheduler name to enable the schedule);
- `homeRefreshSeconds` (5);
- `recordRefreshSeconds` (3);
- `appName` (`jobs`);
- `parentAppName` (null; an existing host app, such as `operations` under Admin).
To place Jobs at **Admin > Operations > Jobs**, register the host Admin and Operations
apps first (Operations has Admin as its parent), then produce with:
```java
new JobsQBitProducer().withQBitConfig(new JobsQBitConfig()
.withParentAppName("operations")).produce(qInstance);
```
The parent must exist and differ from `appName`; Jobs attaches to its children.
With no parent configured, Jobs remains a top-level app. Placement does not grant permissions.
- `appName` (`jobs`).
## Adopting in another app
1. Enable durable jobs in core (above) and create the tables. `docs/ddl/postgres.sql` is the Postgres
@@ -77,30 +63,41 @@ With no parent configured, Jobs remains a top-level app. Placement does not gran
## Build
```bash
./mvnw -o verify # tests, checkstyle (main and test sources), JaCoCo gate
./mvnw verify # tests, checkstyle (main and test sources), JaCoCo gate
```
The local demo server (`src/demo/java`, used for the screenshots) uses released QQQ
4.1.0-RC.7 through the `demo` profile; no locally installed kernel build is needed.
The local demo server (`src/demo/java`, used for the screenshots) runs against a locally installed
QQQ build that has the durable job kernel, under a private version so it never replaces a shared
SNAPSHOT:
```bash
./mvnw -o -Pdemo test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass=com.kingsrook.qbits.jobs.demo.JobsDemoServer
# in a QQQ checkout of feature/durable-jobs-chunked:
mvn install -DskipTests -Drevision=4.1.0-JOBS-SNAPSHOT
# here:
./mvnw -Pdemo -Dqqq.snapshot.version=4.1.0-JOBS-SNAPSHOT test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass=com.kingsrook.qbits.jobs.demo.JobsDemoServer
# then open http://localhost:8000/app/jobs
```
On Windows use `./mvnw.cmd` in place of `./mvnw`.
## Display
## Compatibility and display
The job progress fields and the Log list need QQQ 4.1.0-RC.8 or later; on older metadata
the optional progress fields may be absent and the Log list is left out.
The default build uses released core **4.1.0-RC.5**, which has no durable jobs kernel.
Production requires the kernel and the following core fixes (expected in RC.6):
- `QRun-IO/qqq#1178`: the jobs tables, constants and security metadata (alert types
`jobs.job-failed` and `jobs.job-stalled`).
- `QRun-IO/qqq#1191`: the optional `jobs_effect` outbox and `jobs.effect-unresolved` alert.
Effects child lists require the table and its job join; effect statistics and retention
checks are enabled when the table exists. The app also works without that table.
- `QRun-IO/qqq#1173`: `StorageAction.deleteObject`. Until present, object deletion logs
once and is skipped. Job/child row retention still runs. Cleanup of inputs on retained
clean jobs preserves their references when deletion is unavailable, allowing retry.
Objects belonging to fully purged jobs may need the host's storage lifecycle policy.
The app produces typed data only, with no HTML. The widget label is each card's title,
so the data carries no second title.
- **Tables.** The running, attention and worker tables have two or three flexible
columns. Their cells are composite widget blocks:
- the job's label links to its record, with the process label under it;
- a progress bar uses `percentDone` when present (including zero), otherwise rows/chunks;
- phase labels come from `progressJson.phases` with `currentPhase` as fallback;
- plain text shows the progress message and optional bytes in B/KiB/MiB/GiB;
- a progress bar shows the percent and rows done;
- the status line is a label (or the waiting reason) with the rate and ETA.
Each row also keeps its plain values (`Job`, `Process`, `Status`, `Rows`, `Rate`,
`ETA`, `Path` and so on) for exports and other clients.
@@ -115,13 +112,7 @@ Retention is admin/system-only and restores the original session even on failure
Its output contains standard process summary lines for jobs purged, chunk objects
removed during purge, and input objects removed (uploads plus retained-job inputs).
Jobs with effects outside SENT/FAILED, or with effects needing review, keep their rows
and inputs. When any are skipped, the summary includes "N jobs kept: effects still unsettled".
The Log list follows Live Progress, before Events: level, time, phase and message,
newest first, full width, with the standard child-list 25-row limit and View all link.
Live Progress also shows the ETA and a notice when `logDropped` is positive. The log is
read-only for users, including admins; retention deletes it through the system session in
keyset batches with 64-bit log cursors, before deleting chunks and jobs.
and inputs. When any are skipped, the summary includes "N jobs kept: effects still outstanding".
Needs attention shows each listed job's effect-review count; Live Progress adds an Effects
group (Sent, Pending, Failed, Needs review) when the job has effects.
+13 -32
View File
@@ -23,29 +23,6 @@ and phase 4. Owner decisions: all of that spec's recommendations are accepted (2
- **R6, storage.** Retention deletes finished jobs and their chunk objects from object storage. The
qbit reads and writes no files.
## Progress and log adoption (core RC.8)
Implements sections 2 and 7 of core `JOB-PROGRESS-AND-JOBS-TRAY.md`. The default
BOM is 4.1.0-RC.8; local verification uses `-Dqqq.snapshot.version=4.1.0-SNAPSHOT`.
Optional names remain in `JobsCoreContract` without importing `core.jobs`.
- Home and Live Progress prefer nullable `percentDone`, including explicit zero,
then retain rows/chunks derivation. The snapshot supplies the current phase label;
missing or invalid JSON falls back to `currentPhase`. Both views include the
progress message, optional binary byte counts, and explicit `etaSeconds` ahead
of the existing rate-derived ETA. No HTML, fixed widths, or theme-specific colors.
- Log is a full-width standard child list immediately after Live Progress, before
Events, only if both table and join exist. It displays level, loggedAt, phase and
message, sorted by id ascending, with the standard 25-row preview/View all.
The panel reads through the current viewer and core joined owner lock.
- A positive `logDropped` produces an earlier-lines-not-kept notice in Live Progress.
- Retention deletes logs before chunks/jobs, using the existing 200-row keyset loop
with a long child cursor. Core `JobLogPermissions` explicitly permits system
sessions and denies user mutations even for admins. Tests load the actual checker.
- Hosts set `parentAppName` to their existing Operations app under Admin; the qbit
sets the parent and attaches Jobs as a child, while retaining configurable appName.
Missing/self parents fail validation before registration. Default remains top-level.
## The app
- `jobs` app, label **Jobs**. The nav lists `jobs_job` only. Chunks, row errors, effects and events are child
lists on the job record.
@@ -80,7 +57,7 @@ Optional names remain in `JobsCoreContract` without importing `core.jobs`.
without needsReview), Failed and Needs review. Review effects appear only in Needs review.
The group is omitted when there are no effects or the effects table is absent.
It refreshes every 3s while the job is active.
- Child lists for **Log**, **Chunks**, **Row errors**, **Effects** and **Events**. Effects is
- Child lists for **Chunks**, **Row errors**, **Effects** and **Events**. Effects is
included only when the effects table and its inferred job join exist; it uses the same
joined owner lock as row errors.
- A **Changes made by this job** link to the audit table, filtered to messages containing
@@ -91,9 +68,10 @@ Optional names remain in `JobsCoreContract` without importing `core.jobs`.
## Retention
`jobsPurgeFinished` is a hidden scheduled process, not in the nav. It runs daily by default.
- It deletes jobs that finished more than `retentionDays` ago (default 30), with their chunks, row
errors, effects, logs, events and chunk objects. Effects are removed before chunks and jobs.
errors, effects, events and chunk objects. Effects are removed before chunks and jobs.
- Jobs with any effect outside SENT/FAILED, or any effect needing review, keep their rows
and inputs. The summary reports "N jobs kept: effects still unsettled" when any are skipped.
and inputs. The summary reports "N jobs kept: effects still outstanding" when any are skipped.
Cores without the effects table retain the existing behavior.
- For COMPLETED jobs with no failed rows, it deletes the chunk input objects after
`inputRetentionDays` (default 7).
- Each run's summary is the standard process summary.
@@ -102,9 +80,12 @@ Optional names remain in `JobsCoreContract` without importing `core.jobs`.
`JobsIntegrations.esbTableNames()`, `quickSearchTables()` and `dataAnalysisTables()`: a host passes
these to qqq-esb, qbit-quick-search and qbit-data-analysis.
## Dependencies
Requires QQQ 4.1.0-RC.8 or later (job progress and `jobs_log`). The default `qqq-snapshot` profile and the opt-in
`demo` profile both use released RC.8. Core provides table metadata, joins, possible
values, session security, signal identifiers and storage deletion. The qbit declares
the alert types and retains only field names that core does not export.
The demo server under `src/demo/java` uses the same released kernel.
## Dependencies and stand-ins
The qbit builds against the released QQQ RC through the default `qqq-snapshot` profile. Until the
kernel ships in an RC, the core names it needs (table, field and process names, and the jobs session
keys) are mirrored in `JobsCoreContract`, tagged
`STAND-IN(core=QRun-IO/qqq#1178, removeBy=RC.6)`. When RC.6 is released, those constants switch to
core's `DurableJobsTables`, and `JobsCoreContract` and the test-only table mirror are deleted.
The optional outbox additions carry `STAND-IN(core=QRun-IO/qqq#1191, removeBy=RC.6)`.
The demo server (`-Pdemo`, under `src/demo/java`) runs against a locally installed 4.1.0-SNAPSHOT of
the kernel branch. It is how the screenshots are taken.
+2 -25
View File
@@ -1,4 +1,4 @@
-- Durable jobs schema matching QQQ 4.1.0-RC.7 DurableJobsMetaDataProvider.
-- Durable jobs schema, copied from core prime-durable-jobs.sql (QRun-IO/qqq#1178).
-- Snake-case columns; excludes test fixtures and destructive DROP statements.
CREATE TABLE jobs_job
@@ -21,15 +21,6 @@ CREATE TABLE jobs_job
semantics VARCHAR(20),
chunk_size INTEGER,
max_parallel_chunks INTEGER,
current_phase VARCHAR(250),
percent_done INTEGER,
progress_message VARCHAR(1000),
bytes_done BIGINT,
bytes_total BIGINT,
eta_seconds BIGINT,
progress_json TEXT,
log_seq BIGINT,
log_dropped INTEGER,
rows_total INTEGER,
rows_done INTEGER,
rows_ok INTEGER,
@@ -105,7 +96,7 @@ CREATE TABLE jobs_row_error
CREATE INDEX jobs_row_error_job_idx ON jobs_row_error (job_id);
-- Transactional outbox.
-- STAND-IN(core=QRun-IO/qqq#1191, removeBy=RC.6): transactional outbox.
CREATE TABLE jobs_effect
(
id SERIAL PRIMARY KEY,
@@ -182,17 +173,3 @@ CREATE TABLE jobs_upload
job_id INTEGER,
create_date TIMESTAMPTZ
);
-- RC.8 job progress log; host migrations must also add the jobs_job columns above.
CREATE TABLE jobs_log
(
id BIGSERIAL PRIMARY KEY,
job_id INTEGER NOT NULL REFERENCES jobs_job(id) ON DELETE CASCADE,
chunk_id INTEGER REFERENCES jobs_chunk(id) ON DELETE SET NULL,
attempt INTEGER,
level VARCHAR(10) NOT NULL,
logged_at TIMESTAMPTZ NOT NULL,
phase VARCHAR(250),
message VARCHAR(1000)
);
CREATE INDEX jobs_log_cursor_idx ON jobs_log(job_id, id);
+19 -8
View File
@@ -18,7 +18,6 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
<revision>0.1.0-SNAPSHOT</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
@@ -28,7 +27,7 @@
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.2.1</version>
<version>2.0.0</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -172,7 +171,7 @@
<profiles>
<!-- Built against qqq 4.1. Until qbit-build-parent imports the released 4.1 BOM, this -->
<!-- profile is active by default and imports qqq-bom-pom 4.1.0-RC.8 ahead of the parent's -->
<!-- profile is active by default and imports qqq-bom-pom 4.1.0-RC.5 ahead of the parent's -->
<!-- 4.0.0 BOM (ADR-0007, as in the other qbits). -->
<profile>
<id>qqq-snapshot</id>
@@ -180,7 +179,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.5</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -193,14 +192,26 @@
</dependency>
</dependencies>
</dependencyManagement>
<repositories>
<repository>
<id>sonatype-snapshots</id>
<url>https://central.sonatype.com/repository/maven-snapshots/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>
</profile>
<!-- Local demo server and screenshots: builds src/demo/java against released QQQ RC.7. -->
<!-- The version property remains overridable, as in the default profile. -->
<!-- Local demo server and screenshots only (never in CI): builds src/demo/java against a -->
<!-- locally installed QQQ with the durable job kernel, e.g. -Dqqq.snapshot.version=4.1.0-JOBS-SNAPSHOT. -->
<profile>
<id>demo</id>
<properties>
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-SNAPSHOT</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -227,7 +238,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-frontend-next</artifactId>
<version>1.0.0-RC.38</version>
<version>1.0.0-RC.19</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -23,51 +23,28 @@ package com.kingsrook.qbits.jobs;
import java.util.List;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsTables;
import com.kingsrook.qqq.backend.core.jobs.effects.EffectStatus;
import com.kingsrook.qqq.backend.core.jobs.effects.JobEffectType;
import com.kingsrook.qqq.backend.core.jobs.model.ChunkStatus;
import com.kingsrook.qqq.backend.core.jobs.model.JobStatus;
import com.kingsrook.qqq.backend.core.jobs.runtime.JobSignals;
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
/*******************************************************************************
** Core identifiers and field names used by the Jobs app.
** STAND-IN(core=QRun-IO/qqq#1178, removeBy=RC.6): mirrors core DurableJobsTables and friends; switch to core's constants when RC.6 is released.
*******************************************************************************/
public final class JobsCoreContract
{
// Optional progress and log contract (core 4.1.0-RC.8).
public static final String JOBS_LOG = DurableJobsTables.TABLE_LOG;
public static final String CURRENT_PHASE = "currentPhase";
public static final String PERCENT_DONE = "percentDone";
public static final String PROGRESS_MESSAGE = "progressMessage";
public static final String BYTES_DONE = "bytesDone";
public static final String BYTES_TOTAL = "bytesTotal";
public static final String ETA_SECONDS = "etaSeconds";
public static final String LOG_SEQ = "logSeq";
public static final String LOG_DROPPED = "logDropped";
public static final String PROGRESS_JSON = "progressJson";
public static final String LEVEL = "level";
public static final String LOGGED_AT = "loggedAt";
public static final String PHASE = "phase";
public static final String JOBS_JOB = DurableJobsTables.TABLE_JOB;
public static final String JOBS_CHUNK = DurableJobsTables.TABLE_CHUNK;
public static final String JOBS_ROW_ERROR = DurableJobsTables.TABLE_ROW_ERROR;
public static final String JOBS_EVENT = DurableJobsTables.TABLE_EVENT;
public static final String JOBS_WORKER = DurableJobsTables.TABLE_WORKER;
public static final String JOBS_UPLOAD = DurableJobsTables.TABLE_UPLOAD;
public static final String JOBS_USER = DurableJobsTables.SECURITY_KEY_USER;
public static final String JOBS_USER_ALL_ACCESS = DurableJobsTables.SECURITY_KEY_USER_ALL_ACCESS;
public static final String JOBS_ADMIN = DurableJobsTables.PERMISSION_ADMIN;
public static final String ALERT_JOB_FAILED = JobSignals.ALERT_JOB_FAILED;
public static final String ALERT_JOB_STALLED = JobSignals.ALERT_JOB_STALLED;
public static final String HEARTBEAT_REAPER = JobSignals.HEARTBEAT_REAPER;
public static final String DURABLE_JOBS_METADATA = DurableJobsMetaData.NAME;
public static final String AUDIT = AuditsMetaDataProvider.TABLE_NAME_AUDIT;
public static final String JOBS_JOB = "jobs_job";
public static final String JOBS_CHUNK = "jobs_chunk";
public static final String JOBS_ROW_ERROR = "jobs_row_error";
public static final String JOBS_EVENT = "jobs_event";
public static final String JOBS_WORKER = "jobs_worker";
public static final String JOBS_UPLOAD = "jobs_upload";
public static final String JOBS_USER = "jobs_user";
public static final String JOBS_USER_ALL_ACCESS = "jobs_user_all_access";
public static final String JOBS_ADMIN = "jobs_admin";
public static final String ALERT_JOB_FAILED = "jobs.job-failed";
public static final String ALERT_JOB_STALLED = "jobs.job-stalled";
public static final String HEARTBEAT_REAPER = "jobs.reaper";
public static final String DURABLE_JOBS_METADATA = "durableJobs";
public static final String AUDIT = "audit";
public static final String ID = "id";
public static final String LABEL = "label";
public static final String PROCESS_NAME = "processName";
@@ -141,24 +118,25 @@ public final class JobsCoreContract
public static final String STORAGE_TABLE_NAME = "storageTableName";
public static final String STORAGE_REFERENCE = "storageReference";
public static final String EXPIRES_AT = "expiresAt";
public static final String PLANNING = JobStatus.PLANNING.getPossibleValueId();
public static final String AWAITING_CONFIRMATION = JobStatus.AWAITING_CONFIRMATION.getPossibleValueId();
public static final String QUEUED = JobStatus.QUEUED.getPossibleValueId();
public static final String RUNNING = JobStatus.RUNNING.getPossibleValueId();
public static final String PAUSING = JobStatus.PAUSING.getPossibleValueId();
public static final String PAUSED = JobStatus.PAUSED.getPossibleValueId();
public static final String CANCELLING = JobStatus.CANCELLING.getPossibleValueId();
public static final String CANCELLED = JobStatus.CANCELLED.getPossibleValueId();
public static final String COMPLETED = JobStatus.COMPLETED.getPossibleValueId();
public static final String COMPLETED_WITH_ERRORS = JobStatus.COMPLETED_WITH_ERRORS.getPossibleValueId();
public static final String FAILED = JobStatus.FAILED.getPossibleValueId();
public static final String NEEDS_REVIEW = JobStatus.NEEDS_REVIEW.getPossibleValueId();
public static final String DONE = ChunkStatus.DONE.getPossibleValueId();
public static final String PLANNING = "PLANNING";
public static final String AWAITING_CONFIRMATION = "AWAITING_CONFIRMATION";
public static final String QUEUED = "QUEUED";
public static final String RUNNING = "RUNNING";
public static final String PAUSING = "PAUSING";
public static final String PAUSED = "PAUSED";
public static final String CANCELLING = "CANCELLING";
public static final String CANCELLED = "CANCELLED";
public static final String COMPLETED = "COMPLETED";
public static final String COMPLETED_WITH_ERRORS = "COMPLETED_WITH_ERRORS";
public static final String FAILED = "FAILED";
public static final String NEEDS_REVIEW = "NEEDS_REVIEW";
public static final String DONE = "DONE";
public static final String JOBS_EFFECT = DurableJobsTables.TABLE_EFFECT;
public static final String JOBS_EFFECT_STATUS = EffectStatus.NAME;
public static final String JOBS_EFFECT_TYPE = JobEffectType.NAME;
public static final String ALERT_EFFECT_UNRESOLVED = JobSignals.ALERT_EFFECT_UNRESOLVED;
// STAND-IN(core=QRun-IO/qqq#1191, removeBy=RC.6): transactional outbox contract.
public static final String JOBS_EFFECT = "jobs_effect";
public static final String JOBS_EFFECT_STATUS = "jobs_effect_status";
public static final String JOBS_EFFECT_TYPE = "jobs_effect_type";
public static final String ALERT_EFFECT_UNRESOLVED = "jobs.effect-unresolved";
public static final String EFFECT_NAME = "effectName";
public static final String NEEDS_REVIEW_FIELD = "needsReview";
public static final String ROW_KEY = "rowKey";
@@ -173,13 +151,13 @@ public final class JobsCoreContract
public static final String PROVIDER_REF = "providerRef";
public static final String LAST_ERROR = "lastError";
public static final String SENT_AT = "sentAt";
public static final String PENDING = EffectStatus.PENDING.getPossibleValueId();
public static final String SENDING = EffectStatus.SENDING.getPossibleValueId();
public static final String SENT = EffectStatus.SENT.getPossibleValueId();
public static final String UNKNOWN = EffectStatus.UNKNOWN.getPossibleValueId();
public static final String LOCAL_TX = JobEffectType.LOCAL_TX.getPossibleValueId();
public static final String IDEMPOTENT_EXTERNAL = JobEffectType.IDEMPOTENT_EXTERNAL.getPossibleValueId();
public static final String NON_IDEMPOTENT_EXTERNAL = JobEffectType.NON_IDEMPOTENT_EXTERNAL.getPossibleValueId();
public static final String PENDING = "PENDING";
public static final String SENDING = "SENDING";
public static final String SENT = "SENT";
public static final String UNKNOWN = "UNKNOWN";
public static final String LOCAL_TX = "LOCAL_TX";
public static final String IDEMPOTENT_EXTERNAL = "IDEMPOTENT_EXTERNAL";
public static final String NON_IDEMPOTENT_EXTERNAL = "NON_IDEMPOTENT_EXTERNAL";
public static final List<String> DATA_TABLES = List.of(JOBS_JOB, JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EFFECT, JOBS_EVENT, JOBS_WORKER);
public static final List<String> ALL_TABLES = List.of(JOBS_JOB, JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EVENT, JOBS_WORKER, JOBS_UPLOAD);
@@ -200,7 +178,7 @@ public final class JobsCoreContract
*******************************************************************************/
public static String childJoin(String child)
{
return (DurableJobsMetaDataProvider.childJoinName(child));
return (QJoinMetaData.makeInferredJoinName(JOBS_JOB, child));
}
/*******************************************************************************
@@ -213,13 +191,21 @@ public final class JobsCoreContract
{
return ("Unknown");
}
try
return (switch(status)
{
return (JobStatus.valueOf(status).getPossibleValueLabel());
}
catch(IllegalArgumentException e)
{
return (status);
}
case AWAITING_CONFIRMATION -> "Awaiting Confirmation";
case QUEUED -> "Queued";
case PLANNING -> "Planning";
case RUNNING -> "Running";
case PAUSING -> "Pausing";
case PAUSED -> "Paused";
case CANCELLING -> "Cancelling";
case CANCELLED -> "Cancelled";
case COMPLETED -> "Completed";
case COMPLETED_WITH_ERRORS -> "Completed With Errors";
case FAILED -> "Failed";
case NEEDS_REVIEW -> "Needs Review";
default -> status;
});
}
}
@@ -34,7 +34,6 @@ import org.quartz.CronExpression;
public class JobsQBitConfig implements QBitConfig
{
private String appName = "jobs";
private String parentAppName;
private int retentionDays = 30;
private int inputRetentionDays = 7;
private String purgeCronExpression = "0 30 3 * * ?";
@@ -42,31 +41,6 @@ public class JobsQBitConfig implements QBitConfig
private int recordRefreshSeconds = 3;
private String schedulerName = null;
/*******************************************************************************
** Get the optional host parent app.
*******************************************************************************/
public String getParentAppName()
{
return (parentAppName);
}
/*******************************************************************************
** Set the optional host parent app.
*******************************************************************************/
public void setParentAppName(String parentAppName)
{
this.parentAppName = parentAppName;
}
/*******************************************************************************
** Place Jobs under an existing host app, such as Operations under Admin.
*******************************************************************************/
public JobsQBitConfig withParentAppName(String parentAppName)
{
this.parentAppName = parentAppName;
return (this);
}
/*******************************************************************************
** Report invalid settings before adding any metadata.
*******************************************************************************/
@@ -74,7 +48,6 @@ public class JobsQBitConfig implements QBitConfig
public void validate(QInstance instance, List<String> errors)
{
assertCondition(appName != null && appName.matches("[A-Za-z][A-Za-z0-9_]*"), "appName must be a valid name", errors);
assertCondition(parentAppName == null || (!parentAppName.equals(appName) && instance.getApp(parentAppName) != null), "parentAppName must name an existing, different app", errors);
assertCondition(retentionDays > 0, "retentionDays must be positive", errors);
assertCondition(inputRetentionDays > 0 && inputRetentionDays <= retentionDays, "inputRetentionDays must be positive and no greater than retentionDays", errors);
assertCondition(homeRefreshSeconds > 0 && recordRefreshSeconds > 0, "refresh seconds must be positive", errors);
@@ -82,15 +55,6 @@ public class JobsQBitConfig implements QBitConfig
assertCondition(schedulerName == null || !schedulerName.isBlank(), "schedulerName must be null or nonblank", errors);
}
/*******************************************************************************
** The permission a user needs to see the Jobs app and its home widgets,
** "<appName>.hasAccess" (default "jobs.hasAccess"). Hosts must grant it.
*******************************************************************************/
public String getHasAccessPermissionName()
{
return (appName + ".hasAccess");
}
/*******************************************************************************
** Get appName.
*******************************************************************************/
@@ -45,7 +45,6 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
@@ -67,12 +66,7 @@ import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_LOG;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ROW_ERROR;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LEVEL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LOGGED_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.MESSAGE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PHASE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROW_NUMBER;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SEQ;
@@ -168,28 +162,10 @@ public class JobsQBitProducer implements QBitMetaDataProducer<JobsQBitConfig>
QWidgetMetaData workers = registerWidget(instance, "jobsHomeWorkers", "Workers", WidgetType.TABLE, qBitConfig.getHomeRefreshSeconds());
workers.withPermissionRules(adminRules());
registerWidget(instance, "jobsHomeHistory", "Job history", WidgetType.STACKED_BAR_CHART, qBitConfig.getHomeRefreshSeconds());
for(String name : HOME_WIDGETS)
{
if(!"jobsHomeWorkers".equals(name))
{
instance.getWidget(name).setPermissionRules(appAccessRules());
}
}
QAppMetaData app = new QAppMetaData().withName(qBitConfig.getAppName()).withLabel("Jobs").withIcon(new QIcon("pending_actions"))
.withChild(instance.getTable(JOBS_JOB)).withWidgets(new ArrayList<>(HOME_WIDGETS))
.withPermissionRules(appAccessRules());
app.setParentAppName(qBitConfig.getParentAppName());
instance.addApp(app);
if(qBitConfig.getParentAppName() != null)
{
instance.getApp(qBitConfig.getParentAppName()).withChild(app);
}
instance.addApp(new QAppMetaData().withName(qBitConfig.getAppName()).withLabel("Jobs").withIcon(new QIcon("pending_actions"))
.withChild(instance.getTable(JOBS_JOB)).withWidgets(new ArrayList<>(HOME_WIDGETS)));
registerWidget(instance, "jobsJobProgress", "Live Progress", WidgetType.MULTI_STATISTICS, qBitConfig.getRecordRefreshSeconds());
addRecordSection(instance, "jobsJobProgress", "Live Progress");
if(instance.getTable(JOBS_LOG) != null && instance.getJoin(JobsCoreContract.childJoin(JOBS_LOG)) != null)
{
child(instance, JOBS_LOG, "jobsJobLog", "Log", ID, false);
}
child(instance, JOBS_CHUNK, "jobsJobChunks", "Chunks", SEQ, true);
child(instance, JOBS_ROW_ERROR, "jobsJobRowErrors", "Row Errors", ROW_NUMBER, true);
if(instance.getTable(JOBS_EFFECT) != null && instance.getJoin(JobsCoreContract.childJoin(JOBS_EFFECT)) != null)
@@ -227,10 +203,6 @@ public class JobsQBitProducer implements QBitMetaDataProducer<JobsQBitConfig>
.withOrderBys(List.of(new QFilterOrderBy(order, ascending))).withMaxRows(25).getWidgetMetaData()
.withName(name).withLabel(label).withGridColumns(12).withCodeReference(new QCodeReference(JobsChildRecordListRenderer.class))
.withDefaultValue("refreshSeconds", qBitConfig.getRecordRefreshSeconds());
if(JOBS_LOG.equals(table))
{
widget.withDefaultValue(ChildRecordListRenderer.KEY_ONLY_INCLUDE_FIELD_NAMES, new ArrayList<>(List.of(LEVEL, LOGGED_AT, PHASE, MESSAGE)));
}
instance.addWidget(widget);
addRecordSection(instance, name, label);
}
@@ -244,16 +216,6 @@ public class JobsQBitProducer implements QBitMetaDataProducer<JobsQBitConfig>
.withTier(Tier.T2).withWidgetName(widgetName).withGridColumns(12));
}
/*******************************************************************************
** The Jobs app and its home widgets require "<appName>.hasAccess"; users
** without it never see the app or its widgets (hidden, not locked).
*******************************************************************************/
private QPermissionRules appAccessRules()
{
return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
.withPermissionBaseName(qBitConfig.getAppName()).withDenyBehavior(DenyBehavior.HIDDEN));
}
/*******************************************************************************
** Match the exact core permission, without a .hasAccess suffix.
*******************************************************************************/
@@ -0,0 +1,118 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.jobs;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
/*******************************************************************************
** STAND-IN(core=QRun-IO/qqq#1178, removeBy=RC.6): copy of core session security.
** Puts the durable-jobs security keys into a session: jobs_user = the user's id,
** plus jobs_user_all_access for sessions holding the jobs_admin permission.
**
** A host calls applyToSession from its authentication customizer
** (QAuthenticationModuleCustomizerInterface.finalCustomizeSession), the same as
** for the alerts keys. The kernel also ensures the keys on the sessions it
** builds for workers and on launch/control calls, so a host that forgets still
** gets correct (owner-only) behaviour there; a session without the keys sees no
** jobs through the standard table screens (the lock fails closed).
*******************************************************************************/
public final class JobsSecurity
{
/*******************************************************************************
** not instantiable
*******************************************************************************/
private JobsSecurity()
{
}
/*******************************************************************************
** Replace any jobs keys on the session with ones derived from its user and
** permissions. No-op for null and system sessions.
*******************************************************************************/
public static void applyToSession(QSession session)
{
if(session == null || session instanceof QSystemUserSession)
{
return;
}
Map<String, List<Serializable>> keys = session.getSecurityKeyValues();
if(keys != null)
{
keys.remove(JobsCoreContract.JOBS_USER);
keys.remove(JobsCoreContract.JOBS_USER_ALL_ACCESS);
}
String userId = userId(session);
if(StringUtils.hasContent(userId))
{
session.withSecurityKeyValue(JobsCoreContract.JOBS_USER, userId);
}
if(session.hasPermission(JobsCoreContract.JOBS_ADMIN))
{
session.withSecurityKeyValue(JobsCoreContract.JOBS_USER_ALL_ACCESS, true);
}
}
/*******************************************************************************
** Apply the keys only when the session has none yet.
*******************************************************************************/
public static void ensureSessionKeys(QSession session)
{
if(session == null || session instanceof QSystemUserSession)
{
return;
}
if(session.getSecurityKeyValues(JobsCoreContract.JOBS_USER).isEmpty())
{
applyToSession(session);
}
}
/*******************************************************************************
** The id the jobs tables record as a job's owner.
*******************************************************************************/
public static String userId(QSession session)
{
if(session == null || session.getUser() == null)
{
return (null);
}
return (session.getUser().getIdReference());
}
}
@@ -60,7 +60,6 @@ import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_LOG;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ROW_ERROR;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_UPLOAD;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
@@ -91,7 +90,7 @@ public class JobsPurgeFinishedStep implements BackendStep
}
/*******************************************************************************
** Storage seam for testing deletion failures and retries.
** Storage seam for testing RC.6 deletion semantics against RC.5.
*******************************************************************************/
public JobsPurgeFinishedStep(JobsStorage storage)
{
@@ -128,7 +127,7 @@ public class JobsPurgeFinishedStep implements BackendStep
summary.add(new ProcessSummaryLine(Status.INFO, counts[2], "input objects deleted"));
if(counts[3] > 0)
{
summary.add(new ProcessSummaryLine(Status.INFO, counts[3], "jobs kept: effects still unsettled"));
summary.add(new ProcessSummaryLine(Status.INFO, counts[3], "jobs kept: effects still outstanding"));
}
output.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, summary);
}
@@ -151,7 +150,7 @@ public class JobsPurgeFinishedStep implements BackendStep
for(QRecord job : jobs)
{
lastId = job.getValueInteger(ID);
if(hasUnsettledEffects(lastId))
if(hasOutstandingEffects(lastId))
{
counts[3]++;
continue;
@@ -160,10 +159,6 @@ public class JobsPurgeFinishedStep implements BackendStep
{
removeChildren(JOBS_EFFECT, lastId, counts);
}
if(QContext.getQInstance().getTable(JOBS_LOG) != null)
{
removeChildren(JOBS_LOG, lastId, counts);
}
removeChildren(JOBS_ROW_ERROR, lastId, counts);
removeChildren(JOBS_EVENT, lastId, counts);
removeChildren(JOBS_CHUNK, lastId, counts);
@@ -177,17 +172,17 @@ public class JobsPurgeFinishedStep implements BackendStep
/*******************************************************************************
** Preserve unresolved outbox work, including unknown or missing statuses.
*******************************************************************************/
private boolean hasUnsettledEffects(int jobId) throws QException
private boolean hasOutstandingEffects(int jobId) throws QException
{
if(QContext.getQInstance().getTable(JOBS_EFFECT) == null)
{
return (false);
}
QQueryFilter unsettled = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
QQueryFilter outstanding = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
.withCriteria(STATUS, QCriteriaOperator.NOT_IN, List.of(SENT, FAILED))
.withCriteria(STATUS, QCriteriaOperator.IS_BLANK)
.withCriteria(NEEDS_REVIEW_FIELD, QCriteriaOperator.EQUALS, true);
QQueryFilter filter = new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.EQUALS, jobId).withSubFilter(unsettled).withLimit(1);
QQueryFilter filter = new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.EQUALS, jobId).withSubFilter(outstanding).withLimit(1);
return (!new QueryAction().execute(new QueryInput().withTableName(JOBS_EFFECT).withFilter(filter)).getRecords().isEmpty());
}
@@ -196,7 +191,7 @@ public class JobsPurgeFinishedStep implements BackendStep
*******************************************************************************/
private void removeChildren(String table, int jobId, int[] counts) throws QException
{
long lastId = 0;
int lastId = 0;
while(true)
{
List<QRecord> records = batch(table, new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.EQUALS, jobId), lastId);
@@ -206,7 +201,7 @@ public class JobsPurgeFinishedStep implements BackendStep
}
for(QRecord record : records)
{
lastId = record.getValueLong(ID);
lastId = record.getValueInteger(ID);
if(JOBS_CHUNK.equals(table) && storage.delete(storage.chunkStorageTable(), record.getValueString(INPUT_STORAGE_REF)))
{
counts[1]++;
@@ -239,7 +234,7 @@ public class JobsPurgeFinishedStep implements BackendStep
for(QRecord job : jobs)
{
lastId = job.getValueInteger(ID);
if(!hasUnsettledEffects(lastId))
if(!hasOutstandingEffects(lastId))
{
cleanJobInputs(lastId, counts);
}
@@ -281,7 +276,7 @@ public class JobsPurgeFinishedStep implements BackendStep
/*******************************************************************************
** Keyset pagination remains correct when rows are deleted or updated.
*******************************************************************************/
private List<QRecord> batch(String table, QQueryFilter filter, long lastId) throws QException
private List<QRecord> batch(String table, QQueryFilter filter, int lastId) throws QException
{
return (new QueryAction().execute(new QueryInput().withTableName(table).withFilter(filter.withCriteria(ID, QCriteriaOperator.GREATER_THAN, lastId)
.withOrderBy(new QFilterOrderBy(ID)).withLimit(BATCH_SIZE))).getRecords());
@@ -22,29 +22,49 @@
package com.kingsrook.qbits.jobs.processes;
import java.util.concurrent.atomic.AtomicBoolean;
import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
import com.kingsrook.qqq.backend.core.logging.QLogger;
import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
import static com.kingsrook.qbits.jobs.JobsCoreContract.DURABLE_JOBS_METADATA;
/*******************************************************************************
** Object cleanup through core storage actions and durable jobs configuration.
** STAND-IN(core=QRun-IO/qqq#1173, removeBy=RC.6): reflectively call StorageAction
** deleteObject until the released core contains that method. Never touch disk.
** STAND-IN(core=QRun-IO/qqq#1178, removeBy=RC.6): read durable jobs supplemental
** metadata reflectively until its type is available in the released core.
*******************************************************************************/
public class JobsStorage
{
private static final QLogger LOG = QLogger.getLogger(JobsStorage.class);
private static final AtomicBoolean WARNED = new AtomicBoolean();
/*******************************************************************************
** Look up the host's durable jobs storage table.
** Look up the host's durable jobs storage table without linking core.jobs.
*******************************************************************************/
public String chunkStorageTable() throws QException
{
DurableJobsMetaData metadata = DurableJobsMetaData.of(QContext.getQInstance());
return (metadata == null ? null : metadata.getStorageTableName());
Object metadata = QContext.getQInstance().getSupplementalMetaData(DURABLE_JOBS_METADATA);
if(metadata == null)
{
return (null);
}
try
{
return ((String) metadata.getClass().getMethod("getStorageTableName").invoke(metadata));
}
catch(ReflectiveOperationException e)
{
throw (new QException("Cannot read durable jobs storage metadata", e));
}
}
/*******************************************************************************
** Return true only after successful deletion; failures abort retention for retry.
** Return true only after a successful deletion. Missing APIs leave references
** intact during input cleanup. Invocation failures abort retention for retry.
*******************************************************************************/
public boolean delete(String table, String reference) throws QException
{
@@ -52,7 +72,22 @@ public class JobsStorage
{
return (false);
}
new StorageAction().deleteObject(new StorageInput(table).withReference(reference));
return (true);
try
{
StorageAction.class.getMethod("deleteObject", StorageInput.class).invoke(new StorageAction(), new StorageInput(table).withReference(reference));
return (true);
}
catch(NoSuchMethodException e)
{
if(WARNED.compareAndSet(false, true))
{
LOG.warn("StorageAction.deleteObject is unavailable in this core release; skipping jobs object deletion until RC.6");
}
return (false);
}
catch(ReflectiveOperationException e)
{
throw (new QException("Could not delete durable jobs storage object", e));
}
}
}
@@ -22,10 +22,10 @@
package com.kingsrook.qbits.jobs.widgets;
import com.kingsrook.qbits.jobs.JobsSecurity;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
@@ -41,7 +41,7 @@ public class JobsChildRecordListRenderer extends ChildRecordListRenderer
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
DurableJobsSecurity.ensureSessionKeys(QContext.getQSession());
JobsSecurity.ensureSessionKeys(QContext.getQSession());
return (super.render(input));
}
}
@@ -52,24 +52,6 @@ public final class JobsFormat
/*******************************************************************************
** Binary byte units, with one decimal place above bytes.
*******************************************************************************/
public static String bytes(long value)
{
double amount = Math.max(0, value);
String[] units = {"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"};
int unit = 0;
while(amount >= 1024 && unit < units.length - 1)
{
amount /= 1024;
unit++;
}
return (unit == 0 ? count((long) amount) + " B" : String.format(Locale.US, "%.1f %s", amount, units[unit]));
}
/*******************************************************************************
** A count with thousands separators.
*******************************************************************************/
@@ -22,7 +22,6 @@
package com.kingsrook.qbits.jobs.widgets;
import java.io.IOException;
import java.io.Serializable;
import java.math.BigDecimal;
import java.net.URLEncoder;
@@ -44,13 +43,12 @@ import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import com.fasterxml.jackson.databind.JsonNode;
import com.kingsrook.qbits.jobs.JobsSecurity;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
@@ -75,18 +73,14 @@ import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ATTENTION_STATUSES;
import static com.kingsrook.qbits.jobs.JobsCoreContract.AUDIT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.BYTES_DONE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.BYTES_TOTAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_DONE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_FAILED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_TOTAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED_WITH_ERRORS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CURRENT_PHASE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.DONE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.DRAINING;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ERROR_MESSAGE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ETA_SECONDS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.FAILED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.FINISHED_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.HEARTBEAT_AT;
@@ -99,16 +93,12 @@ import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_WORKER;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LABEL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LAST_PROGRESS_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LOG_DROPPED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.MESSAGE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW_FIELD;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PENDING;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PERCENT_DONE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.POD_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PRIORITY;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PROCESS_NAME;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PROGRESS_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PROGRESS_MESSAGE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.QUEUED_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.QUEUED_STATUSES;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_DONE;
@@ -153,7 +143,7 @@ public class JobsWidgetRenderer extends AbstractWidgetRenderer
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
DurableJobsSecurity.ensureSessionKeys(QContext.getQSession());
JobsSecurity.ensureSessionKeys(QContext.getQSession());
Instant now = Instant.now();
QWidgetData data = switch(input.getWidgetMetaData().getName())
{
@@ -177,7 +167,7 @@ public class JobsWidgetRenderer extends AbstractWidgetRenderer
*******************************************************************************/
public static List<QRecord> query(String table, QQueryFilter filter) throws QException
{
DurableJobsSecurity.ensureSessionKeys(QContext.getQSession());
JobsSecurity.ensureSessionKeys(QContext.getQSession());
return (new QueryAction().execute(new QueryInput().withTableName(table).withFilter(filter)
.withShouldFetchHeavyFields(true)).getRecords());
}
@@ -315,13 +305,6 @@ public class JobsWidgetRenderer extends AbstractWidgetRenderer
progress.withBlock(text("Unknown total", null, null));
}
progress.withBlock(small(rowText + " rows"));
for(String detail : progressDetails(job))
{
progress.withBlock(small(detail));
}
row.put("Phase", phaseLabel(job));
row.put("Message", job.getValueString(PROGRESS_MESSAGE));
row.put("Bytes", bytes(job));
row.put(JOB_CELL, jobCell(job, null));
row.put(PROGRESS_CELL, progress);
@@ -515,7 +498,7 @@ public class JobsWidgetRenderer extends AbstractWidgetRenderer
rowStats.add(statistic("Total", number(job, ROWS_TOTAL), null));
}
MultiStatisticsData.StatisticsGroupData rows = new MultiStatisticsData.StatisticsGroupData().withHeader("Rows")
.withSubheader(percentValue(job) == null ? "Total not known yet" : percent + " done").withStatisticList(rowStats);
.withSubheader(job.getValue(ROWS_TOTAL) == null ? "Total not known yet" : percent + " done").withStatisticList(rowStats);
List<MultiStatisticsData.StatisticsGroupData.Statistic> chunkStats = new ArrayList<>();
chunkStats.add(statistic("Done", number(job, CHUNKS_DONE), null));
@@ -531,7 +514,7 @@ public class JobsWidgetRenderer extends AbstractWidgetRenderer
{
speedNote = statusLabel(status) + " " + JobsFormat.time(job.getValueInstant(FINISHED_AT), now);
}
else if(job.getValue(ETA_SECONDS) != null || RUNNING_STATUSES.contains(status))
else if(RUNNING_STATUSES.contains(status))
{
speedNote = "ETA " + eta(job, now);
}
@@ -542,17 +525,6 @@ public class JobsWidgetRenderer extends AbstractWidgetRenderer
MultiStatisticsData.StatisticsGroupData speed = new MultiStatisticsData.StatisticsGroupData().withHeader("Speed").withSubheader(speedNote)
.withStatisticList(List.of(statistic("Rows per minute", Math.round(rate(job, now)), null)));
List<MultiStatisticsData.StatisticsGroupData> groups = new ArrayList<>(List.of(rows, chunks, speed));
List<String> details = progressDetails(job);
if(!details.isEmpty())
{
groups.addFirst(new MultiStatisticsData.StatisticsGroupData().withHeader("Progress")
.withSubheader(String.join(" | ", details)).withStatisticList(List.of()));
}
if(number(job, LOG_DROPPED) > 0)
{
groups.add(new MultiStatisticsData.StatisticsGroupData().withHeader("Log")
.withSubheader(JobsFormat.count(number(job, LOG_DROPPED)) + " earlier lines not kept").withStatisticList(List.of()));
}
if(QContext.getQInstance().getTable(JOBS_EFFECT) != null)
{
List<QRecord> effects = query(JOBS_EFFECT, new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.EQUALS, job.getValue(ID)));
@@ -788,69 +760,10 @@ public class JobsWidgetRenderer extends AbstractWidgetRenderer
/*******************************************************************************
** Resolve the phase label from the optional snapshot; old or malformed data
** still displays the reported phase name.
*******************************************************************************/
private static String phaseLabel(QRecord job)
{
String current = job.getValueString(CURRENT_PHASE);
String json = job.getValueString(PROGRESS_JSON);
if(current != null && json != null)
{
try
{
JsonNode snapshot = JsonUtils.toObject(json, JsonNode.class);
for(JsonNode phase : snapshot == null ? List.<JsonNode>of() : snapshot.path("phases"))
{
if(current.equals(phase.path("name").asText()) && !phase.path("label").asText().isBlank())
{
return (phase.path("label").asText());
}
}
}
catch(IOException ignored)
{
// The denormalized progress remains usable without a valid snapshot.
}
}
return (current);
}
/*******************************************************************************
** Optional byte counts, retaining unknown totals rather than inventing zero.
*******************************************************************************/
private static String bytes(QRecord job)
{
String done = job.getValue(BYTES_DONE) == null ? null : JobsFormat.bytes(number(job, BYTES_DONE));
String total = job.getValue(BYTES_TOTAL) == null ? null : JobsFormat.bytes(number(job, BYTES_TOTAL));
return (total == null ? done : done == null ? total + " total" : done + " / " + total);
}
/*******************************************************************************
** Plain typed text shared by the home and record progress views.
*******************************************************************************/
private static List<String> progressDetails(QRecord job)
{
List<String> details = new ArrayList<>();
for(String detail : new String[] {phaseLabel(job), job.getValueString(PROGRESS_MESSAGE), bytes(job)})
{
if(detail != null && !detail.isBlank())
{
details.add(detail);
}
}
return (details);
}
/*******************************************************************************
** Explicit percent, else rows then chunks; null when no total is known.
** Percent done from rows, else from chunks; null when neither total is known.
*******************************************************************************/
private static BigDecimal percentValue(QRecord job)
{
if(job.getValue(PERCENT_DONE) != null)
{
return (BigDecimal.valueOf(Math.max(0, Math.min(100, number(job, PERCENT_DONE)))));
}
long total = number(job, ROWS_TOTAL);
long done = number(job, ROWS_DONE);
if(total <= 0)
@@ -892,10 +805,6 @@ public class JobsWidgetRenderer extends AbstractWidgetRenderer
*******************************************************************************/
private static String eta(QRecord job, Instant now)
{
if(job.getValue(ETA_SECONDS) != null)
{
return ("in " + JobsFormat.span(number(job, ETA_SECONDS)));
}
double rate = rate(job, now);
if(!RUNNING_STATUSES.contains(job.getValueString(STATUS)) || rate <= 0 || job.getValue(ROWS_TOTAL) == null)
{
@@ -34,18 +34,18 @@ import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
@@ -137,9 +137,9 @@ public class BaseJobsTest
instance = new QInstance();
instance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK));
new DurableJobsMetaDataProvider().defineAll(instance, "memory", null);
instance.withSupplementalMetaData(new DurableJobsMetaData());
new AuditsMetaDataProvider().defineAll(instance, "memory", null);
JobsTablesStandIn.defineAll(instance);
instance.addTable(new QTableMetaData().withName(AUDIT).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.INTEGER)).withField(new QFieldMetaData(MESSAGE, QFieldType.STRING)));
config = new JobsQBitConfig();
new JobsQBitProducer().withQBitConfig(config).produce(instance);
new QInstanceEnricher(instance).enrich();
@@ -1,109 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.jobs;
import java.util.List;
import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** The Jobs app and its home widgets are hidden from users without
** "jobs.hasAccess"; Workers additionally needs jobs_admin.
*******************************************************************************/
class JobsAppPermissionTest extends BaseJobsTest
{
private static final List<String> NON_ADMIN_WIDGETS = JobsQBitProducer.HOME_WIDGETS.stream().filter(w -> !"jobsHomeWorkers".equals(w)).toList();
/*******************************************************************************
** The permission name is public and follows the configured app name.
*******************************************************************************/
@Test
void permissionNameAndRules()
{
assertThat(config.getHasAccessPermissionName()).isEqualTo("jobs.hasAccess");
assertThat(new JobsQBitConfig().withAppName("bg").getHasAccessPermissionName()).isEqualTo("bg.hasAccess");
assertThat(instance.getApp("jobs").getPermissionRules().getLevel()).isEqualTo(PermissionLevel.HAS_ACCESS_PERMISSION);
assertThat(instance.getApp("jobs").getPermissionRules().getPermissionBaseName()).isEqualTo("jobs");
assertThat(instance.getApp("jobs").getPermissionRules().getDenyBehavior()).isEqualTo(DenyBehavior.HIDDEN);
assertThat(instance.getWidget("jobsJobProgress").getPermissionRules().getLevel()).isNotEqualTo(PermissionLevel.HAS_ACCESS_PERMISSION);
for(String name : NON_ADMIN_WIDGETS)
{
assertThat(instance.getWidget(name).getPermissionRules().getPermissionBaseName()).isEqualTo("jobs");
assertThat(instance.getWidget(name).getPermissionRules().getDenyBehavior()).isEqualTo(DenyBehavior.HIDDEN);
}
}
/*******************************************************************************
** No permissions: no app, no home widgets, every render denied.
*******************************************************************************/
@Test
void noPermissionsSeesNothing() throws Exception
{
instance.setEnforcePermissionsInCoreActions(true);
login("nobody", false);
MetaDataOutput output = new MetaDataAction().execute(new MetaDataInput());
assertThat(output.getApps()).doesNotContainKey("jobs");
assertThat(output.getWidgets()).doesNotContainKeys(JobsQBitProducer.HOME_WIDGETS.toArray(String[]::new));
for(String name : JobsQBitProducer.HOME_WIDGETS)
{
assertThatThrownBy(() -> render(name, null)).isInstanceOf(QPermissionDeniedException.class);
}
}
/*******************************************************************************
** jobs.hasAccess: app and the five widgets, Workers only with jobs_admin.
*******************************************************************************/
@Test
void hasAccessSeesAppAndWidgets() throws Exception
{
instance.setEnforcePermissionsInCoreActions(true);
login("user", false).withPermission(config.getHasAccessPermissionName());
MetaDataOutput output = new MetaDataAction().execute(new MetaDataInput());
assertThat(output.getApps()).containsKey("jobs");
assertThat(output.getWidgets()).containsKeys(NON_ADMIN_WIDGETS.toArray(String[]::new));
assertThat(output.getWidgets()).doesNotContainKey("jobsHomeWorkers");
for(String name : NON_ADMIN_WIDGETS)
{
assertThat(render(name, null)).isNotNull();
}
assertThatThrownBy(() -> render("jobsHomeWorkers", null)).isInstanceOf(QPermissionDeniedException.class);
login("admin", true).withPermission(config.getHasAccessPermissionName());
assertThat(new MetaDataAction().execute(new MetaDataInput()).getWidgets()).containsKey("jobsHomeWorkers");
assertThat(render("jobsHomeWorkers", null)).isNotNull();
}
}
@@ -1,290 +0,0 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.jobs;
import java.time.temporal.ChronoUnit;
import java.util.List;
import com.kingsrook.qbits.jobs.processes.JobsPurgeFinishedStep;
import com.kingsrook.qbits.jobs.widgets.JobsFormat;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import org.junit.jupiter.api.Test;
import static com.kingsrook.qbits.jobs.JobsCoreContract.BYTES_DONE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.BYTES_TOTAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_TOTAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CURRENT_PHASE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ETA_SECONDS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_LOG;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_USER;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LEVEL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LOGGED_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LOG_DROPPED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LOG_SEQ;
import static com.kingsrook.qbits.jobs.JobsCoreContract.MESSAGE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PERCENT_DONE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PHASE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PROGRESS_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PROGRESS_MESSAGE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_TOTAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.RUNNING;
import static com.kingsrook.qbits.jobs.JobsCoreContract.USER_ID;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
/*******************************************************************************
** Optional RC.8 progress and logs, including the actual core log permission checker.
*******************************************************************************/
class JobsProgressTest extends BaseJobsTest
{
/*******************************************************************************
** Add optional fields and a log using the real kernel permission checker by
** name, preserving the default suite's coverage of older kernel metadata.
*******************************************************************************/
private void enableProgress() throws Exception
{
for(String field : List.of(CURRENT_PHASE, PROGRESS_MESSAGE, PROGRESS_JSON))
{
instance.getTable(JOBS_JOB).withField(new QFieldMetaData(field, QFieldType.TEXT));
}
for(String field : List.of(PERCENT_DONE, LOG_DROPPED))
{
instance.getTable(JOBS_JOB).withField(new QFieldMetaData(field, QFieldType.INTEGER));
}
for(String field : List.of(BYTES_DONE, BYTES_TOTAL, ETA_SECONDS, LOG_SEQ))
{
instance.getTable(JOBS_JOB).withField(new QFieldMetaData(field, QFieldType.LONG));
}
instance.setEnforcePermissionsInCoreActions(true);
//////////////////////////////////////////////////////////////////////
// core 4.1.0-RC.8 defines jobs_log; older metadata gets this mirror //
//////////////////////////////////////////////////////////////////////
if(instance.getTable(JOBS_LOG) != null)
{
return;
}
instance.addTable(new QTableMetaData().withName(JOBS_LOG).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.LONG))
.withField(new QFieldMetaData(JOB_ID, QFieldType.INTEGER))
.withField(new QFieldMetaData(LEVEL, QFieldType.STRING))
.withField(new QFieldMetaData(LOGGED_AT, QFieldType.DATE_TIME))
.withField(new QFieldMetaData(PHASE, QFieldType.STRING))
.withField(new QFieldMetaData(MESSAGE, QFieldType.STRING))
.withPermissionRules(new QPermissionRules().withCustomPermissionChecker(new QCodeReference(
Class.forName("com.kingsrook.qqq.backend.core.jobs.progress.JobLogPermissions"))))
.withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(JOBS_USER).withFieldName(JOBS_JOB + "." + USER_ID)
.withJoinNameChain(List.of(JobsCoreContract.childJoin(JOBS_LOG)))));
instance.addJoin(new QJoinMetaData().withLeftTable(JOBS_JOB).withRightTable(JOBS_LOG).withType(JoinType.ONE_TO_MANY)
.withJoinOn(new JoinOn(ID, JOB_ID)).withInferredName());
clearPresentation();
new JobsQBitProducer().withQBitConfig(config).produce(instance);
new QInstanceEnricher(instance).enrich();
}
/*******************************************************************************
** Rich fields override counters, resolve labels, and remain typed and flexible.
*******************************************************************************/
@Test
void richProgressOnHomeAndRecord() throws Exception
{
enableProgress();
QRecord job = job(USER_A, RUNNING, null);
update(job, new QRecord().withValue(CURRENT_PHASE, "upload").withValue(PERCENT_DONE, 73)
.withValue(PROGRESS_MESSAGE, "Uploading manifest").withValue(BYTES_DONE, 1536L).withValue(BYTES_TOTAL, 1048576L)
.withValue(ETA_SECONDS, 90L).withValue(LOG_DROPPED, 12)
.withValue(PROGRESS_JSON, "{\"phases\":[{\"name\":\"upload\",\"label\":\"Upload backup\"}]}"));
login(USER_A, false).withPermission(config.getHasAccessPermissionName());
TableData home = (TableData) render("jobsHomeActive", null);
assertThat(home.getRows()).singleElement().satisfies(row -> assertThat(row)
.containsEntry("Progress", "73%").containsEntry("Phase", "Upload backup")
.containsEntry("Message", "Uploading manifest").containsEntry("Bytes", "1.5 KiB / 1.0 MiB").containsEntry("ETA", "in 1m"));
assertThat(home.getColumns()).hasSize(3).allSatisfy(column -> assertThat(column.getWidth()).isNull());
MultiStatisticsData record = (MultiStatisticsData) render("jobsJobProgress", job.getValueString(ID));
assertThat(record.getStatisticsGroupData()).extracting(MultiStatisticsData.StatisticsGroupData::getSubheader)
.contains("Upload backup | Uploading manifest | 1.5 KiB / 1.0 MiB", "73% done", "ETA in 1m", "12 earlier lines not kept");
login(USER_B, false).withPermission(config.getHasAccessPermissionName());
assertThat(((TableData) render("jobsHomeActive", null)).getRows()).isEmpty();
assertThat(((MultiStatisticsData) render("jobsJobProgress", job.getValueString(ID))).getStatisticsGroupData()).isEmpty();
}
/*******************************************************************************
** Null percent uses rows then chunks, zero is explicit, malformed JSON is safe.
*******************************************************************************/
@Test
void fallbacksAndPartialProgress() throws Exception
{
enableProgress();
QRecord job = job(USER_A, RUNNING, null);
update(job, new QRecord().withValue(CURRENT_PHASE, "copy").withValue(PROGRESS_JSON, "bad json").withValue(BYTES_TOTAL, 1024L));
assertThat(((TableData) render("jobsHomeActive", null)).getRows().getFirst())
.containsEntry("Progress", "45%").containsEntry("Phase", "copy").containsEntry("Bytes", "1.0 KiB total");
update(job, new QRecord().withValue(ROWS_TOTAL, null).withValue(PROGRESS_JSON, "null"));
assertThat(((TableData) render("jobsHomeActive", null)).getRows().getFirst()).containsEntry("Progress", "40%");
update(job, new QRecord().withValue(PERCENT_DONE, 0).withValue(BYTES_TOTAL, null).withValue(BYTES_DONE, 0L));
assertThat(((TableData) render("jobsHomeActive", null)).getRows().getFirst()).containsEntry("Progress", "0%").containsEntry("Bytes", "0 B");
update(job, new QRecord().withValue(PERCENT_DONE, null).withValue(CHUNKS_TOTAL, null));
assertThat(((TableData) render("jobsHomeActive", null)).getRows().getFirst()).containsEntry("Progress", "Unknown");
assertThat(JobsFormat.bytes(1073741824L)).isEqualTo("1.0 GiB");
assertThat(JobsFormat.bytes(Long.MAX_VALUE)).isEqualTo("8.0 EiB");
}
/*******************************************************************************
** The log is ordered, full width and owner scoped, with no user mutations.
*******************************************************************************/
@Test
void logPanelAndPermissions() throws Exception
{
enableProgress();
QRecord job = job(USER_A, RUNNING, null);
log(job, 1000000002L);
log(job, 1000000001L);
assertThat(instance.getWidget("jobsJobLog").getGridColumns()).isEqualTo(12);
assertThat(instance.getTable(JOBS_JOB).getSections()).extracting(QFieldSection::getWidgetName)
.containsSubsequence("jobsJobProgress", "jobsJobLog", "jobsJobEvents");
login(USER_A, false).withPermission("jobs_job.read");
ChildRecordListData data = (ChildRecordListData) render("jobsJobLog", job.getValueString(ID));
assertThat(data.getQueryOutput().getRecords()).extracting(record -> record.getValueLong(ID)).containsExactly(1000000002L, 1000000001L);
assertThat(data.getOnlyIncludeFieldNames()).containsExactly(LEVEL, LOGGED_AT, PHASE, MESSAGE);
assertThat(data.getCanAddChildRecord()).isFalse();
assertThatThrownBy(() -> new DeleteAction().execute(new DeleteInput().withTableName(JOBS_LOG).withPrimaryKey(1000000001L))).hasMessageContaining("read-only");
assertThatThrownBy(() -> log(job, 1000000003L)).hasMessageContaining("read-only");
login("admin", true).withPermission("jobs_job.read");
assertThatThrownBy(() -> new UpdateAction().execute(new UpdateInput().withTableName(JOBS_LOG)
.withRecord(new QRecord().withValue(ID, 1000000001L).withValue(MESSAGE, "changed")))).hasMessageContaining("read-only");
login(USER_B, false).withPermission("jobs_job.read");
assertThatThrownBy(() -> render("jobsJobLog", job.getValueString(ID))).hasMessageContaining("Could not find");
}
/*******************************************************************************
** The system purge crosses a full batch with LONG IDs; active logs remain.
*******************************************************************************/
@Test
void purgeLogBatchesWithSystemPermission() throws Exception
{
enableProgress();
QRecord old = job(USER_A, COMPLETED, now.minus(31, ChronoUnit.DAYS));
QRecord active = job(USER_B, RUNNING, null);
for(long id = 1000000000L; id < 1000000205L; id++)
{
log(old, id);
}
log(active, 1000000300L);
QRecord held = job(USER_A, COMPLETED, now.minus(31, ChronoUnit.DAYS));
effect(held, JobsCoreContract.PENDING, false);
log(held, 1000000301L);
var caller = login("admin", true);
new JobsPurgeFinishedStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
assertThat(QContext.getQSession()).isSameAs(caller);
QContext.setQSession(new QSystemUserSession());
assertThat(records(JOBS_LOG)).extracting(record -> record.getValue(JOB_ID)).containsExactlyInAnyOrder(active.getValue(ID), held.getValue(ID));
assertThat(records(JOBS_JOB)).extracting(record -> record.getValue(ID)).containsExactlyInAnyOrder(active.getValue(ID), held.getValue(ID));
}
/*******************************************************************************
** Log metadata is optional; hosts choose the Admin/Operations parent.
*******************************************************************************/
@Test
void optionalLogAndHostNavigation() throws Exception
{
assertThat(instance.getWidget("jobsJobLog")).as("core RC.8 defines jobs_log, so the Log list is produced").isNotNull();
enableProgress();
instance.getJoins().remove(JobsCoreContract.childJoin(JOBS_LOG));
instance.getWidgets().remove("jobsJobLog");
clearPresentation();
new JobsQBitProducer().produce(instance);
assertThat(instance.getWidget("jobsJobLog")).isNull();
QAppMetaData admin = new QAppMetaData().withName("admin").withLabel("Admin");
QAppMetaData operations = new QAppMetaData().withName("operations").withLabel("Operations");
operations.setParentAppName("admin");
instance.addApp(admin.withChild(operations));
instance.addApp(operations);
config.withAppName("backgroundJobs").withParentAppName("operations");
clearPresentation();
new JobsQBitProducer().withQBitConfig(config).produce(instance);
assertThat(instance.getApp("backgroundJobs").getParentAppName()).isEqualTo("operations");
assertThat(operations.getChildren()).extracting(child -> child.getName()).containsExactly("backgroundJobs");
config.setParentAppName("missing");
assertThatThrownBy(() -> config.validate(instance)).hasMessageContaining("parentAppName");
config.setParentAppName("backgroundJobs");
assertThatThrownBy(() -> config.validate(instance)).hasMessageContaining("parentAppName");
config.setParentAppName(null);
config.validate(instance);
}
/*******************************************************************************
** Rebuild presentation against changed optional metadata in these tests.
*******************************************************************************/
private void clearPresentation()
{
instance.getQBits().clear();
instance.getWidgets().clear();
instance.getApps().remove("jobs");
instance.getProcesses().remove(JobsQBitProducer.PURGE_PROCESS);
instance.getTable(JOBS_JOB).getSections().removeIf(section -> section.getWidgetName() != null);
}
/*******************************************************************************
** Persist progress through core actions.
*******************************************************************************/
private void update(QRecord job, QRecord values) throws Exception
{
var output = new UpdateAction().execute(new UpdateInput().withTableName(JOBS_JOB).withRecord(values.withValue(ID, job.getValue(ID))));
assertThat(output.getRecords().getFirst().getErrors()).isNullOrEmpty();
}
/*******************************************************************************
** Explicit LONG cursors make ordering assertions deterministic. Core memory
** sequence bookkeeping currently restricts inserts to the integer range.
*******************************************************************************/
private void log(QRecord job, long id) throws Exception
{
insert(JOBS_LOG, new QRecord().withValue(ID, id).withValue(JOB_ID, job.getValue(ID)).withValue(LEVEL, "INFO")
.withValue(LOGGED_AT, now).withValue(PHASE, "upload").withValue(MESSAGE, "Wrote manifest"));
}
}
@@ -30,8 +30,6 @@ import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
@@ -66,17 +64,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
*******************************************************************************/
class JobsQBitProducerTest extends BaseJobsTest
{
/*******************************************************************************
** The real kernel configuration passes startup validation without waivers.
*******************************************************************************/
@Test
void durableJobsMetadataValidates()
{
QInstanceValidator validator = new QInstanceValidator();
DurableJobsMetaData.of(instance).validate(instance, validator);
assertThat(validator.getErrors()).isEmpty();
}
/*******************************************************************************
** Fail before registering anything when the kernel has not been enabled.
*******************************************************************************/
@@ -97,13 +84,13 @@ class JobsQBitProducerTest extends BaseJobsTest
@Test
void navigationWidgetsAndRecordSections()
{
assertThat(instance.getTables()).hasSize(12);
assertThat(instance.getTables()).hasSize(8);
assertThat(instance.getApp("jobs").getChildren()).extracting(child -> child.getName()).containsExactly(JOBS_JOB);
assertThat(instance.getApp("jobs").getWidgets()).containsExactlyElementsOf(JobsQBitProducer.HOME_WIDGETS);
assertThat(instance.getWidgets()).hasSize(13);
assertThat(instance.getWidgets()).hasSize(12);
assertThat(instance.getWidgets().values()).allSatisfy(widget -> assertThat(widget.getType()).isNotEqualTo(WidgetType.HTML.getType()));
assertThat(instance.getTable(JOBS_JOB).getSections()).extracting(QFieldSection::getWidgetName)
.containsSubsequence("jobsJobProgress", "jobsJobLog", "jobsJobChunks", "jobsJobRowErrors", "jobsJobEffects", "jobsJobEvents", "jobsJobAudit");
.containsSubsequence("jobsJobProgress", "jobsJobChunks", "jobsJobRowErrors", "jobsJobEffects", "jobsJobEvents", "jobsJobAudit");
for(String name : JobsQBitProducer.HOME_WIDGETS)
{
assertThat(instance.getWidget(name).getDefaultValues()).containsEntry("refreshSeconds", 5);
@@ -116,7 +103,7 @@ class JobsQBitProducerTest extends BaseJobsTest
assertThat(metadata.getGroupId()).isEqualTo("com.kingsrook.qbits");
assertThat(metadata.getArtifactId()).isEqualTo("qbit-jobs");
assertThat(metadata.getNamespace()).isEqualTo("jobs");
assertThat(metadata.getVersion()).matches("\\d+\\.\\d+\\.\\d+(-SNAPSHOT|-RC\\.\\d+)?");
assertThat(metadata.getVersion()).isEqualTo("0.1.0-SNAPSHOT");
}
/*******************************************************************************
@@ -126,13 +113,12 @@ class JobsQBitProducerTest extends BaseJobsTest
void schedulerAndAuditAreOptional() throws Exception
{
QInstance other = new QInstance();
new DurableJobsMetaDataProvider().defineAll(other, "memory", null);
other.withSupplementalMetaData(new DurableJobsMetaData());
JobsTablesStandIn.defineAll(other);
JobsQBitConfig custom = new JobsQBitConfig().withAppName("backgroundJobs").withSchedulerName("scheduler")
.withPurgeCronExpression("0 0 4 * * ?").withHomeRefreshSeconds(9).withRecordRefreshSeconds(8);
new JobsQBitProducer().withQBitConfig(custom).produce(other);
assertThat(other.getWidget("jobsJobAudit")).isNull();
assertThat(other.getWidgets()).hasSize(12);
assertThat(other.getWidgets()).hasSize(11);
assertThat(other.getApp("backgroundJobs")).isNotNull();
QScheduleMetaData schedule = other.getProcess(JobsQBitProducer.PURGE_PROCESS).getSchedule();
assertThat(schedule.getSchedulerName()).isEqualTo("scheduler");
@@ -225,20 +211,18 @@ class JobsQBitProducerTest extends BaseJobsTest
}
/*******************************************************************************
** Hosts can omit the effects panel by removing its table or join.
** Older kernels need neither the outbox nor its join to register the app.
*******************************************************************************/
@Test
void effectsTableAndJoinAreOptional() throws Exception
{
QInstance withoutTable = new QInstance();
new DurableJobsMetaDataProvider().defineAll(withoutTable, "memory", null);
withoutTable.withSupplementalMetaData(new DurableJobsMetaData());
JobsTablesStandIn.defineAll(withoutTable);
withoutTable.getTables().remove(JOBS_EFFECT);
new JobsQBitProducer().produce(withoutTable);
assertThat(withoutTable.getWidget("jobsJobEffects")).isNull();
QInstance withoutJoin = new QInstance();
new DurableJobsMetaDataProvider().defineAll(withoutJoin, "memory", null);
withoutJoin.withSupplementalMetaData(new DurableJobsMetaData());
JobsTablesStandIn.defineAll(withoutJoin);
withoutJoin.getJoins().remove(JobsCoreContract.childJoin(JOBS_EFFECT));
new JobsQBitProducer().produce(withoutJoin);
assertThat(withoutJoin.getWidget("jobsJobEffects")).isNull();
@@ -248,7 +232,7 @@ class JobsQBitProducerTest extends BaseJobsTest
** Outbox metadata carries the core field defaults, references and labels.
*******************************************************************************/
@Test
void effectsCoreContract() throws Exception
void effectsStandInContract() throws Exception
{
assertThat(instance.getTable(JOBS_EFFECT).getFields()).hasSize(26);
assertThat(instance.getTable(JOBS_EFFECT).getField(JOB_ID).getPossibleValueSourceName()).isEqualTo(JOBS_JOB);
@@ -30,16 +30,12 @@ import com.kingsrook.qbits.jobs.processes.JobsStorage;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
@@ -91,19 +87,18 @@ class JobsRetentionTest extends BaseJobsTest
children(job(USER_B, RUNNING, now.minus(40, ChronoUnit.DAYS)));
children(job(USER_A, COMPLETED, now.minus(2, ChronoUnit.DAYS)));
children(job(USER_A, NEEDS_REVIEW, now.minus(40, ChronoUnit.DAYS)));
List<Integer> auditIds = records(AUDIT).stream().map(record -> record.getValueInteger(ID)).toList();
RecordingStorage storage = new RecordingStorage();
QSession caller = login("admin", true);
List<ProcessSummaryLine> summary = run(new JobsPurgeFinishedStep(storage));
assertThat(QContext.getQSession()).isSameAs(caller);
assertThat(summary).extracting(ProcessSummaryLine::getCount).containsExactly(4, 4, 4);
DurableJobsSecurity.ensureSessionKeys(caller);
JobsSecurity.ensureSessionKeys(caller);
assertThat(records(JOBS_JOB)).hasSize(3).extracting(record -> record.getValueString(STATUS)).containsExactly(RUNNING, COMPLETED, NEEDS_REVIEW);
for(String table : List.of(JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EVENT, JOBS_UPLOAD))
{
assertThat(records(table)).hasSize(3);
}
assertThat(records(AUDIT)).extracting(record -> record.getValueInteger(ID)).containsAll(auditIds);
assertThat(records(AUDIT)).hasSize(7);
assertThat(storage.deleted).hasSize(8);
assertThat(run(new JobsPurgeFinishedStep(storage))).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 0);
}
@@ -160,30 +155,20 @@ class JobsRetentionTest extends BaseJobsTest
}
/*******************************************************************************
** Core storage deletes input objects before retention clears their references.
** RC.5 safely skips unavailable object deletion and preserves retry references.
*******************************************************************************/
@Test
void releasedCoreDeletesObjects() throws Exception
void releasedCoreSkipsObjectDeletion() throws Exception
{
QRecord clean = job(USER_A, COMPLETED, now.minus(8, ChronoUnit.DAYS));
clearFailures(clean);
children(clean);
instance.addTable(new QTableMetaData().withName("objects").withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.STRING)).withField(new QFieldMetaData("data", QFieldType.BLOB)));
instance.withSupplementalMetaData(new DurableJobsMetaData().withStorageTableName("objects"));
QSession caller = QContext.getQSession();
assertThatThrownBy(() -> run(new JobsPurgeFinishedStep())).isInstanceOf(QException.class);
assertThat(QContext.getQSession()).isSameAs(caller);
instance.withSupplementalMetaData(new StorageMetadata());
assertThat(run(new JobsPurgeFinishedStep())).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 0);
assertThat(records(JOBS_CHUNK).getFirst().getValue(INPUT_STORAGE_REF)).isNotNull();
insert("objects", new QRecord().withValue(ID, "chunk/" + clean.getValue(ID)).withValue("data", new byte[] {1}));
assertThat(run(new JobsPurgeFinishedStep())).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 1);
assertThat(records(JOBS_CHUNK).getFirst().getValue(INPUT_STORAGE_REF)).isNull();
JobsStorage storage = new JobsStorage();
assertThat(storage.chunkStorageTable()).isEqualTo("objects");
assertThat(records("objects")).isEmpty();
assertThatThrownBy(() -> storage.delete("objects", "missing-object")).isInstanceOf(QException.class);
assertThat(storage.delete("", "object")).isFalse();
assertThat(storage.delete("objects", " ")).isFalse();
assertThat(storage.delete("objects", "another-object")).isFalse();
assertThat(storage.delete(null, "object")).isFalse();
assertThat(storage.delete("objects", null)).isFalse();
}
@@ -200,7 +185,7 @@ class JobsRetentionTest extends BaseJobsTest
storage.fail = true;
assertThatThrownBy(() -> run(new JobsPurgeFinishedStep(storage))).hasMessageContaining("storage unavailable");
assertThat(QContext.getQSession()).isSameAs(caller);
DurableJobsSecurity.ensureSessionKeys(caller);
JobsSecurity.ensureSessionKeys(caller);
assertThat(records(JOBS_JOB)).hasSize(1);
assertThat(records(JOBS_CHUNK)).hasSize(1);
storage.fail = false;
@@ -224,13 +209,12 @@ class JobsRetentionTest extends BaseJobsTest
@Test
void absentStorageMetadata() throws Exception
{
instance.getSupplementalMetaData().remove(DURABLE_JOBS_METADATA);
assertThat(new JobsStorage().chunkStorageTable()).isNull();
assertThat(run(new JobsPurgeFinishedStep())).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 0);
}
/*******************************************************************************
** Unsettled effects preserve every child and input; settled effects are paged away.
** Outstanding effects preserve every child and input; settled effects are paged away.
*******************************************************************************/
@Test
void effectsBlockPurgeUntilSettled() throws Exception
@@ -259,8 +243,8 @@ class JobsRetentionTest extends BaseJobsTest
List<ProcessSummaryLine> summary = run(new JobsPurgeFinishedStep(storage));
assertThat(QContext.getQSession()).isSameAs(caller);
assertThat(summary).extracting(ProcessSummaryLine::getCount).containsExactly(1, 1, 1, 4);
assertThat(summary.getLast().getMessage()).isEqualTo("jobs kept: effects still unsettled");
DurableJobsSecurity.ensureSessionKeys(caller);
assertThat(summary.getLast().getMessage()).isEqualTo("jobs kept: effects still outstanding");
JobsSecurity.ensureSessionKeys(caller);
assertThat(records(JOBS_JOB)).extracting(record -> record.getValue(ID)).containsExactlyElementsOf(kept.stream().map(record -> record.getValue(ID)).toList());
assertThat(records(JOBS_EFFECT)).hasSize(4);
for(String table : List.of(JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EVENT, JOBS_UPLOAD))
@@ -280,7 +264,7 @@ class JobsRetentionTest extends BaseJobsTest
}
/*******************************************************************************
** Retention tolerates a host omitting the effects table.
** Effects remain optional for retention against older cores.
*******************************************************************************/
@Test
void purgeWithoutEffectsTable() throws Exception
@@ -312,7 +296,7 @@ class JobsRetentionTest extends BaseJobsTest
}
/*******************************************************************************
** Test storage records deletions and simulates failures without using disk.
** Test storage models successful RC.6 object deletion, without using disk.
*******************************************************************************/
private static class RecordingStorage extends JobsStorage
{
@@ -348,4 +332,26 @@ class JobsRetentionTest extends BaseJobsTest
}
}
/*******************************************************************************
** Test-only supplemental metadata for the RC.5 reflection bridge.
*******************************************************************************/
public static class StorageMetadata implements QSupplementalInstanceMetaData
{
/*******************************************************************************
** Core metadata contract name.
*******************************************************************************/
@Override
public String getName()
{
return (DURABLE_JOBS_METADATA);
}
/*******************************************************************************
** Future core getter discovered reflectively by the adapter.
*******************************************************************************/
public String getStorageTableName()
{
return ("objects");
}
}
}
@@ -22,7 +22,6 @@
package com.kingsrook.qbits.jobs;
import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
@@ -46,13 +45,13 @@ class JobsSecurityTest
{
QSession session = new QSession().withUser(new QUser().withIdReference("user"))
.withSecurityKeyValue(JOBS_USER, "wrong").withSecurityKeyValue(JOBS_USER_ALL_ACCESS, true);
DurableJobsSecurity.applyToSession(session);
JobsSecurity.applyToSession(session);
assertThat(session.getSecurityKeyValues(JOBS_USER)).containsExactly("user");
assertThat(session.getSecurityKeyValues(JOBS_USER_ALL_ACCESS)).isEmpty();
session.withPermission(JOBS_ADMIN);
DurableJobsSecurity.applyToSession(session);
JobsSecurity.applyToSession(session);
assertThat(session.getSecurityKeyValues(JOBS_USER_ALL_ACCESS)).containsExactly(true);
DurableJobsSecurity.ensureSessionKeys(session);
JobsSecurity.ensureSessionKeys(session);
assertThat(session.getSecurityKeyValues(JOBS_USER)).containsExactly("user");
}
@@ -62,15 +61,15 @@ class JobsSecurityTest
@Test
void nullAnonymousAndSystem()
{
DurableJobsSecurity.applyToSession(null);
DurableJobsSecurity.ensureSessionKeys(null);
DurableJobsSecurity.applyToSession(new QSystemUserSession());
DurableJobsSecurity.ensureSessionKeys(new QSystemUserSession());
assertThat(DurableJobsSecurity.userId(null)).isNull();
JobsSecurity.applyToSession(null);
JobsSecurity.ensureSessionKeys(null);
JobsSecurity.applyToSession(new QSystemUserSession());
JobsSecurity.ensureSessionKeys(new QSystemUserSession());
assertThat(JobsSecurity.userId(null)).isNull();
QSession anonymous = new QSession();
DurableJobsSecurity.ensureSessionKeys(anonymous);
assertThat(DurableJobsSecurity.userId(anonymous)).isNull();
JobsSecurity.ensureSessionKeys(anonymous);
assertThat(JobsSecurity.userId(anonymous)).isNull();
assertThat(anonymous.getSecurityKeyValues(JOBS_USER)).isEmpty();
DurableJobsSecurity.ensureSessionKeys(new QSession().withUser(new QUser().withIdReference("")));
JobsSecurity.ensureSessionKeys(new QSession().withUser(new QUser().withIdReference("")));
}
}
@@ -0,0 +1,350 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2026. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.jobs;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
import com.kingsrook.qqq.backend.core.model.metadata.security.QSecurityKeyType;
import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ACTOR_USER_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ATTEMPT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHECKPOINT_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_DONE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_FAILED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_TOTAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNK_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNK_SEQ;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNK_SIZE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CLIENT_CHANNEL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CONTENT_TYPE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CREATE_DATE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.CREDENTIAL_SCOPE_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.DETAIL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.DRAINING;
import static com.kingsrook.qbits.jobs.JobsCoreContract.EFFECT_NAME;
import static com.kingsrook.qbits.jobs.JobsCoreContract.EFFECT_TYPE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ERROR_CODE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ERROR_MESSAGE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.EXPIRES_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.FAILED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.FIELD_NAME;
import static com.kingsrook.qbits.jobs.JobsCoreContract.FILE_NAME;
import static com.kingsrook.qbits.jobs.JobsCoreContract.FINISHED_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.HEARTBEAT_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.IDEMPOTENCY;
import static com.kingsrook.qbits.jobs.JobsCoreContract.IDEMPOTENCY_KEY;
import static com.kingsrook.qbits.jobs.JobsCoreContract.IDEMPOTENT_EXTERNAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.INPUT_STORAGE_REF;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT_STATUS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT_TYPE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ROW_ERROR;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_UPLOAD;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_USER;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_USER_ALL_ACCESS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_WORKER;
import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LABEL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LAST_ERROR;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LAST_PROGRESS_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LEASE_EXPIRES_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LEASE_OWNER;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LEASE_TOKEN;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LEASE_VERSION;
import static com.kingsrook.qbits.jobs.JobsCoreContract.LOCAL_TX;
import static com.kingsrook.qbits.jobs.JobsCoreContract.MAX_ATTEMPTS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.MAX_PARALLEL_CHUNKS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.MESSAGE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.MODE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.MODIFY_DATE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW_FIELD;
import static com.kingsrook.qbits.jobs.JobsCoreContract.NEXT_ATTEMPT_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.NON_IDEMPOTENT_EXTERNAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.NOTIFY_EMAIL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.OUTPUT_STORAGE_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PARAMS_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PARENT_JOB_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PAYLOAD_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PAYLOAD_REF;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PENDING;
import static com.kingsrook.qbits.jobs.JobsCoreContract.POD;
import static com.kingsrook.qbits.jobs.JobsCoreContract.POD_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PRIORITY;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PROCESS_NAME;
import static com.kingsrook.qbits.jobs.JobsCoreContract.PROVIDER_REF;
import static com.kingsrook.qbits.jobs.JobsCoreContract.QUEUED_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.RANGE_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.RECONCILE_ATTEMPTS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.RECORD_PK;
import static com.kingsrook.qbits.jobs.JobsCoreContract.REF;
import static com.kingsrook.qbits.jobs.JobsCoreContract.RESOLVED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.RETRY_JOB_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_DONE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_FAILED;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_IN;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_OK;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_TOTAL;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROW_KEY;
import static com.kingsrook.qbits.jobs.JobsCoreContract.ROW_NUMBER;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SAVED_BULK_LOAD_PROFILE_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SEMANTICS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SENDING;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SENT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SENT_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SEQ;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SIZE_BYTES;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SOURCE_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SOURCE_TYPE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.STARTED_AT;
import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS_REASON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.STORAGE_REFERENCE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.STORAGE_TABLE_NAME;
import static com.kingsrook.qbits.jobs.JobsCoreContract.SUMMARY_JSON;
import static com.kingsrook.qbits.jobs.JobsCoreContract.TABLE_NAME;
import static com.kingsrook.qbits.jobs.JobsCoreContract.TARGET_NAME;
import static com.kingsrook.qbits.jobs.JobsCoreContract.THREADS;
import static com.kingsrook.qbits.jobs.JobsCoreContract.TYPE;
import static com.kingsrook.qbits.jobs.JobsCoreContract.UNKNOWN;
import static com.kingsrook.qbits.jobs.JobsCoreContract.USER_ID;
import static com.kingsrook.qbits.jobs.JobsCoreContract.VERSION;
/*******************************************************************************
** STAND-IN(core=QRun-IO/qqq#1178, removeBy=RC.6): test-only mirror of the core
** tables, field types, primary keys, owner locks and child joins. Includes uploads
** so retention is exercised against the complete contract.
*******************************************************************************/
public final class JobsTablesStandIn
{
/*******************************************************************************
** No instances.
*******************************************************************************/
private JobsTablesStandIn()
{
}
/*******************************************************************************
** Define the core contract on a memory backend without importing core.jobs.
*******************************************************************************/
public static void defineAll(QInstance instance) throws QException
{
instance.addSecurityKeyType(new QSecurityKeyType().withName(JOBS_USER).withAllAccessKeyName(JOBS_USER_ALL_ACCESS));
instance.addTable(new QTableMetaData().withName(JOBS_JOB).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(LABEL, QFieldType.STRING).withIsEditable(true))
.withField(new QFieldMetaData(PROCESS_NAME, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(TABLE_NAME, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(MODE, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(STATUS, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(STATUS_REASON, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(PRIORITY, QFieldType.INTEGER).withIsEditable(true))
.withField(new QFieldMetaData(USER_ID, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(CLIENT_CHANNEL, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(CREDENTIAL_SCOPE_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(SOURCE_TYPE, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(SOURCE_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(PARAMS_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(IDEMPOTENCY, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(SEMANTICS, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(CHUNK_SIZE, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(MAX_PARALLEL_CHUNKS, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(ROWS_TOTAL, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(ROWS_DONE, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(ROWS_OK, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(ROWS_FAILED, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(CHUNKS_TOTAL, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(CHUNKS_DONE, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(CHUNKS_FAILED, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(SUMMARY_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(OUTPUT_STORAGE_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(PARENT_JOB_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(SAVED_BULK_LOAD_PROFILE_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(NOTIFY_EMAIL, QFieldType.BOOLEAN).withIsEditable(true))
.withField(new QFieldMetaData(QUEUED_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(STARTED_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(FINISHED_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(LAST_PROGRESS_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(ERROR_MESSAGE, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(LEASE_VERSION, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(CREATE_DATE, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(MODIFY_DATE, QFieldType.DATE_TIME).withIsEditable(false)));
instance.addTable(new QTableMetaData().withName(JOBS_CHUNK).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(JOB_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(SEQ, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(STATUS, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(RANGE_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(INPUT_STORAGE_REF, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(ATTEMPT, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(MAX_ATTEMPTS, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(LEASE_OWNER, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(LEASE_TOKEN, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(LEASE_EXPIRES_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(HEARTBEAT_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(ROWS_IN, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(ROWS_OK, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(ROWS_FAILED, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(SUMMARY_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(CHECKPOINT_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(ERROR_MESSAGE, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(STARTED_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(FINISHED_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(LEASE_VERSION, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(CREATE_DATE, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(MODIFY_DATE, QFieldType.DATE_TIME).withIsEditable(false)));
instance.addTable(new QTableMetaData().withName(JOBS_ROW_ERROR).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(JOB_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(CHUNK_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(ROW_NUMBER, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(RECORD_PK, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(FIELD_NAME, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(ERROR_CODE, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(MESSAGE, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(RETRY_JOB_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(RESOLVED, QFieldType.BOOLEAN).withIsEditable(true))
.withField(new QFieldMetaData(CREATE_DATE, QFieldType.DATE_TIME).withIsEditable(false)));
instance.addTable(new QTableMetaData().withName(JOBS_EVENT).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(JOB_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(CHUNK_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(TYPE, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(ACTOR_USER_ID, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(CLIENT_CHANNEL, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(POD, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(DETAIL, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(CREATE_DATE, QFieldType.DATE_TIME).withIsEditable(false)));
instance.addTable(new QTableMetaData().withName(JOBS_WORKER).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(POD_ID, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(STARTED_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(HEARTBEAT_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(THREADS, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(DRAINING, QFieldType.BOOLEAN).withIsEditable(false))
.withField(new QFieldMetaData(VERSION, QFieldType.STRING).withIsEditable(false)));
instance.addTable(new QTableMetaData().withName(JOBS_UPLOAD).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(REF, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(USER_ID, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(FILE_NAME, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(CONTENT_TYPE, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(SIZE_BYTES, QFieldType.LONG).withIsEditable(false))
.withField(new QFieldMetaData(STORAGE_TABLE_NAME, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(STORAGE_REFERENCE, QFieldType.STRING).withIsEditable(false))
.withField(new QFieldMetaData(EXPIRES_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(JOB_ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(CREATE_DATE, QFieldType.DATE_TIME).withIsEditable(false)));
defineEffects(instance);
instance.getTable(JOBS_JOB).withSection(new QFieldSection()
.withName("fields").withLabel("Job").withTier(Tier.T1)
.withFieldNames(new ArrayList<>(instance.getTable(JOBS_JOB).getFields().keySet())));
instance.getTable(JOBS_JOB).withRecordSecurityLock(lock(USER_ID, null));
instance.getTable(JOBS_UPLOAD).withRecordSecurityLock(lock(USER_ID, null));
for(String child : List.of(JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EFFECT, JOBS_EVENT))
{
String join = JobsCoreContract.childJoin(child);
instance.addJoin(new QJoinMetaData().withLeftTable(JOBS_JOB).withRightTable(child).withType(JoinType.ONE_TO_MANY)
.withJoinOn(new JoinOn(ID, JOB_ID)).withInferredName());
instance.getTable(child).withRecordSecurityLock(lock(JOBS_JOB + "." + USER_ID, List.of(join)));
}
instance.getTable(JOBS_CHUNK).withUniqueKey(new UniqueKey(JOB_ID, SEQ));
instance.getTable(JOBS_WORKER).withUniqueKey(new UniqueKey(POD_ID));
instance.getTable(JOBS_UPLOAD).withUniqueKey(new UniqueKey(REF));
}
/*******************************************************************************
** STAND-IN(core=QRun-IO/qqq#1191, removeBy=RC.6): outbox fields and possible values.
*******************************************************************************/
private static void defineEffects(QInstance instance) throws QException
{
instance.addPossibleValueSource(new QPossibleValueSource().withName(JOBS_JOB).withType(QPossibleValueSourceType.TABLE).withTableName(JOBS_JOB));
instance.addPossibleValueSource(new QPossibleValueSource().withName(JOBS_CHUNK).withType(QPossibleValueSourceType.TABLE).withTableName(JOBS_CHUNK));
instance.addPossibleValueSource(new QPossibleValueSource().withName(JOBS_EFFECT_STATUS).withType(QPossibleValueSourceType.ENUM).withEnumValues(List.of(
new QPossibleValue<>(PENDING, "Pending"), new QPossibleValue<>(SENDING, "Sending"), new QPossibleValue<>(SENT, "Sent"),
new QPossibleValue<>(FAILED, "Failed"), new QPossibleValue<>(UNKNOWN, "Unknown"))));
instance.addPossibleValueSource(new QPossibleValueSource().withName(JOBS_EFFECT_TYPE).withType(QPossibleValueSourceType.ENUM).withEnumValues(List.of(
new QPossibleValue<>(LOCAL_TX, "Local Transaction"), new QPossibleValue<>(IDEMPOTENT_EXTERNAL, "Idempotent External"),
new QPossibleValue<>(NON_IDEMPOTENT_EXTERNAL, "Non-Idempotent External"))));
instance.addTable(new QTableMetaData().withName(JOBS_EFFECT).withBackendName("memory").withPrimaryKeyField(ID)
.withField(new QFieldMetaData(ID, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(JOB_ID, QFieldType.INTEGER).withIsEditable(false).withPossibleValueSourceName(JOBS_JOB))
.withField(new QFieldMetaData(EFFECT_NAME, QFieldType.STRING).withIsEditable(false).withMaxLength(100))
.withField(new QFieldMetaData(STATUS, QFieldType.STRING).withIsEditable(false).withMaxLength(30).withPossibleValueSourceName(JOBS_EFFECT_STATUS))
.withField(new QFieldMetaData(NEEDS_REVIEW_FIELD, QFieldType.BOOLEAN).withIsEditable(false).withDefaultValue(false))
.withField(new QFieldMetaData(ROW_KEY, QFieldType.STRING).withIsEditable(false).withMaxLength(250))
.withField(new QFieldMetaData(TARGET_NAME, QFieldType.STRING).withIsEditable(false).withMaxLength(100))
.withField(new QFieldMetaData(EFFECT_TYPE, QFieldType.STRING).withIsEditable(false).withMaxLength(30).withPossibleValueSourceName(JOBS_EFFECT_TYPE))
.withField(new QFieldMetaData(CHUNK_ID, QFieldType.INTEGER).withIsEditable(false).withPossibleValueSourceName(JOBS_CHUNK))
.withField(new QFieldMetaData(CHUNK_SEQ, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(IDEMPOTENCY_KEY, QFieldType.STRING).withIsEditable(false).withMaxLength(500))
.withField(new QFieldMetaData(PAYLOAD_JSON, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(PAYLOAD_REF, QFieldType.STRING).withIsEditable(false).withMaxLength(1000))
.withField(new QFieldMetaData(ATTEMPT, QFieldType.INTEGER).withIsEditable(false).withDefaultValue(0))
.withField(new QFieldMetaData(MAX_ATTEMPTS, QFieldType.INTEGER).withIsEditable(false))
.withField(new QFieldMetaData(RECONCILE_ATTEMPTS, QFieldType.INTEGER).withIsEditable(false).withDefaultValue(0))
.withField(new QFieldMetaData(NEXT_ATTEMPT_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(LEASE_OWNER, QFieldType.STRING).withIsEditable(false).withMaxLength(250))
.withField(new QFieldMetaData(LEASE_TOKEN, QFieldType.STRING).withIsEditable(false).withMaxLength(100))
.withField(new QFieldMetaData(LEASE_EXPIRES_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(LEASE_VERSION, QFieldType.INTEGER).withIsEditable(false).withDefaultValue(0))
.withField(new QFieldMetaData(PROVIDER_REF, QFieldType.STRING).withIsEditable(false).withMaxLength(500))
.withField(new QFieldMetaData(LAST_ERROR, QFieldType.TEXT).withIsEditable(false))
.withField(new QFieldMetaData(SENT_AT, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(CREATE_DATE, QFieldType.DATE_TIME).withIsEditable(false))
.withField(new QFieldMetaData(MODIFY_DATE, QFieldType.DATE_TIME).withIsEditable(false))
.withUniqueKey(new UniqueKey(IDEMPOTENCY_KEY)));
}
/*******************************************************************************
** The same read/write owner lock as the kernel, including join traversal.
*******************************************************************************/
private static RecordSecurityLock lock(String field, List<String> joins)
{
RecordSecurityLock lock = new RecordSecurityLock().withSecurityKeyType(JOBS_USER).withFieldName(field)
.withLockScope(RecordSecurityLock.LockScope.READ_AND_WRITE);
if(joins != null)
{
lock.withJoinNameChain(joins);
}
return (lock);
}
}
@@ -482,7 +482,7 @@ class JobsWidgetRendererTest extends BaseJobsTest
}
/*******************************************************************************
** Attention and progress tolerate a host omitting the effects table.
** RC.5 instances still render attention and progress without an effects table.
*******************************************************************************/
@Test
void widgetsWithoutEffectsTable() throws Exception