Merge pull request 'build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)' (#30) from deps/align-gitea into develop

pom.xml: parent -> 2.2.1
pom.xml: qqq.snapshot.version: 4.1.0-RC.8 -> 4.1.0-RC.26
pom.xml: qqq.snapshot.version: 4.1.0-RC.7 -> 4.1.0-RC.26
pom.xml: qqq-frontend-next: 1.0.0-RC.19 -> 1.0.0-RC.38
pom.xml: removed Central snapshot repository
pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.circleci/config.yml

@@ -11,4 +11,4 @@ workflows:
           branches:
             ignore: automation/qbit-rc-pending
           tags:
-            ignore: /.*/
+            only: /^v\d+\.\d+\.\d+$/

.editorconfig

@@ -0,0 +1,20 @@
+# QRun-IO qbit .editorconfig: the Kingsrook layout (3-space indentation, never tabs).
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 3
+tab_width = 3
+
+[*.md]
+trim_trailing_whitespace = false
+
+[*.{yml,yaml}]
+indent_size = 2
+
+[*.{cmd,bat}]
+end_of_line = crlf

.gitattributes

@@ -0,0 +1,3 @@
+* text=auto eol=lf
+*.cmd text eol=crlf
+mvnw text eol=lf

.gitea/workflows/build.yaml

@@ -0,0 +1,7 @@
+name: build
+on:
+  push:
+jobs:
+  maven:
+    uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
+    secrets: inherit

.gitea/workflows/qrun-release.yaml

@@ -1,24 +0,0 @@
-# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
-name: qrun-release
-on:
-  push: {}
-  workflow_dispatch: {}
-permissions:
-  contents: read
-  actions: read
-jobs:
-  release:
-    uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-maven.yaml@qrun-release-v1
-    with:
-      runner: linux-small
-      browsers: true
-      browser-runner: browser-build
-  postpublish:
-    if: ${{ needs.release.outputs.capture == 'true' }}
-    needs: release
-    permissions:
-      contents: write
-      actions: write
-    uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
-    with:
-      runner: linux-small

.github/dependabot.yml

@@ -0,0 +1,25 @@
+# Weekly, grouped dependency updates. Minor and patch updates arrive as one PR per ecosystem;
+# major updates stay separate so each gets its own review. At most three version-update PRs per
+# ecosystem are open at once. Security updates are enabled in the repository settings and are
+# not limited by this schedule.
+version: 2
+updates:
+  - package-ecosystem: maven
+    directory: /
+    target-branch: develop
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 3
+    groups:
+      maven-minor-and-patch:
+        update-types: [minor, patch]
+
+  - package-ecosystem: github-actions
+    directory: /
+    target-branch: develop
+    schedule:
+      interval: weekly
+    open-pull-requests-limit: 3
+    groups:
+      github-actions:
+        patterns: ['*']

.github/workflows/release-rc.yml

@@ -0,0 +1,74 @@
+name: Release candidate
+'on':
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Plan a cut without changing release state
+        type: boolean
+        default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
+  pull_request:
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
+permissions:
+  contents: write
+  actions: read
+  checks: read
+jobs:
+  release-rc:
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
+    with:
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.gitignore

@@ -1,31 +1,21 @@
-# Maven
 target/
-pom.xml.tag
-pom.xml.releaseBackup
-pom.xml.versionsBackup
-pom.xml.next
-release.properties
-
-# IDE
-.idea/
+.env
+.idea
 *.iml
-.project
-.classpath
-.settings/
-.vscode/
-*.swp
-*.swo
-
-# OS
-.DS_Store
-Thumbs.db
-
-# Build
 *.class
+*.log
 *.jar
 *.war
+*.nar
 *.ear
-
-# Logs
-*.log
-logs/
+*.zip
+*.tar.gz
+*.rar
+hs_err_pid*
+.DS_Store
+*.sw*
+.flattened-pom.xml
+dependency-reduced-pom.xml
+/.env.local
+/.cache/
+.claude/settings.local.json

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,3 @@
+wrapperVersion=3.3.4
+distributionType=only-script
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip

.qqq-ci.yml

@@ -6,10 +6,11 @@ browsers: true
 api_checks: false
 docs_only: false
 release_on_main: false
-release_tag_policy: none
-test_branch_policy: all
+release_tag_policy: stable
+test_branch_policy: non-release
 publish:
-  snapshot: false
-  release_candidate: false
-  release: false
+  snapshot: true
+  release_candidate: true
+  release: true
   asciidoc: false
+release_completion: true

AGENTS.md

@@ -0,0 +1,79 @@
+# AGENTS.md
+
+This file provides guidance to Codex when working with this repository.
+
+## Project overview
+
+QBit Jobs is the typed Jobs dashboard, record panels, alert declarations and retention
+process for QQQ durable jobs. Read `docs/DESIGN.md` and `README.md` first. Core owns
+all seven `jobs_*` tables and control actions. This qbit adds no tables and no client UI.
+
+## Build commands
+
+Java 21; always use the wrapper and offline mode in this workspace:
+
+```bash
+./mvnw -o compile
+./mvnw -o test
+./mvnw -o test -Dtest=JobsWidgetRendererTest
+./mvnw -o verify
+./mvnw -o -Pdemo test-compile
+```
+
+Windows: `./mvnw.cmd -o ...`. Verify enforces checkstyle on main/test sources and
+JaCoCo >=70% instructions, >=90% classes. Default tests use JUnit 5, AssertJ, MOCK auth
+and the memory backend. No Docker or external services. Do not commit in this sandbox.
+
+## Architecture by package
+
+- `com.kingsrook.qbits.jobs`: `JobsQBitProducer` registers the app and metadata directly;
+  `JobsQBitConfig` supplies validated host settings; `JobsCoreContract` references core identifiers
+  and field names; core `DurableJobsSecurity` derives owner/admin session keys; `JobsIntegrations`
+  supplies opt-in ESB, Quick Search and Data Analysis table lists.
+- `widgets`: `JobsWidgetRenderer` dispatches the eight home/record panels;
+  `JobsChartData` provides typed stacked bars; `JobsChildRecordListRenderer` ensures
+  session keys before delegating to core's child-list renderer.
+- `processes`: `JobsPurgeFinishedStep` performs retention with 200-row keyset batches;
+  `JobsStorage` uses core storage actions and durable jobs metadata.
+- Tests: `BaseJobsTest` uses core `DurableJobsMetaDataProvider` and `DurableJobsMetaData`,
+  plus producer, renderer, security and retention suites. Memory storage verifies real
+  object deletion; a recording storage seam exercises failures and retries.
+
+## Invariants
+
+- Navigation contains only `jobs_job`. Processes are hidden, with plain labels.
+- Query dashboard and record data in the current viewer's session through core actions.
+  Never elevate a renderer. Apply missing jobs session keys before any read; owner locks
+  and joined child locks are core's responsibility. Workers require exact `jobs_admin`.
+- Only retention uses a system session, after authorization, and restores its caller.
+- Runtime code reads/writes no filesystem files. Version comes from a packaged resource.
+  Storage operations use core StorageAction. Never replace object storage with temp files.
+- Delete row errors/events/chunks/uploads before jobs. Purge only old terminal jobs.
+  Input-only cleanup requires COMPLETED and exactly zero failures. Clear a reference
+  only after successful deletion. Storage errors fail the run for retry.
+- Typed widgets only; no HTML markup. Group table information into a few flexible
+  columns, without fixed widths. Child lists use core's builder and record id parameter.
+- Keep config instance-local. No static config, sessions, data caches or tables.
+
+## Naming and code style
+
+- Package `com.kingsrook.qbits.jobs`; qbit `com.kingsrook.qbits:qbit-jobs`, namespace `jobs`.
+- Widgets `jobsHome*` / `jobsJob*`; process `jobsPurgeFinished`; app name configurable.
+- All core table/field/status/join/security/alert names belong in `JobsCoreContract`.
+- Three-space indentation, braces on new lines, `return (x);`, flower-box Javadoc on
+  every class/method, fluent `withX` setters and the Kingsrook Apache license header.
+- Enforce the repository's `checkstyle/config.xml`; do not weaken gates to pass builds.
+
+## Dependencies
+
+Parent `com.kingsrook:qbit-build-parent:2.0.0`; QQQ 4.1.0-RC.7 or later is required.
+The default `qqq-snapshot` profile imports released core BOM `4.1.0-RC.7`.
+The opt-in `demo` profile uses the same release, adds `src/demo/java` and test
+RDBMS/Javalin/H2 dependencies for the demo server.
+
+## Release flow
+
+CircleCI uses `kingsrook/qqq-orb@0.6.8`: feature branches test; `develop` publishes
+SNAPSHOTs; `release/*` publishes RCs; `v*` tags publish stable releases. The GitHub
+release-candidate workflow follows the repository standard. Publishing requires explicit
+owner authorization. Update CHANGELOG before a release; do not change another repository from this workspace.

CHANGELOG.md

@@ -0,0 +1,58 @@
+# Changelog
+
+All notable changes follow [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
+and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - Unreleased
+
+### Security
+
+- The Jobs app and its five non-admin home widgets now require `<appName>.hasAccess`
+  (default `jobs.hasAccess`) and are hidden otherwise. Behaviour change: hosts must grant
+  this permission to the users who should see the Jobs app. Workers still needs `jobs_admin`.
+
+### Changed
+
+- Built on QQQ 4.1.0-RC.7; stand-ins removed.
+
+### Added
+
+- RC.8 optional job progress fields: phase labels, explicit percent with legacy fallback,
+  message, binary byte counts, ETA and dropped-log notice in typed widgets.
+- Full-width owner-scoped Log child list after Live Progress, ascending by log ID.
+- System-session retention of log lines, including 64-bit cursors and 200-row batches.
+- Host `parentAppName` configuration for Admin > Operations > Jobs placement.
+- Progress/log, owner isolation, real core log permission and navigation tests.
+
+### Changed
+
+- Default QQQ BOM from 4.1.0-RC.5 to 4.1.0-RC.8; existing `qqq.snapshot.version`
+  override verifies against locally installed 4.1.0-SNAPSHOT without the demo profile.
+- Storage adapter test now verifies failure/reference preservation with the current API.
+
+### Earlier additions
+
+- Optional effects outbox child list, viewer-scoped attention review counts and typed
+  Live Progress effect statistics; Data Analysis includes `jobs_effect`.
+- `jobs.effect-unresolved` warning alert for effects requiring outcome review.
+- Retention deletes settled effects and preserves jobs and inputs with outstanding
+  effects, reporting the number kept in the process summary.
+- Outbox Postgres table and indexes, and coverage for effect counts, owner isolation
+  and retention.
+
+- Jobs app with six typed home widgets for live progress, throughput, attention,
+  worker health and 30-day history, scoped to each viewer's record permissions.
+- Job progress, ordered chunk/error/event child lists, and filtered audit links.
+- Core-compatible failure/stall alert declarations and a reaper heartbeat monitor.
+- Configurable, hidden, scheduled retention with system-session isolation, paged
+  deletion, object cleanup and standard process summary lines.
+- Opt-in ESB, Quick Search and Data Analysis registration lists.
+- Postgres DDL for the core tables and indexes.
+- Memory-backed tests using core durable jobs metadata and storage actions.
+- Offline Maven wrapper build with checkstyle and JaCoCo coverage enforcement.
+- Readable home widgets:
+  - block cells with record links and progress bars;
+  - process and status labels;
+  - relative or UTC times;
+  - grouped overview statistics;
+  - no repeated titles.

CLAUDE.md

@@ -1,25 +1,88 @@
-# qbit-template-application
+# CLAUDE.md
 
-GitHub template repository for scaffolding Application QBits (complete mini-applications
-with tables, processes, widgets, and UI navigation) on the QQQ platform.
+This file provides guidance to Claude Code when working with this repository.
 
-## QQQ 4.0 baseline — 2026-09-24
+## Project overview
 
-The historical review below predates the 4.0 migration. This template targets Java 21 and the public QQQ 4.0.0 release from Maven Central. The corrected APIs, license references and actual template commands are in README.md and docs/00-getting-started.md. Preserve the documented multi-instance and scaffold limitations.
+QBit Jobs is the typed Jobs dashboard, record panels, alert declarations and retention
+process for QQQ durable jobs. Read `docs/DESIGN.md` and `README.md` first. Core owns
+all six `jobs_*` tables and control actions. This qbit adds no tables and no client UI.
 
-## Knowledge base
+## Build commands
 
-Reviewed dossier and platform knowledge live in the second-brain vault:
+Java 21; always use the wrapper and offline mode in this workspace:
 
-- Hub: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/qqq-hub.md`
-- This repo's dossier: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/repos/qbit-template-application.md`
-- QBit mechanics refresher: `R:/Git.Local/KofTwentyTwo/second-brain/knowledge/qqq/architecture/metadata-model.md`
+```bash
+./mvnw -o compile
+./mvnw -o test
+./mvnw -o test -Dtest=JobsWidgetRendererTest
+./mvnw -o verify
+./mvnw -o -Pdemo test-compile
+```
 
-Reviewed at commit `fb9395ac0d07` (branch `main`, 2026-07-04).
+Windows: `./mvnw.cmd -o ...`. Verify enforces checkstyle on main/test sources and
+JaCoCo >=70% instructions, >=90% classes. Default tests use JUnit 5, AssertJ, MOCK auth
+and the memory backend. No Docker or external services. Do not commit in this sandbox.
 
-Key caveat from that review: the scaffold in `src/main` references six qqq APIs that do not
-exist in any qqq version (wrong packages for `MetaDataProducerInterface` /
-`AbstractWidgetRenderer` / `RenderWidgetInput`, plus nonexistent `QInstance.addAppSection`,
-`QProcessMetaData.withInputFields`, `QAppSection.withSourceQBitName`) — it has never compiled,
-and CI on `main` is red. Verify against the qqq checkout (`R:/Git.Local/QRunIO/qqq`) before
-trusting any API usage shown here.
+## Architecture by package
+
+- `com.kingsrook.qbits.jobs`: `JobsQBitProducer` registers the app and metadata directly;
+  `JobsQBitConfig` supplies validated host settings; `JobsCoreContract` holds temporary
+  core identifiers; `JobsSecurity` derives owner/admin session keys; `JobsIntegrations`
+  supplies opt-in ESB, Quick Search and Data Analysis table lists.
+- `widgets`: `JobsWidgetRenderer` dispatches the eight home/record panels;
+  `JobsChartData` provides typed stacked bars; `JobsChildRecordListRenderer` ensures
+  session keys before delegating to core's child-list renderer.
+- `processes`: `JobsPurgeFinishedStep` performs retention with 200-row keyset batches;
+  `JobsStorage` isolates reflective core storage/metadata compatibility.
+- Tests: `BaseJobsTest`, `JobsTablesStandIn`, `JobsAlertsValidationStandIn`, plus producer,
+  renderer, security and retention suites. Test storage emulates the future deletion API
+  without filesystem access; the real RC.5 adapter's skip behavior is tested separately.
+
+## Invariants
+
+- Navigation contains only `jobs_job`. Processes are hidden, with plain labels.
+- Query dashboard and record data in the current viewer's session through core actions.
+  Never elevate a renderer. Apply missing jobs session keys before any read; owner locks
+  and joined child locks are core's responsibility. Workers require exact `jobs_admin`.
+- Only retention uses a system session, after authorization, and restores its caller.
+- Runtime code reads/writes no filesystem files. Version comes from a packaged resource.
+  Storage operations use core StorageAction. Never replace object storage with temp files.
+- Delete row errors/events/chunks/uploads before jobs. Purge only old terminal jobs.
+  Input-only cleanup requires COMPLETED and exactly zero failures. Clear a reference
+  only after successful deletion. Storage errors fail the run for retry.
+- Typed widgets only; no HTML markup. Group table information into a few flexible
+  columns, without fixed widths. Child lists use core's builder and record id parameter.
+- Keep config instance-local. No static config, sessions, data caches or tables.
+
+## Naming and code style
+
+- Package `com.kingsrook.qbits.jobs`; qbit `com.kingsrook.qbits:qbit-jobs`, namespace `jobs`.
+- Widgets `jobsHome*` / `jobsJob*`; process `jobsPurgeFinished`; app name configurable.
+- All core table/field/status/join/security/alert names belong in `JobsCoreContract`.
+- Three-space indentation, braces on new lines, `return (x);`, flower-box Javadoc on
+  every class/method, fluent `withX` setters and the Kingsrook Apache license header.
+- Enforce the repository's `checkstyle/config.xml`; do not weaken gates to pass builds.
+
+## Dependencies and stand-ins
+
+Parent `com.kingsrook:qbit-build-parent:2.0.0`; default `qqq-snapshot` profile imports
+released core BOM `4.1.0-RC.5`. Main code and default tests must not import `core.jobs`.
+The opt-in `demo` profile selects locally installed `4.1.0-SNAPSHOT`, adds `src/demo/java`
+and test RDBMS/Javalin/H2 dependencies. No demo server is provided yet.
+
+Remove stand-ins when RC.6 supplies the contracts:
+
+- `JobsCoreContract`, `JobsSecurity`, test `JobsTablesStandIn`: core #1178.
+- `JobsStorage`: #1173 deleteObject method and #1178 durableJobs metadata getter.
+- Test `JobsAlertsValidationStandIn`: #1178 kernel identifiers use underscores rejected
+  by the released alert-name validator. Only those two errors are waived in tests;
+  production validation remains strict. A core validator fix is required for startup.
+
+## Release flow
+
+CircleCI uses `kingsrook/qqq-orb@0.6.8`: feature branches test; `develop` publishes
+SNAPSHOTs; `release/*` publishes RCs; `v*` tags publish stable releases. The GitHub
+release-candidate workflow follows the repository standard. Publishing requires explicit
+owner authorization. Update CHANGELOG and resolve the RC.6 compatibility requirements
+before a release; do not change another repository from this workspace.

CODE_OF_CONDUCT.md

@@ -0,0 +1,3 @@
+# Code of Conduct
+
+See Project level [Code of Conduct](https://github.com/Kingsrook/qqq/blob/develop/CODE_OF_CONDUCT.md)

CODE_STYLE.md

@@ -0,0 +1,3 @@
+# Code Style Guide
+
+* See Project [Code Style Guide](https://github.com/Kingsrook/qqq/blob/develop/CODE_STYLE.md)

CONTRIBUTING.md

@@ -0,0 +1,3 @@
+# Contributing to QBit Jobs
+
+* See Project [Contributing](https://github.com/Kingsrook/qqq/blob/develop/CONTRIBUTING.md)

LICENSE

@@ -176,7 +176,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   Copyright 2022-2025 QRunIO, LLC
+   Copyright 2024 Kingsrook, LLC
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

NOTICE

@@ -1,6 +1,6 @@
-qBit Template - Application
-Copyright 2022-2025 QRunIO, LLC
+QBit Jobs
+Copyright 2026 Kingsrook, LLC
 
-This product includes software developed at QRunIO, LLC (https://qrun.io/).
+This product includes software developed at Kingsrook, LLC (https://kingsrook.com/).
 
 Licensed under the Apache License, Version 2.0.

README.md

@@ -1,55 +1,130 @@
-# QBit Template - Application
+# QBit Jobs
 
-Template repository for creating **Application QBits** - complete, self-contained applications or major functional modules for the QQQ framework.
+The Jobs app for QQQ durable background jobs. Its home dashboard shows live progress, throughput,
+failures, workers and 30 days of history. It adds progress panels and child lists on the job record,
+declares the failure, stall and effect-review alert types, and purges old jobs. The durable job kernel itself
+(tables, leases, plans, control actions, uploads) is in QQQ core. Design: [docs/DESIGN.md](docs/DESIGN.md),
+from QRun-IO/qqq `docs/specs/DURABLE-JOBS-AND-ASSISTED-BULK-LOAD.md`.
 
-## When to Use This Template
-
-Use this template when building:
-- Complete mini-applications (WMS, OMS, CRM)
-- Major functional modules with multiple tables
-- QBits that require UI navigation (QAppSection)
-- Self-contained features with business logic
-
-## Quick Start
-
-Requires **Java 21**, **Maven 3.8+**, and QQQ **4.0.0**. Create a repository with **Use this template**, customize its Maven coordinates and Java package/classes, then run `mvn clean verify`. Follow the [Getting Started guide](docs/00-getting-started.md) for the full sequence.
-
-## Structure
+## Requirements
+- Java 21.
+- QQQ 4.1.0-RC.7 or later with durable jobs enabled: `DurableJobsMetaDataProvider.defineAll(...)` and
+  `DurableJobsMetaData` on the instance.
+- A storage table for chunk objects and uploads, which should be object storage (S3).
 
+## Install
+```xml
+<dependency>
+   <groupId>com.kingsrook.qbits</groupId>
+   <artifactId>qbit-jobs</artifactId>
+   <version>0.1.0</version>
+</dependency>
 ```
-src/main/java/com/kingsrook/qbits/example/
-├── ExampleAppQBitConfig.java
-├── ExampleAppQBitProducer.java
-├── model/
-│   ├── ExampleEntity.java
-│   └── ExampleChildEntity.java
-├── processes/
-│   ├── ExampleProcessStep.java
-│   └── ExampleProcessMetaDataProducer.java
-└── widgets/
-    ├── ExampleDashboardWidgetRenderer.java
-    └── ExampleDashboardWidgetMetaDataProducer.java
+```java
+new DurableJobsMetaDataProvider().defineAll(qInstance, "rdbms", DurableJobsMetaDataProvider.snakeCaseColumnNames());
+qInstance.withSupplementalMetaData(new DurableJobsMetaData().withStorageTableName("objectStorageJobs"));
+new JobsQBitProducer().withQBitConfig(new JobsQBitConfig()).produce(qInstance);
 ```
 
-## Key Characteristics
+## Tables
+The tables are core's, prefixed `jobs_`. This qbit adds no tables.
 
-| Feature | Description |
-|---------|-------------|
-| QAppSection | Required - provides UI navigation |
-| Tables | Multiple interconnected business tables |
-| Processes | Full CRUD with business logic |
-| Widgets | Dashboard components |
-| Security | Permission key support |
-| Integration | Hooks for other QBits |
+| Table | What it holds |
+|---|---|
+| `jobs_job` | One row per job: status, progress counters, owner, channel, inputs, outputs. **Nav entry.** |
+| `jobs_chunk` | The leased units of work, with attempts, checkpoints and per-chunk counts. Child list on the job. |
+| `jobs_row_error` | Which rows failed and why. Child list on the job; reportable. |
+| `jobs_effect` | Side effects outside the database (outbox): email, HTTP, S3, other databases; child list on the job. |
+| `jobs_log` | Append-only, owner-scoped log lines (level, time, phase, message); optional child list on the job. |
+| `jobs_event` | The job's timeline (launched, reclaimed, paused, completed and so on). Child list on the job. |
+| `jobs_worker` | The pods that run jobs. |
+| `jobs_upload` | Uploaded files (refs, owners, expiry). |
 
-## Documentation
+## Permissions
+- Users see their own jobs: core's record lock on `jobs_job.userId` through the `jobs_user` key.
+- The Jobs app and its home widgets require `<appName>.hasAccess` (default `jobs.hasAccess`, from
+  `JobsQBitConfig.getHasAccessPermissionName()`); without it they are hidden. Hosts must grant it to the
+  users who should see the Jobs app. The header Jobs tray and record locks are unaffected.
+- `jobs_admin` sees and controls every job, and sees the Workers widget.
+- Apply the keys in your auth customizer with `DurableJobsSecurity.applyToSession(session)`.
 
-- [Getting Started](docs/00-getting-started.md)
-- [Tables and Relationships](docs/01-tables-and-relationships.md)
-- [Processes](docs/02-processes.md)
-- [Widgets and UI](docs/03-widgets-and-ui.md)
+## Configuration
+`JobsQBitConfig`:
+- `retentionDays` (30);
+- `inputRetentionDays` (7);
+- `purgeCronExpression` (`0 30 3 * * ?`, daily at 03:30 UTC);
+- `schedulerName` (null; set to a host scheduler name to enable the schedule);
+- `homeRefreshSeconds` (5);
+- `recordRefreshSeconds` (3);
+- `appName` (`jobs`);
+- `parentAppName` (null; an existing host app, such as `operations` under Admin).
 
-## See Also
+To place Jobs at **Admin > Operations > Jobs**, register the host Admin and Operations
+apps first (Operations has Admin as its parent), then produce with:
+```java
+new JobsQBitProducer().withQBitConfig(new JobsQBitConfig()
+   .withParentAppName("operations")).produce(qInstance);
+```
+The parent must exist and differ from `appName`; Jobs attaches to its children.
+With no parent configured, Jobs remains a top-level app. Placement does not grant permissions.
 
-- [qbit-template-extension](https://github.com/QRun-IO/qbit-template-extension) - For infrastructure extensions
-- [qbit-template-data](https://github.com/QRun-IO/qbit-template-data) - For reference data providers
+## Adopting in another app
+1. Enable durable jobs in core (above) and create the tables. `docs/ddl/postgres.sql` is the Postgres
+   DDL with snake_case columns.
+2. Produce this qbit.
+3. Wire the optional integrations from `JobsIntegrations`: ESB, Quick Search and Data Analysis.
+4. If qbit-alerts is installed, the alert types appear in it automatically.
+
+## Build
+```bash
+./mvnw -o verify    # tests, checkstyle (main and test sources), JaCoCo gate
+```
+
+The local demo server (`src/demo/java`, used for the screenshots) uses released QQQ
+4.1.0-RC.7 through the `demo` profile; no locally installed kernel build is needed.
+```bash
+./mvnw -o -Pdemo test-compile exec:java -Dexec.classpathScope=test -Dexec.mainClass=com.kingsrook.qbits.jobs.demo.JobsDemoServer
+# then open http://localhost:8000/app/jobs
+```
+On Windows use `./mvnw.cmd` in place of `./mvnw`.
+
+## Display
+
+The job progress fields and the Log list need QQQ 4.1.0-RC.8 or later; on older metadata
+the optional progress fields may be absent and the Log list is left out.
+
+The app produces typed data only, with no HTML. The widget label is each card's title,
+so the data carries no second title.
+- **Tables.** The running, attention and worker tables have two or three flexible
+  columns. Their cells are composite widget blocks:
+  - the job's label links to its record, with the process label under it;
+  - a progress bar uses `percentDone` when present (including zero), otherwise rows/chunks;
+  - phase labels come from `progressJson.phases` with `currentPhase` as fallback;
+  - plain text shows the progress message and optional bytes in B/KiB/MiB/GiB;
+  - the status line is a label (or the waiting reason) with the rate and ETA.
+  Each row also keeps its plain values (`Job`, `Process`, `Status`, `Rows`, `Rate`,
+  `ETA`, `Path` and so on) for exports and other clients.
+- **Labels.** Statuses show core's labels, and processes show their metadata labels.
+- **Times.** Times within a day are relative ("4m ago", "in 3m"). Older times are
+  absolute UTC ("Oct 14, 08:46 UTC"). Chart buckets use UTC.
+- **Statistics.** Statistic values are whole numbers, so rates are rounded to rows per
+  minute. Rows/min (last hour) is the rows done by jobs that progressed in the last
+  hour, divided by 60.
+
+Retention is admin/system-only and restores the original session even on failure.
+Its output contains standard process summary lines for jobs purged, chunk objects
+removed during purge, and input objects removed (uploads plus retained-job inputs).
+Jobs with effects outside SENT/FAILED, or with effects needing review, keep their rows
+and inputs. When any are skipped, the summary includes "N jobs kept: effects still unsettled".
+The Log list follows Live Progress, before Events: level, time, phase and message,
+newest first, full width, with the standard child-list 25-row limit and View all link.
+Live Progress also shows the ETA and a notice when `logDropped` is positive. The log is
+read-only for users, including admins; retention deletes it through the system session in
+keyset batches with 64-bit log cursors, before deleting chunks and jobs.
+
+Needs attention shows each listed job's effect-review count; Live Progress adds an Effects
+group (Sent, Pending, Failed, Needs review) when the job has effects.
+
+
+## License
+Apache License 2.0. See [LICENSE](LICENSE).

SECURITY.md

@@ -0,0 +1,3 @@
+# Security Policy
+
+ * See Project [Security Policy](https://github.com/Kingsrook/qqq/blob/develop/SECURITY.md)

checkstyle/config.xml

@@ -0,0 +1,246 @@
+<?xml version="1.0"?>
+<!--
+  ~ QQQ - Low-code Application Framework for Engineers.
+  ~ Copyright (C) 2021-2022.  Kingsrook, LLC
+  ~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+  ~ contact@kingsrook.com
+  ~ https://github.com/Kingsrook/
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
+<!--
+   See reference at https://checkstyle.sourceforge.io/checks.html
+-->
+<module name = "Checker">
+   <property name="charset" value="UTF-8"/>
+
+   <property name="severity" value="warning"/>
+
+   <property name="fileExtensions" value="java, properties, xml"/>
+   <!-- Checks for whitespace                        -->
+   <!-- See http://checkstyle.sf.net/config_whitespace.html -->
+   <module name="FileTabCharacter">
+      <property name="eachLine" value="true"/>
+   </module>
+
+   <!--
+   <module name="LineLength">
+      <property name="max" value="100"/>
+      <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
+   </module>
+   -->
+
+   <module name="TreeWalker">
+      <module name="SuppressWarningsHolder"/>
+      <module name="OuterTypeFilename"/>
+      <module name="IllegalTokenText">
+         <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
+         <property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
+         <property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
+      </module>
+      <module name="AvoidEscapedUnicodeCharacters">
+         <property name="allowEscapesForControlCharacters" value="true"/>
+         <property name="allowByTailComment" value="true"/>
+         <property name="allowNonPrintableEscapes" value="true"/>
+      </module>
+      <module name="AvoidStarImport"/>
+      <module name="OneTopLevelClass"/>
+      <module name="NoLineWrap"/>
+      <module name="EmptyBlock">
+         <property name="option" value="TEXT"/>
+         <property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
+      </module>
+      <module name="NeedBraces"/>
+      <module name="LeftCurly">
+         <property name="option" value="nl"/>
+      </module>
+      <module name="RightCurly">
+         <property name="option" value="alone"/>
+      </module>
+      <!--
+      <module name="WhitespaceAround">
+         <property name="allowEmptyConstructors" value="true"/>
+         <property name="allowEmptyMethods" value="true"/>
+         <property name="allowEmptyTypes" value="true"/>
+         <property name="allowEmptyLoops" value="true"/>
+         <message key="ws.notFollowed"
+          value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
+          <message key="ws.notPreceded"
+          value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
+      </module>
+      -->
+      <module name="OneStatementPerLine"/>
+      <module name="MultipleVariableDeclarations"/>
+      <module name="ArrayTypeStyle"/>
+      <module name="MissingSwitchDefault"/>
+      <module name="FallThrough"/>
+      <module name="UpperEll"/>
+      <module name="ModifierOrder"/>
+      <module name="EmptyLineSeparator">
+         <property name="allowNoEmptyLineBetweenFields" value="true"/>
+      </module>
+      <module name="SeparatorWrap">
+         <property name="id" value="SeparatorWrapDot"/>
+         <property name="tokens" value="DOT"/>
+         <property name="option" value="nl"/>
+      </module>
+      <module name="SeparatorWrap">
+         <property name="id" value="SeparatorWrapComma"/>
+         <property name="tokens" value="COMMA"/>
+         <property name="option" value="EOL"/>
+      </module>
+      <module name="SeparatorWrap">
+         <!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
+         <property name="id" value="SeparatorWrapEllipsis"/>
+         <property name="tokens" value="ELLIPSIS"/>
+         <property name="option" value="EOL"/>
+      </module>
+      <module name="SeparatorWrap">
+         <!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
+         <property name="id" value="SeparatorWrapArrayDeclarator"/>
+         <property name="tokens" value="ARRAY_DECLARATOR"/>
+         <property name="option" value="EOL"/>
+      </module>
+      <module name="SeparatorWrap">
+         <property name="id" value="SeparatorWrapMethodRef"/>
+         <property name="tokens" value="METHOD_REF"/>
+         <property name="option" value="nl"/>
+      </module>
+      <module name="PackageName">
+         <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
+         <message key="name.invalidPattern"
+          value="Package name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="TypeName">
+         <message key="name.invalidPattern"
+          value="Type name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="MemberName">
+         <property name="format" value="^[_a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
+         <message key="name.invalidPattern" value="Member name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="ParameterName">
+         <property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
+         <message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="CatchParameterName">
+         <property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
+         <message key="name.invalidPattern" value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="LocalVariableName">
+         <property name="tokens" value="VARIABLE_DEF"/>
+         <property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
+         <message key="name.invalidPattern" value="Local variable name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="ClassTypeParameterName">
+         <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
+         <message key="name.invalidPattern" value="Class type name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="MethodTypeParameterName">
+         <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
+         <message key="name.invalidPattern" value="Method type name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="InterfaceTypeParameterName">
+         <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
+         <message key="name.invalidPattern" value="Interface type name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="NoFinalizer"/>
+      <module name="GenericWhitespace">
+         <message key="ws.followed" value="GenericWhitespace ''{0}'' is followed by whitespace."/>
+         <message key="ws.preceded" value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
+         <message key="ws.illegalFollow" value="GenericWhitespace ''{0}'' should followed by whitespace."/>
+         <message key="ws.notPreceded" value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
+      </module>
+      <module name="Indentation">
+         <property name="basicOffset" value="3"/>
+         <property name="braceAdjustment" value="0"/>
+         <property name="caseIndent" value="3"/>
+         <property name="throwsIndent" value="6"/>
+         <property name="lineWrappingIndentation" value="3"/>
+         <property name="arrayInitIndent" value="6"/>
+      </module>
+      <!--
+      <module name="AbbreviationAsWordInName">
+         <property name="ignoreFinal" value="false"/>
+         <property name="allowedAbbreviationLength" value="1"/>
+      </module>
+      <module name="OverloadMethodsDeclarationOrder"/>
+      <module name="VariableDeclarationUsageDistance"/>
+      <module name="CustomImportOrder">
+         <property name="sortImportsInGroupAlphabetically" value="true"/>
+         <property name="separateLineBetweenGroups" value="true"/>
+         <property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
+      </module>
+      -->
+      <module name="CustomImportOrder">
+         <property name="customImportOrderRules" value="SPECIAL_IMPORTS###STANDARD_JAVA_PACKAGE###THIRD_PARTY_PACKAGE###STATIC"/>
+         <property name="specialImportsRegExp" value="^javax\."/>
+         <property name="standardPackageRegExp" value="^java\."/>
+         <property name="sortImportsInGroupAlphabetically" value="true"/>
+         <property name="separateLineBetweenGroups" value="false"/>
+      </module>
+      <module name="MethodParamPad"/>
+      <module name="NoWhitespaceBefore">
+         <property name="tokens" value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
+         <property name="allowLineBreaks" value="true"/>
+      </module>
+      <module name="ParenPad"/>
+      <module name="OperatorWrap">
+         <property name="option" value="NL"/>
+         <property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
+      </module>
+      <module name="AnnotationLocation">
+         <property name="id" value="AnnotationLocationMostCases"/>
+         <property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
+      </module>
+      <module name="AnnotationLocation">
+         <property name="id" value="AnnotationLocationVariables"/>
+         <property name="tokens" value="VARIABLE_DEF"/>
+         <property name="allowSamelineMultipleAnnotations" value="true"/>
+      </module>
+      <module name="JavadocMethod">
+         <property name="allowMissingParamTags" value="true"/>
+         <property name="allowMissingReturnTag" value="true"/>
+         <property name="allowedAnnotations" value="Override, Test"/>
+      </module>
+      <module name="MissingJavadocMethod">
+         <property name="scope" value="private"/>
+      </module>
+      <module name="MissingJavadocType">
+         <property name="scope" value="private"/>
+      </module>
+      <module name="MethodName">
+         <property name="format" value="^[a-z][a-zA-Z0-9_]*$"/>
+         <message key="name.invalidPattern"
+          value="Method name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="EmptyCatchBlock">
+         <property name="exceptionVariableName" value="expected"/>
+      </module>
+      <module name="CommentsIndentation"/>
+
+      <!-- this would be nice, but requires the the @inheritDoc javadoc tag... -->
+      <module name="MissingOverride"/>
+
+   </module>
+   <module name="Header">
+     <property name="headerFile" value="checkstyle/license.txt"/>
+     <property name="fileExtensions" value="java"/>
+     <property name="ignoreLines" value="3"/>
+   </module>
+   <module name="SuppressWarningsFilter"/>
+</module>

checkstyle/license.txt

@@ -0,0 +1,19 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

docs/00-getting-started.md

@@ -1,30 +0,0 @@
-# Getting Started
-
-Targets Java 21, Maven 3.8 or later, and QQQ 4.0.0. Maven resolves the framework dependencies from Maven Central.
-
-1. Choose **Use this template** on GitHub, then clone the repository you created.
-2. In `pom.xml`, set your own `groupId`, `artifactId`, name, description, and version.
-3. Use your IDE's package refactoring to rename `com.kingsrook.qbits.example` throughout `src/`. Rename the `Example*` classes and update their imports and references with the IDE's rename refactoring.
-4. Update the QBit producer's `GROUP_ID`, `ARTIFACT_ID`, and `VERSION` constants to match your project. Give the metadata names their own stable names before combining the QBit with other examples.
-5. Build the generated project:
-
-```bash
-mvn clean verify
-```
-
-The repository provides example Java sources; customization uses normal package/class refactoring. Keep example coordinates unpublished. After customizing, add tests for your QBit's behavior before setting up its publishing workflow.
-
-## Register with a host application
-
-The following uses the original example names; substitute the names chosen above:
-
-```java
-new ExampleAppQBitProducer()
-   .withConfig(new ExampleAppQBitConfig()
-      .withBackendName("rdbms"))
-   .produce(qInstance, "my-application");
-```
-
-Register the configured backend in the host first. If you add table prefixes or multiple QBit instances, update related table references, possible-value-source names, and process/widget names consistently. A table prefix alone does not namespace all metadata.
-
-[Tables and Relationships](01-tables-and-relationships.md) describes the model; the `processes/` and `widgets/` directories contain the remaining examples.

docs/01-tables-and-relationships.md

@@ -1,50 +0,0 @@
-# Tables and Relationships
-
-Application QBits typically have multiple interconnected tables representing a business domain.
-
-## Entity Design
-
-Use `@QMetaDataProducingEntity` for automatic metadata generation:
-
-```java
-@QMetaDataProducingEntity(producePossibleValueSource = true)
-public class Order extends QRecordEntity
-{
-   public static final String TABLE_NAME = "order";
-
-   @QField(isPrimaryKey = true)
-   private Integer id;
-
-   @QField(isRequired = true, possibleValueSourceName = "customer")
-   private Integer customerId;
-
-   @QField(isRequired = true)
-   private String status;
-}
-```
-
-## Relationships
-
-Define foreign keys using `possibleValueSourceName`:
-
-```java
-@QField(isRequired = true, possibleValueSourceName = "exampleEntity")
-private Integer exampleEntityId;
-```
-
-## Table Prefixing
-
-Application QBits support optional table prefixing:
-
-```java
-// Without prefix: order, orderLine
-// With prefix "sales": sales_order, sales_orderLine
-
-new MyAppQBitProducer()
-   .withConfig(new MyAppQBitConfig()
-      .withBackendName("rdbms")
-      .withTableNamePrefix("sales"))
-   .produce(qInstance, "sales-app");
-```
-
-The producer applies the prefix to table names. When extending this pattern, also update possible-value-source names, relationship targets, process tables, and navigation references; table renaming alone does not provide complete multi-instance support.

docs/02-processes.md

@@ -1,7 +0,0 @@
-# Processes
-
-Start with [ExampleProcessMetaDataProducer](../src/main/java/com/kingsrook/qbits/example/processes/ExampleProcessMetaDataProducer.java) and [ExampleProcessStep](../src/main/java/com/kingsrook/qbits/example/processes/ExampleProcessStep.java). The producer defines the process, its table, inputs, and backend step; the step transforms `PENDING` records to `PROCESSED` and returns them in the output.
-
-In QQQ 4.0, import `MetaDataProducerInterface` from `com.kingsrook.qqq.backend.core.model.metadata`. Define process inputs through `withInputMetaData(new QFunctionInputMetaData().withField(...))`. An `AbstractTransformStep` must implement both `runOnePage(RunBackendStepInput, RunBackendStepOutput)` and `getProcessSummary(RunBackendStepOutput, boolean)`; keep the example's implementations when renaming it.
-
-To make the process discoverable in navigation, add its registered name to a `QAppSection` through `withProcesses(List.of(ExampleProcessMetaDataProducer.NAME))`, then include the section in a `QAppMetaData` registered with `qInstance.addApp(...)`. Use the actual registered name if you introduce a prefix. Add tests for record changes and failure cases when replacing the example business logic.

docs/03-widgets-and-ui.md

@@ -1,23 +0,0 @@
-# Widgets and UI
-
-The application producer registers a `QAppMetaData` containing a `QAppSection`. The section references metadata names already registered in the instance; it is not registered directly on `QInstance`.
-
-```java
-QAppSection section = new QAppSection()
-   .withName("exampleSection")
-   .withLabel("Example Application")
-   .withTables(List.of(ExampleEntity.TABLE_NAME, ExampleChildEntity.TABLE_NAME))
-   .withProcesses(List.of(ExampleProcessMetaDataProducer.NAME))
-   .withWidgets(List.of(ExampleDashboardWidgetMetaDataProducer.NAME));
-
-qInstance.addApp(new QAppMetaData()
-   .withName("exampleApp")
-   .withLabel("Example Application")
-   .withSection(section));
-```
-
-Use [ExampleAppQBitProducer](../src/main/java/com/kingsrook/qbits/example/ExampleAppQBitProducer.java) as the starting point. Keep these references consistent when enabling/disabling tables or introducing prefixes.
-
-The widget has two parts: [its metadata producer](../src/main/java/com/kingsrook/qbits/example/widgets/ExampleDashboardWidgetMetaDataProducer.java) declares a statistics widget and its renderer, and [its renderer](../src/main/java/com/kingsrook/qbits/example/widgets/ExampleDashboardWidgetRenderer.java) returns a `RenderWidgetOutput` containing `StatisticsData`. Replace the example count with a query or calculation. Choose a widget type that matches the returned data model.
-
-For QQQ 4.0, `AbstractWidgetRenderer` is in `com.kingsrook.qqq.backend.core.actions.dashboard.widgets`; `RenderWidgetInput` and `RenderWidgetOutput` are in `com.kingsrook.qqq.backend.core.model.actions.widgets`.

docs/DESIGN.md

@@ -0,0 +1,110 @@
+# QBit Jobs: design
+
+The approved design is QRun-IO/qqq `docs/specs/DURABLE-JOBS-AND-ASSISTED-BULK-LOAD.md` (#1171), section 2.12
+and phase 4. Owner decisions: all of that spec's recommendations are accepted (2026-10-12/13).
+
+**Split of responsibilities.**
+- **Core** (QQQ 4.1.x) owns the durable job kernel: the `jobs_*` tables
+  (`DurableJobsMetaDataProvider.defineAll`), leases, the per-pod runtime, the plans, the control
+  processes (record actions), uploads, realtime events, alerts and notifications. Without this qbit,
+  jobs still run and the tables still have standard screens.
+- **This qbit** owns the **Jobs app**: the home dashboard, the job record panels, the alert type
+  declarations, retention, and the Quick Search, Data Analysis and ESB registration hooks. It adds no
+  tables and no UI code: everything is typed widget data served through `/qqq/v1`, so any client (Next,
+  a CLI, iOS) can render it.
+
+## Requirements traced (owner R1-R6)
+- **R3, reportable.** Jobs, chunks, row errors, effects and events are standard tables, with full record
+  screens, reports and Data Analysis. Quick Search registers jobs only. The home shows live progress,
+  throughput, failures and history. Every job links to its audit trail.
+- **R4, notifications.** The alert types `jobs.job-failed`, `jobs.job-stalled` and `jobs.effect-unresolved`
+  ("Job effect needs review", WARNING), and the `jobs.reaper` heartbeat monitor, are declared here
+  so that qbit-alerts routes them. Completion notifications come from core.
+- **R6, storage.** Retention deletes finished jobs and their chunk objects from object storage. The
+  qbit reads and writes no files.
+
+## Progress and log adoption (core RC.8)
+
+Implements sections 2 and 7 of core `JOB-PROGRESS-AND-JOBS-TRAY.md`. The default
+BOM is 4.1.0-RC.8; local verification uses `-Dqqq.snapshot.version=4.1.0-SNAPSHOT`.
+Optional names remain in `JobsCoreContract` without importing `core.jobs`.
+
+- Home and Live Progress prefer nullable `percentDone`, including explicit zero,
+  then retain rows/chunks derivation. The snapshot supplies the current phase label;
+  missing or invalid JSON falls back to `currentPhase`. Both views include the
+  progress message, optional binary byte counts, and explicit `etaSeconds` ahead
+  of the existing rate-derived ETA. No HTML, fixed widths, or theme-specific colors.
+- Log is a full-width standard child list immediately after Live Progress, before
+  Events, only if both table and join exist. It displays level, loggedAt, phase and
+  message, sorted by id ascending, with the standard 25-row preview/View all.
+  The panel reads through the current viewer and core joined owner lock.
+- A positive `logDropped` produces an earlier-lines-not-kept notice in Live Progress.
+- Retention deletes logs before chunks/jobs, using the existing 200-row keyset loop
+  with a long child cursor. Core `JobLogPermissions` explicitly permits system
+  sessions and denies user mutations even for admins. Tests load the actual checker.
+- Hosts set `parentAppName` to their existing Operations app under Admin; the qbit
+  sets the parent and attaches Jobs as a child, while retaining configurable appName.
+  Missing/self parents fail validation before registration. Default remains top-level.
+
+## The app
+- `jobs` app, label **Jobs**. The nav lists `jobs_job` only. Chunks, row errors, effects and events are child
+  lists on the job record.
+- **Home**, built from typed widgets read in the viewer's session (record locks apply: a user sees
+  their own jobs, `jobs_admin` sees all). Live widgets set `defaultValues.refreshSeconds` (polling; the
+  realtime `job.progress` channel is the push path for clients that subscribe).
+  1. **Overview** (`MULTI_STATISTICS`): two groups. **Now** has Running, Queued and Needs
+     attention, each linking to the matching jobs. **Last 24 hours** has Completed, Rows
+     processed and Rows/min (last hour). It refreshes every 5s.
+  2. **Running and queued** (`TABLE`, three block columns): the job (its label links to the
+     record, with the process label under it), the progress (a bar, the percent, and rows done
+     of the total), and the status (a label or the waiting reason, then the rate and ETA, or
+     the start time). It refreshes every 5s.
+  3. **Rows per hour** (`STACKED_BAR_CHART`): rows per hour over the last 24 hours, by
+     process label.
+  4. **Needs attention** (`TABLE`): FAILED, NEEDS_REVIEW and COMPLETED_WITH_ERRORS jobs from the
+     last 7 days. Failed and needs-review jobs come first, newest first within each group.
+     Each row shows the job, the error on its own line, the rows failed and the finish time.
+     The status cell includes "N effects need review" when applicable, from one viewer-scoped
+     effects query for the listed jobs. RUNNING jobs remain excluded; core moves them to NEEDS_REVIEW.
+  5. **Workers** (`TABLE`): pods from `jobs_worker`, with threads, draining, and last heartbeat
+     age (admins only).
+  6. **Job history** (`STACKED_BAR_CHART`): jobs per day over the last 30 days, by final
+     status label.
+  Times within a day are relative; older ones are absolute UTC. Data carries no inner titles,
+  because the widget label is the card title.
+- **Job record**
+  - A **Live Progress** panel (`MULTI_STATISTICS`) has three groups: Rows (succeeded, failed,
+    total, and the percent done), Chunks (done, failed, total) and Speed (rows per minute, with
+    the ETA, the finish time or the status).
+    An optional fourth group, Effects, shows Sent, Pending (PENDING, SENDING and UNKNOWN
+    without needsReview), Failed and Needs review. Review effects appear only in Needs review.
+    The group is omitted when there are no effects or the effects table is absent.
+    It refreshes every 3s while the job is active.
+  - Child lists for **Log**, **Chunks**, **Row errors**, **Effects** and **Events**. Effects is
+    included only when the effects table and its inferred job join exist; it uses the same
+    joined owner lock as row errors.
+  - A **Changes made by this job** link to the audit table, filtered to messages containing
+    `job:<id>/` (core writes `job:<id>/chunk:<seq>` into every audit the job causes).
+  - The record actions are core's: Start, Pause, Resume, Cancel, Retry Failed, Retry Failed Rows,
+    Download Failed Rows, Restart, Run Again.
+
+## Retention
+`jobsPurgeFinished` is a hidden scheduled process, not in the nav. It runs daily by default.
+- It deletes jobs that finished more than `retentionDays` ago (default 30), with their chunks, row
+  errors, effects, logs, events and chunk objects. Effects are removed before chunks and jobs.
+- Jobs with any effect outside SENT/FAILED, or any effect needing review, keep their rows
+  and inputs. The summary reports "N jobs kept: effects still unsettled" when any are skipped.
+- For COMPLETED jobs with no failed rows, it deletes the chunk input objects after
+  `inputRetentionDays` (default 7).
+- Each run's summary is the standard process summary.
+
+## Integrations (opt-in, no dependency on the other modules)
+`JobsIntegrations.esbTableNames()`, `quickSearchTables()` and `dataAnalysisTables()`: a host passes
+these to qqq-esb, qbit-quick-search and qbit-data-analysis.
+
+## Dependencies
+Requires QQQ 4.1.0-RC.8 or later (job progress and `jobs_log`). The default `qqq-snapshot` profile and the opt-in
+`demo` profile both use released RC.8. Core provides table metadata, joins, possible
+values, session security, signal identifiers and storage deletion. The qbit declares
+the alert types and retains only field names that core does not export.
+The demo server under `src/demo/java` uses the same released kernel.

docs/ddl/postgres.sql

@@ -0,0 +1,198 @@
+-- Durable jobs schema matching QQQ 4.1.0-RC.7 DurableJobsMetaDataProvider.
+-- Snake-case columns; excludes test fixtures and destructive DROP statements.
+
+CREATE TABLE jobs_job
+(
+   id                        SERIAL PRIMARY KEY,
+   label                     VARCHAR(250),
+   process_name              VARCHAR(250) NOT NULL,
+   table_name                VARCHAR(250),
+   mode                      VARCHAR(20),
+   status                    VARCHAR(30) NOT NULL,
+   status_reason             VARCHAR(500),
+   priority                  INTEGER DEFAULT 5,
+   user_id                   VARCHAR(250),
+   client_channel            VARCHAR(250),
+   credential_scope_json     TEXT,
+   source_type               VARCHAR(30),
+   source_json               TEXT,
+   params_json               TEXT,
+   idempotency               VARCHAR(30),
+   semantics                 VARCHAR(20),
+   chunk_size                INTEGER,
+   max_parallel_chunks       INTEGER,
+   current_phase             VARCHAR(250),
+   percent_done              INTEGER,
+   progress_message          VARCHAR(1000),
+   bytes_done                BIGINT,
+   bytes_total               BIGINT,
+   eta_seconds               BIGINT,
+   progress_json             TEXT,
+   log_seq                   BIGINT,
+   log_dropped               INTEGER,
+   rows_total                INTEGER,
+   rows_done                 INTEGER,
+   rows_ok                   INTEGER,
+   rows_failed               INTEGER,
+   chunks_total              INTEGER,
+   chunks_done               INTEGER,
+   chunks_failed             INTEGER,
+   summary_json              TEXT,
+   output_storage_json       TEXT,
+   parent_job_id             INTEGER,
+   saved_bulk_load_profile_id INTEGER,
+   notify_email              BOOLEAN DEFAULT FALSE,
+   queued_at                 TIMESTAMPTZ,
+   started_at                TIMESTAMPTZ,
+   finished_at               TIMESTAMPTZ,
+   last_progress_at          TIMESTAMPTZ,
+   error_message             TEXT,
+   lease_version             INTEGER DEFAULT 0,
+   create_date               TIMESTAMPTZ,
+   modify_date               TIMESTAMPTZ
+);
+
+CREATE INDEX jobs_job_claim_idx ON jobs_job (status, priority, queued_at);
+
+CREATE INDEX jobs_job_user_idx ON jobs_job (user_id, status);
+
+CREATE TABLE jobs_chunk
+(
+   id                SERIAL PRIMARY KEY,
+   job_id            INTEGER NOT NULL REFERENCES jobs_job (id) ON DELETE CASCADE,
+   seq               INTEGER NOT NULL,
+   status            VARCHAR(30) NOT NULL,
+   range_json        TEXT,
+   input_storage_ref VARCHAR(1000),
+   attempt           INTEGER DEFAULT 0,
+   max_attempts      INTEGER DEFAULT 3,
+   lease_owner       VARCHAR(250),
+   lease_token       VARCHAR(100),
+   lease_expires_at  TIMESTAMPTZ,
+   heartbeat_at      TIMESTAMPTZ,
+   rows_in           INTEGER,
+   rows_ok           INTEGER,
+   rows_failed       INTEGER,
+   summary_json      TEXT,
+   checkpoint_json   TEXT,
+   error_message     TEXT,
+   started_at        TIMESTAMPTZ,
+   finished_at       TIMESTAMPTZ,
+   lease_version     INTEGER DEFAULT 0,
+   create_date       TIMESTAMPTZ,
+   modify_date       TIMESTAMPTZ,
+   UNIQUE (job_id, seq)
+);
+
+CREATE INDEX jobs_chunk_status_idx ON jobs_chunk (job_id, status);
+
+CREATE INDEX jobs_chunk_lease_idx ON jobs_chunk (status, lease_expires_at);
+
+CREATE TABLE jobs_row_error
+(
+   id           SERIAL PRIMARY KEY,
+   job_id       INTEGER NOT NULL REFERENCES jobs_job (id) ON DELETE CASCADE,
+   chunk_id     INTEGER,
+   row_number   INTEGER,
+   record_pk    VARCHAR(250),
+   field_name   VARCHAR(250),
+   error_code   VARCHAR(100),
+   message      TEXT,
+   retry_job_id INTEGER,
+   resolved     BOOLEAN DEFAULT FALSE,
+   create_date  TIMESTAMPTZ
+);
+
+CREATE INDEX jobs_row_error_job_idx ON jobs_row_error (job_id);
+
+-- Transactional outbox.
+CREATE TABLE jobs_effect
+(
+   id                 SERIAL PRIMARY KEY,
+   job_id             INTEGER NOT NULL REFERENCES jobs_job (id) ON DELETE CASCADE,
+   effect_name        VARCHAR(100),
+   status             VARCHAR(30),
+   needs_review       BOOLEAN DEFAULT FALSE,
+   row_key            VARCHAR(250),
+   target_name        VARCHAR(100),
+   effect_type        VARCHAR(30),
+   chunk_id           INTEGER,
+   chunk_seq          INTEGER,
+   idempotency_key    VARCHAR(500),
+   payload_json       TEXT,
+   payload_ref        VARCHAR(1000),
+   attempt            INTEGER DEFAULT 0,
+   max_attempts       INTEGER,
+   reconcile_attempts INTEGER DEFAULT 0,
+   next_attempt_at    TIMESTAMPTZ,
+   lease_owner        VARCHAR(250),
+   lease_token        VARCHAR(100),
+   lease_expires_at   TIMESTAMPTZ,
+   lease_version      INTEGER DEFAULT 0,
+   provider_ref       VARCHAR(500),
+   last_error         TEXT,
+   sent_at            TIMESTAMPTZ,
+   create_date        TIMESTAMPTZ,
+   modify_date        TIMESTAMPTZ
+);
+
+CREATE UNIQUE INDEX jobs_effect_idempotency_idx ON jobs_effect (idempotency_key);
+
+CREATE INDEX jobs_effect_status_idx ON jobs_effect (status, next_attempt_at);
+
+CREATE INDEX jobs_effect_job_idx ON jobs_effect (job_id);
+
+CREATE TABLE jobs_event
+(
+   id             SERIAL PRIMARY KEY,
+   job_id         INTEGER NOT NULL REFERENCES jobs_job (id) ON DELETE CASCADE,
+   chunk_id       INTEGER,
+   type           VARCHAR(30),
+   actor_user_id  VARCHAR(250),
+   client_channel VARCHAR(250),
+   pod            VARCHAR(250),
+   detail         TEXT,
+   create_date    TIMESTAMPTZ
+);
+
+CREATE INDEX jobs_event_job_idx ON jobs_event (job_id);
+
+CREATE TABLE jobs_worker
+(
+   id           SERIAL PRIMARY KEY,
+   pod_id       VARCHAR(250) NOT NULL UNIQUE,
+   started_at   TIMESTAMPTZ,
+   heartbeat_at TIMESTAMPTZ,
+   threads      INTEGER,
+   draining     BOOLEAN,
+   version      VARCHAR(100)
+);
+
+CREATE TABLE jobs_upload
+(
+   id                 SERIAL PRIMARY KEY,
+   ref                VARCHAR(100) NOT NULL UNIQUE,
+   user_id            VARCHAR(250),
+   file_name          VARCHAR(250),
+   content_type       VARCHAR(250),
+   size_bytes         BIGINT,
+   storage_table_name VARCHAR(250),
+   storage_reference  VARCHAR(1000),
+   expires_at         TIMESTAMPTZ,
+   job_id             INTEGER,
+   create_date        TIMESTAMPTZ
+);
+
+-- RC.8 job progress log; host migrations must also add the jobs_job columns above.
+CREATE TABLE jobs_log
+(
+   id BIGSERIAL PRIMARY KEY,
+   job_id INTEGER NOT NULL REFERENCES jobs_job(id) ON DELETE CASCADE,
+   chunk_id INTEGER REFERENCES jobs_chunk(id) ON DELETE SET NULL,
+   attempt INTEGER,
+   level VARCHAR(10) NOT NULL,
+   logged_at TIMESTAMPTZ NOT NULL,
+   phase VARCHAR(250),
+   message VARCHAR(1000)
+);
+CREATE INDEX jobs_log_cursor_idx ON jobs_log(job_id, id);

mvnw

@@ -0,0 +1,295 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Apache Maven Wrapper startup batch script, version 3.3.4
+#
+# Optional ENV vars
+# -----------------
+#   JAVA_HOME - location of a JDK home dir, required when download maven via java source
+#   MVNW_REPOURL - repo url base for downloading maven distribution
+#   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+#   MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
+# ----------------------------------------------------------------------------
+
+set -euf
+[ "${MVNW_VERBOSE-}" != debug ] || set -x
+
+# OS specific support.
+native_path() { printf %s\\n "$1"; }
+case "$(uname)" in
+CYGWIN* | MINGW*)
+  [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
+  native_path() { cygpath --path --windows "$1"; }
+  ;;
+esac
+
+# set JAVACMD and JAVACCMD
+set_java_home() {
+  # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
+  if [ -n "${JAVA_HOME-}" ]; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ]; then
+      # IBM's JDK on AIX uses strange locations for the executables
+      JAVACMD="$JAVA_HOME/jre/sh/java"
+      JAVACCMD="$JAVA_HOME/jre/sh/javac"
+    else
+      JAVACMD="$JAVA_HOME/bin/java"
+      JAVACCMD="$JAVA_HOME/bin/javac"
+
+      if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
+        echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
+        echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
+        return 1
+      fi
+    fi
+  else
+    JAVACMD="$(
+      'set' +e
+      'unset' -f command 2>/dev/null
+      'command' -v java
+    )" || :
+    JAVACCMD="$(
+      'set' +e
+      'unset' -f command 2>/dev/null
+      'command' -v javac
+    )" || :
+
+    if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
+      echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
+      return 1
+    fi
+  fi
+}
+
+# hash string like Java String::hashCode
+hash_string() {
+  str="${1:-}" h=0
+  while [ -n "$str" ]; do
+    char="${str%"${str#?}"}"
+    h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
+    str="${str#?}"
+  done
+  printf %x\\n $h
+}
+
+verbose() { :; }
+[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
+
+die() {
+  printf %s\\n "$1" >&2
+  exit 1
+}
+
+trim() {
+  # MWRAPPER-139:
+  #   Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
+  #   Needed for removing poorly interpreted newline sequences when running in more
+  #   exotic environments such as mingw bash on Windows.
+  printf "%s" "${1}" | tr -d '[:space:]'
+}
+
+scriptDir="$(dirname "$0")"
+scriptName="$(basename "$0")"
+
+# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
+while IFS="=" read -r key value; do
+  case "${key-}" in
+  distributionUrl) distributionUrl=$(trim "${value-}") ;;
+  distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
+  esac
+done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
+[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
+
+case "${distributionUrl##*/}" in
+maven-mvnd-*bin.*)
+  MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
+  case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
+  *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
+  :Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
+  :Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
+  :Linux*x86_64*) distributionPlatform=linux-amd64 ;;
+  *)
+    echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
+    distributionPlatform=linux-amd64
+    ;;
+  esac
+  distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
+  ;;
+maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
+*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
+esac
+
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
+[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
+distributionUrlName="${distributionUrl##*/}"
+distributionUrlNameMain="${distributionUrlName%.*}"
+distributionUrlNameMain="${distributionUrlNameMain%-bin}"
+MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
+MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
+
+exec_maven() {
+  unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
+  exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
+}
+
+if [ -d "$MAVEN_HOME" ]; then
+  verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+  exec_maven "$@"
+fi
+
+case "${distributionUrl-}" in
+*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
+*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
+esac
+
+# prepare tmp dir
+if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
+  clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
+  trap clean HUP INT TERM EXIT
+else
+  die "cannot create temp dir"
+fi
+
+mkdir -p -- "${MAVEN_HOME%/*}"
+
+# Download and Install Apache Maven
+verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+verbose "Downloading from: $distributionUrl"
+verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+# select .zip or .tar.gz
+if ! command -v unzip >/dev/null; then
+  distributionUrl="${distributionUrl%.zip}.tar.gz"
+  distributionUrlName="${distributionUrl##*/}"
+fi
+
+# verbose opt
+__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
+[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
+
+# normalize http auth
+case "${MVNW_PASSWORD:+has-password}" in
+'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
+has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
+esac
+
+if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
+  verbose "Found wget ... using wget"
+  wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
+elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
+  verbose "Found curl ... using curl"
+  curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
+elif set_java_home; then
+  verbose "Falling back to use Java to download"
+  javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
+  targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
+  cat >"$javaSource" <<-END
+	public class Downloader extends java.net.Authenticator
+	{
+	  protected java.net.PasswordAuthentication getPasswordAuthentication()
+	  {
+	    return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
+	  }
+	  public static void main( String[] args ) throws Exception
+	  {
+	    setDefault( new Downloader() );
+	    java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
+	  }
+	}
+	END
+  # For Cygwin/MinGW, switch paths to Windows format before running javac and java
+  verbose " - Compiling Downloader.java ..."
+  "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
+  verbose " - Running Downloader.java ..."
+  "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
+fi
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+if [ -n "${distributionSha256Sum-}" ]; then
+  distributionSha256Result=false
+  if [ "$MVN_CMD" = mvnd.sh ]; then
+    echo "Checksum validation is not supported for maven-mvnd." >&2
+    echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
+    exit 1
+  elif command -v sha256sum >/dev/null; then
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
+      distributionSha256Result=true
+    fi
+  elif command -v shasum >/dev/null; then
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
+      distributionSha256Result=true
+    fi
+  else
+    echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
+    echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
+    exit 1
+  fi
+  if [ $distributionSha256Result = false ]; then
+    echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
+    echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
+    exit 1
+  fi
+fi
+
+# unzip and move
+if command -v unzip >/dev/null; then
+  unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
+else
+  tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
+fi
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+actualDistributionDir=""
+
+# First try the expected directory name (for regular distributions)
+if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
+  if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
+    actualDistributionDir="$distributionUrlNameMain"
+  fi
+fi
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if [ -z "$actualDistributionDir" ]; then
+  # enable globbing to iterate over items
+  set +f
+  for dir in "$TMP_DOWNLOAD_DIR"/*; do
+    if [ -d "$dir" ]; then
+      if [ -f "$dir/bin/$MVN_CMD" ]; then
+        actualDistributionDir="$(basename "$dir")"
+        break
+      fi
+    fi
+  done
+  set -f
+fi
+
+if [ -z "$actualDistributionDir" ]; then
+  verbose "Contents of $TMP_DOWNLOAD_DIR:"
+  verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
+  die "Could not find Maven distribution directory in extracted archive"
+fi
+
+verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
+mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
+
+clean || :
+exec_maven "$@"

mvnw.cmd

@@ -0,0 +1,189 @@
+<# : batch portion
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM    http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Apache Maven Wrapper startup batch script, version 3.3.4
+@REM
+@REM Optional ENV vars
+@REM   MVNW_REPOURL - repo url base for downloading maven distribution
+@REM   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+@REM   MVNW_VERBOSE - true: enable verbose log; others: silence the output
+@REM ----------------------------------------------------------------------------
+
+@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
+@SET __MVNW_CMD__=
+@SET __MVNW_ERROR__=
+@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
+@SET PSModulePath=
+@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
+  IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
+)
+@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
+@SET __MVNW_PSMODULEP_SAVE=
+@SET __MVNW_ARG0_NAME__=
+@SET MVNW_USERNAME=
+@SET MVNW_PASSWORD=
+@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
+@echo Cannot start maven from wrapper >&2 && exit /b 1
+@GOTO :EOF
+: end batch / begin powershell #>
+
+$ErrorActionPreference = "Stop"
+if ($env:MVNW_VERBOSE -eq "true") {
+  $VerbosePreference = "Continue"
+}
+
+# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
+$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
+if (!$distributionUrl) {
+  Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
+}
+
+switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
+  "maven-mvnd-*" {
+    $USE_MVND = $true
+    $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
+    $MVN_CMD = "mvnd.cmd"
+    break
+  }
+  default {
+    $USE_MVND = $false
+    $MVN_CMD = $script -replace '^mvnw','mvn'
+    break
+  }
+}
+
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
+if ($env:MVNW_REPOURL) {
+  $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
+  $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
+}
+$distributionUrlName = $distributionUrl -replace '^.*/',''
+$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
+
+$MAVEN_M2_PATH = "$HOME/.m2"
+if ($env:MAVEN_USER_HOME) {
+  $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
+}
+
+if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
+    New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
+}
+
+$MAVEN_WRAPPER_DISTS = $null
+if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
+  $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
+} else {
+  $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
+}
+
+$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
+$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
+$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
+
+if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
+  Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+  Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
+  exit $?
+}
+
+if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
+  Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
+}
+
+# prepare tmp dir
+$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
+$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
+$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
+trap {
+  if ($TMP_DOWNLOAD_DIR.Exists) {
+    try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+    catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+  }
+}
+
+New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
+
+# Download and Install Apache Maven
+Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+Write-Verbose "Downloading from: $distributionUrl"
+Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+$webclient = New-Object System.Net.WebClient
+if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
+  $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
+}
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
+if ($distributionSha256Sum) {
+  if ($USE_MVND) {
+    Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
+  }
+  Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
+  if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
+    Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
+  }
+}
+
+# unzip and move
+Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+$actualDistributionDir = ""
+
+# First try the expected directory name (for regular distributions)
+$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
+$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
+if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
+  $actualDistributionDir = $distributionUrlNameMain
+}
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if (!$actualDistributionDir) {
+  Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
+    $testPath = Join-Path $_.FullName "bin/$MVN_CMD"
+    if (Test-Path -Path $testPath -PathType Leaf) {
+      $actualDistributionDir = $_.Name
+    }
+  }
+}
+
+if (!$actualDistributionDir) {
+  Write-Error "Could not find Maven distribution directory in extracted archive"
+}
+
+Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
+try {
+  Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
+} catch {
+  if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
+    Write-Error "fail to move MAVEN_HOME"
+  }
+} finally {
+  try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+  catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+}
+
+Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"

pom.xml

@@ -1,90 +1,265 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-   <modelVersion>4.0.0</modelVersion>
+<!--
+  ~ Copyright 2026 Kingsrook, LLC
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
 
-   <groupId>com.kingsrook.qbits</groupId>
-   <artifactId>qbit-example-app</artifactId>
-   <version>0.1.0-SNAPSHOT</version>
-   <packaging>jar</packaging>
-
-   <name>QBit Example Application</name>
-   <description>Template for Application QBits - complete mini-applications with UI navigation</description>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 
    <properties>
+      <qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
+      <revision>0.1.0-SNAPSHOT</revision>
       <maven.compiler.source>21</maven.compiler.source>
       <maven.compiler.target>21</maven.compiler.target>
-      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-      <qqq.version>4.0.0</qqq.version>
+      <jacoco.version>0.8.15</jacoco.version>
    </properties>
 
+   <parent>
+      <groupId>com.kingsrook</groupId>
+      <artifactId>qbit-build-parent</artifactId>
+      <version>2.2.1</version>
+   </parent>
+
+   <groupId>com.kingsrook.qbits</groupId>
+   <artifactId>qbit-jobs</artifactId>
+   <version>${revision}</version>
+   <packaging>jar</packaging>
+
+   <name>QBit Jobs</name>
+   <description>QBit providing the Jobs app for QQQ durable background jobs: live progress, throughput, failures, history, job record panels, alert types and retention</description>
+   <url>https://github.com/QRun-IO/qbit-jobs</url>
+
+   <modelVersion>4.0.0</modelVersion>
+
+   <licenses>
+      <license>
+         <name>Apache License, Version 2.0</name>
+         <url>https://www.apache.org/licenses/LICENSE-2.0</url>
+         <distribution>repo</distribution>
+      </license>
+   </licenses>
+
+   <developers>
+      <developer>
+         <id>kingsrook</id>
+         <name>Kingsrook Team</name>
+         <email>contact@kingsrook.com</email>
+         <organization>Kingsrook, LLC</organization>
+         <organizationUrl>https://kingsrook.com</organizationUrl>
+      </developer>
+   </developers>
+
+   <scm>
+      <connection>scm:git:git://github.com/QRun-IO/qbit-jobs.git</connection>
+      <developerConnection>scm:git:ssh://github.com:QRun-IO/qbit-jobs.git</developerConnection>
+      <url>https://github.com/QRun-IO/qbit-jobs</url>
+      <tag>HEAD</tag>
+   </scm>
+
    <dependencies>
-      <!-- QQQ Backend Core -->
       <dependency>
          <groupId>com.kingsrook.qqq</groupId>
          <artifactId>qqq-backend-core</artifactId>
-         <version>${qqq.version}</version>
       </dependency>
 
-      <!-- Testing -->
       <dependency>
          <groupId>org.junit.jupiter</groupId>
          <artifactId>junit-jupiter</artifactId>
-         <version>5.10.0</version>
          <scope>test</scope>
       </dependency>
+
       <dependency>
          <groupId>org.assertj</groupId>
          <artifactId>assertj-core</artifactId>
          <version>3.27.7</version>
          <scope>test</scope>
       </dependency>
-      <dependency>
-         <groupId>com.kingsrook.qqq</groupId>
-         <artifactId>qqq-backend-module-rdbms</artifactId>
-         <version>${qqq.version}</version>
-         <scope>test</scope>
-      </dependency>
-      <dependency>
-         <groupId>com.h2database</groupId>
-         <artifactId>h2</artifactId>
-         <version>2.2.224</version>
-         <scope>test</scope>
-      </dependency>
    </dependencies>
 
    <build>
+      <resources>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>true</filtering>
+         </resource>
+      </resources>
       <plugins>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-compiler-plugin</artifactId>
-            <version>3.11.0</version>
-         </plugin>
-         <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-surefire-plugin</artifactId>
-            <version>3.1.2</version>
+            <artifactId>maven-checkstyle-plugin</artifactId>
+            <version>3.3.1</version>
+            <dependencies>
+               <dependency>
+                  <groupId>com.puppycrawl.tools</groupId>
+                  <artifactId>checkstyle</artifactId>
+                  <version>10.12.6</version>
+               </dependency>
+            </dependencies>
+            <configuration>
+               <configLocation>checkstyle/config.xml</configLocation>
+               <headerLocation>checkstyle/license.txt</headerLocation>
+               <consoleOutput>true</consoleOutput>
+               <failOnViolation>true</failOnViolation>
+               <violationSeverity>warning</violationSeverity>
+               <includeTestSourceDirectory>true</includeTestSourceDirectory>
+            </configuration>
+            <executions>
+               <execution>
+                  <id>validate</id>
+                  <phase>validate</phase>
+                  <goals><goal>check</goal></goals>
+               </execution>
+            </executions>
          </plugin>
          <plugin>
             <groupId>org.jacoco</groupId>
             <artifactId>jacoco-maven-plugin</artifactId>
-            <version>0.8.10</version>
+            <version>${jacoco.version}</version>
             <executions>
                <execution>
+                  <id>prepare-agent</id>
                   <goals>
                      <goal>prepare-agent</goal>
                   </goals>
                </execution>
                <execution>
                   <id>report</id>
-                  <phase>test</phase>
+                  <phase>verify</phase>
                   <goals>
                      <goal>report</goal>
                   </goals>
                </execution>
+               <execution>
+                  <id>check</id>
+                  <goals>
+                     <goal>check</goal>
+                  </goals>
+                  <configuration>
+                     <rules>
+                        <rule>
+                           <element>BUNDLE</element>
+                           <limits>
+                              <limit>
+                                 <counter>INSTRUCTION</counter>
+                                 <value>COVEREDRATIO</value>
+                                 <minimum>0.70</minimum>
+                              </limit>
+                              <limit>
+                                 <counter>CLASS</counter>
+                                 <value>COVEREDRATIO</value>
+                                 <minimum>0.90</minimum>
+                              </limit>
+                           </limits>
+                        </rule>
+                     </rules>
+                  </configuration>
+               </execution>
             </executions>
          </plugin>
       </plugins>
    </build>
+
+   <profiles>
+      <!-- Built against qqq 4.1. Until qbit-build-parent imports the released 4.1 BOM, this    -->
+      <!-- profile is active by default and imports qqq-bom-pom 4.1.0-RC.8 ahead of the parent's -->
+      <!-- 4.0.0 BOM (ADR-0007, as in the other qbits).                                          -->
+      <profile>
+         <id>qqq-snapshot</id>
+         <activation>
+            <activeByDefault>true</activeByDefault>
+         </activation>
+         <properties>
+            <qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
+         </properties>
+         <dependencyManagement>
+            <dependencies>
+               <dependency>
+                  <groupId>com.kingsrook.qqq</groupId>
+                  <artifactId>qqq-bom-pom</artifactId>
+                  <version>${qqq.snapshot.version}</version>
+                  <type>pom</type>
+                  <scope>import</scope>
+               </dependency>
+            </dependencies>
+         </dependencyManagement>
+      </profile>
+
+      <!-- Local demo server and screenshots: builds src/demo/java against released QQQ RC.7. -->
+      <!-- The version property remains overridable, as in the default profile. -->
+      <profile>
+         <id>demo</id>
+         <properties>
+            <qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
+         </properties>
+         <dependencyManagement>
+            <dependencies>
+               <dependency>
+                  <groupId>com.kingsrook.qqq</groupId>
+                  <artifactId>qqq-bom-pom</artifactId>
+                  <version>${qqq.snapshot.version}</version>
+                  <type>pom</type>
+                  <scope>import</scope>
+               </dependency>
+            </dependencies>
+         </dependencyManagement>
+         <dependencies>
+            <dependency>
+               <groupId>com.kingsrook.qqq</groupId>
+               <artifactId>qqq-backend-module-rdbms</artifactId>
+               <scope>test</scope>
+            </dependency>
+            <dependency>
+               <groupId>com.kingsrook.qqq</groupId>
+               <artifactId>qqq-middleware-javalin</artifactId>
+               <scope>test</scope>
+            </dependency>
+            <dependency>
+               <groupId>com.kingsrook.qqq</groupId>
+               <artifactId>qqq-frontend-next</artifactId>
+               <version>1.0.0-RC.38</version>
+               <scope>test</scope>
+            </dependency>
+            <dependency>
+               <groupId>com.h2database</groupId>
+               <artifactId>h2</artifactId>
+               <version>2.2.220</version>
+               <scope>test</scope>
+            </dependency>
+         </dependencies>
+         <build>
+            <plugins>
+               <plugin>
+                  <groupId>org.codehaus.mojo</groupId>
+                  <artifactId>build-helper-maven-plugin</artifactId>
+                  <version>3.6.0</version>
+                  <executions>
+                     <execution>
+                        <id>add-demo-source</id>
+                        <phase>generate-test-sources</phase>
+                        <goals>
+                           <goal>add-test-source</goal>
+                        </goals>
+                        <configuration>
+                           <sources>
+                              <source>src/demo/java</source>
+                           </sources>
+                        </configuration>
+                     </execution>
+                  </executions>
+               </plugin>
+            </plugins>
+         </build>
+      </profile>
+   </profiles>
 </project>

src/demo/java/com/kingsrook/qbits/jobs/demo/DemoHistory.java

@@ -0,0 +1,158 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.jobs.demo;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.SystemScope;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+
+
+/*******************************************************************************
+ ** 30 days of finished jobs for the demo's history and throughput widgets:
+ ** realistic processes, sizes, durations and outcomes, with their chunks.
+ *******************************************************************************/
+final class DemoHistory
+{
+   private static final String[][] PROCESSES = {
+      {"demo_customer.bulkInsert", "Import customers", "demo_customer"},
+      {"demo_customer.bulkEdit", "Bulk edit customers", "demo_customer"},
+      {JobsDemoServer.RECALC, "Recalculate customer tiers", "demo_customer"},
+      {"demo_customer.bulkDelete", "Delete inactive customers", "demo_customer"},
+      {JobsDemoServer.NIGHTLY, "Nightly partner export", null}
+   };
+
+
+
+   /*******************************************************************************
+    ** not instantiable
+    *******************************************************************************/
+   private DemoHistory()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert the history (as the system user; the rows are owned by the demo user).
+    *******************************************************************************/
+   static void insert(Random random) throws QException
+   {
+      Instant now = Instant.now();
+      for(int i = 0; i < 140; i++)
+      {
+         String[] process  = PROCESSES[random.nextInt(PROCESSES.length)];
+         boolean  recent   = i < 30;
+         Instant  finished = recent ? now.minus(random.nextInt(23 * 60) + 5, ChronoUnit.MINUTES) : now.minus(1 + random.nextInt(29), ChronoUnit.DAYS).minus(random.nextInt(1440), ChronoUnit.MINUTES);
+         int      rows     = process[0].equals(JobsDemoServer.NIGHTLY) ? 0 : 200 + random.nextInt(18_000);
+         int      roll     = random.nextInt(100);
+         String   outcome  = roll < 78 ? "COMPLETED" : roll < 90 ? "COMPLETED_WITH_ERRORS" : roll < 96 ? "FAILED" : "CANCELLED";
+         String   status   = rows == 0 && "COMPLETED_WITH_ERRORS".equals(outcome) ? "COMPLETED" : outcome;
+         int      failed   = "COMPLETED_WITH_ERRORS".equals(status) ? 1 + random.nextInt(Math.max(1, rows / 200)) : 0;
+         int      chunks   = Math.max(1, (rows + 499) / 500);
+         Instant  started  = finished.minusSeconds(Math.max(4, rows / (40 + random.nextInt(80))));
+         String   error    = "FAILED".equals(status) ? (process[0].equals(JobsDemoServer.NIGHTLY) ? "Partner SFTP host refused the connection" : "Lost database connection during chunk 3; retries exhausted") : null;
+
+         QRecord job = new QRecord()
+            .withValue("label", process[1])
+            .withValue("processName", process[0])
+            .withValue("tableName", process[2])
+            .withValue("mode", "EXECUTE")
+            .withValue("status", status)
+            .withValue("priority", 5)
+            .withValue("userId", JobsDemoServer.USER_ID)
+            .withValue("clientChannel", random.nextInt(5) == 0 ? "assistant/turn-" + (100 + i) : "next-ui")
+            .withValue("sourceType", process[2] == null ? "STEPS" : "TABLE_FILTER")
+            .withValue("idempotency", "NATURAL")
+            .withValue("semantics", "CHUNKED")
+            .withValue("chunkSize", 500)
+            .withValue("maxParallelChunks", 1)
+            .withValue("rowsTotal", rows)
+            .withValue("rowsDone", "CANCELLED".equals(status) ? rows / 3 : rows)
+            .withValue("rowsOk", ("CANCELLED".equals(status) ? rows / 3 : rows) - failed)
+            .withValue("rowsFailed", failed)
+            .withValue("chunksTotal", chunks)
+            .withValue("chunksDone", "FAILED".equals(status) ? Math.max(0, chunks - 1) : chunks)
+            .withValue("chunksFailed", "FAILED".equals(status) ? 1 : 0)
+            .withValue("queuedAt", started.minusSeconds(3))
+            .withValue("startedAt", started)
+            .withValue("finishedAt", finished)
+            .withValue("lastProgressAt", finished)
+            .withValue("errorMessage", error)
+            .withValue("leaseVersion", 0)
+            .withValue("createDate", started.minusSeconds(3))
+            .withValue("modifyDate", finished);
+         Integer jobId = SystemScope.call(() -> new InsertAction().execute(new InsertInput("jobs_job").withRecord(job)).getRecords().get(0).getValueInteger("id"));
+
+         List<QRecord> chunkRecords = new ArrayList<>();
+         long          span         = Math.max(1, finished.getEpochSecond() - started.getEpochSecond());
+         for(int c = 0; c < chunks; c++)
+         {
+            int     chunkRows = Math.min(500, rows - c * 500);
+            Instant chunkEnd  = started.plusSeconds(span * (c + 1) / chunks);
+            chunkRecords.add(new QRecord().withValue("jobId", jobId).withValue("seq", c)
+               .withValue("status", "FAILED".equals(status) && c == chunks - 1 ? "FAILED" : "DONE")
+               .withValue("attempt", 1).withValue("maxAttempts", 3).withValue("leaseVersion", 1)
+               .withValue("rowsIn", Math.max(0, chunkRows)).withValue("rowsOk", Math.max(0, chunkRows)).withValue("rowsFailed", 0)
+               .withValue("startedAt", chunkEnd.minusSeconds(Math.max(1, span / chunks))).withValue("finishedAt", chunkEnd)
+               .withValue("createDate", started).withValue("modifyDate", chunkEnd));
+         }
+         SystemScope.call(() -> new InsertAction().execute(new InsertInput("jobs_chunk").withRecords(chunkRecords)));
+
+         List<QRecord> events = new ArrayList<>();
+         events.add(event(jobId, "LAUNCHED", started.minusSeconds(3), "Launched " + process[0]));
+         events.add(event(jobId, "STARTED", started, "Started on demo-pod-1"));
+         events.add(event(jobId, "COMPLETED".equals(status) || "COMPLETED_WITH_ERRORS".equals(status) ? "COMPLETED" : status, finished, status));
+         SystemScope.call(() -> new InsertAction().execute(new InsertInput("jobs_event").withRecords(events)));
+
+         if(failed > 0)
+         {
+            List<QRecord> errors = new ArrayList<>();
+            for(int e = 0; e < Math.min(failed, 25); e++)
+            {
+               errors.add(new QRecord().withValue("jobId", jobId).withValue("rowNumber", 2 + random.nextInt(Math.max(1, rows)))
+                  .withValue("fieldName", "email").withValue("errorCode", "BadInputStatusMessage").withValue("message", "Customer has no email address")
+                  .withValue("resolved", false).withValue("createDate", finished));
+            }
+            SystemScope.call(() -> new InsertAction().execute(new InsertInput("jobs_row_error").withRecords(errors)));
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   private static QRecord event(Integer jobId, String type, Instant at, String detail)
+   {
+      return (new QRecord().withValue("jobId", jobId).withValue("type", type).withValue("actorUserId", JobsDemoServer.USER_ID)
+         .withValue("pod", "demo-pod-1").withValue("detail", detail).withValue("createDate", at));
+   }
+
+}

src/demo/java/com/kingsrook/qbits/jobs/demo/DemoSchema.java

@@ -0,0 +1,193 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs.demo;
+
+
+/*******************************************************************************
+ ** The demo's H2 schema: core's jobs_* tables (snake_case columns, from the
+ ** Postgres DDL in docs/ddl/postgres.sql) and the demo customer table.
+ *******************************************************************************/
+final class DemoSchema
+{
+   static final String H2_DDL = """
+CREATE TABLE jobs_job
+(
+   id                        INT AUTO_INCREMENT PRIMARY KEY,
+   label                     VARCHAR(250),
+   process_name              VARCHAR(250) NOT NULL,
+   table_name                VARCHAR(250),
+   mode                      VARCHAR(20),
+   status                    VARCHAR(30) NOT NULL,
+   status_reason             VARCHAR(500),
+   priority                  INTEGER DEFAULT 5,
+   user_id                   VARCHAR(250),
+   client_channel            VARCHAR(250),
+   credential_scope_json     VARCHAR(1000000),
+   source_type               VARCHAR(30),
+   source_json               VARCHAR(1000000),
+   params_json               VARCHAR(1000000),
+   idempotency               VARCHAR(30),
+   semantics                 VARCHAR(20),
+   chunk_size                INTEGER,
+   max_parallel_chunks       INTEGER,
+   rows_total                INTEGER,
+   rows_done                 INTEGER,
+   rows_ok                   INTEGER,
+   rows_failed               INTEGER,
+   chunks_total              INTEGER,
+   chunks_done               INTEGER,
+   chunks_failed             INTEGER,
+   summary_json              VARCHAR(1000000),
+   output_storage_json       VARCHAR(1000000),
+   parent_job_id             INTEGER,
+   saved_bulk_load_profile_id INTEGER,
+   notify_email              BOOLEAN DEFAULT FALSE,
+   queued_at                 TIMESTAMP WITH TIME ZONE,
+   started_at                TIMESTAMP WITH TIME ZONE,
+   finished_at               TIMESTAMP WITH TIME ZONE,
+   last_progress_at          TIMESTAMP WITH TIME ZONE,
+   error_message             VARCHAR(1000000),
+   lease_version             INTEGER DEFAULT 0,
+   create_date               TIMESTAMP WITH TIME ZONE,
+   modify_date               TIMESTAMP WITH TIME ZONE
+);
+
+CREATE INDEX jobs_job_claim_idx ON jobs_job (status, priority, queued_at);
+
+CREATE INDEX jobs_job_user_idx ON jobs_job (user_id, status);
+
+CREATE TABLE jobs_chunk
+(
+   id                INT AUTO_INCREMENT PRIMARY KEY,
+   job_id            INTEGER NOT NULL REFERENCES jobs_job (id) ON DELETE CASCADE,
+   seq               INTEGER NOT NULL,
+   status            VARCHAR(30) NOT NULL,
+   range_json        VARCHAR(1000000),
+   input_storage_ref VARCHAR(1000),
+   attempt           INTEGER DEFAULT 0,
+   max_attempts      INTEGER DEFAULT 3,
+   lease_owner       VARCHAR(250),
+   lease_token       VARCHAR(100),
+   lease_expires_at  TIMESTAMP WITH TIME ZONE,
+   heartbeat_at      TIMESTAMP WITH TIME ZONE,
+   rows_in           INTEGER,
+   rows_ok           INTEGER,
+   rows_failed       INTEGER,
+   summary_json      VARCHAR(1000000),
+   checkpoint_json   VARCHAR(1000000),
+   error_message     VARCHAR(1000000),
+   started_at        TIMESTAMP WITH TIME ZONE,
+   finished_at       TIMESTAMP WITH TIME ZONE,
+   lease_version     INTEGER DEFAULT 0,
+   create_date       TIMESTAMP WITH TIME ZONE,
+   modify_date       TIMESTAMP WITH TIME ZONE,
+   UNIQUE (job_id, seq)
+);
+
+CREATE INDEX jobs_chunk_status_idx ON jobs_chunk (job_id, status);
+
+CREATE INDEX jobs_chunk_lease_idx ON jobs_chunk (status, lease_expires_at);
+
+CREATE TABLE jobs_row_error
+(
+   id           INT AUTO_INCREMENT PRIMARY KEY,
+   job_id       INTEGER NOT NULL REFERENCES jobs_job (id) ON DELETE CASCADE,
+   chunk_id     INTEGER,
+   row_number   INTEGER,
+   record_pk    VARCHAR(250),
+   field_name   VARCHAR(250),
+   error_code   VARCHAR(100),
+   message      VARCHAR(1000000),
+   retry_job_id INTEGER,
+   resolved     BOOLEAN DEFAULT FALSE,
+   create_date  TIMESTAMP WITH TIME ZONE
+);
+
+CREATE INDEX jobs_row_error_job_idx ON jobs_row_error (job_id);
+
+CREATE TABLE jobs_event
+(
+   id             INT AUTO_INCREMENT PRIMARY KEY,
+   job_id         INTEGER NOT NULL REFERENCES jobs_job (id) ON DELETE CASCADE,
+   chunk_id       INTEGER,
+   type           VARCHAR(30),
+   actor_user_id  VARCHAR(250),
+   client_channel VARCHAR(250),
+   pod            VARCHAR(250),
+   detail         VARCHAR(1000000),
+   create_date    TIMESTAMP WITH TIME ZONE
+);
+
+CREATE INDEX jobs_event_job_idx ON jobs_event (job_id);
+
+CREATE TABLE jobs_worker
+(
+   id           INT AUTO_INCREMENT PRIMARY KEY,
+   pod_id       VARCHAR(250) NOT NULL UNIQUE,
+   started_at   TIMESTAMP WITH TIME ZONE,
+   heartbeat_at TIMESTAMP WITH TIME ZONE,
+   threads      INTEGER,
+   draining     BOOLEAN,
+   version      VARCHAR(100)
+);
+
+CREATE TABLE jobs_storage
+(
+   reference VARCHAR(1000) PRIMARY KEY,
+   contents  BYTEA
+);
+
+CREATE TABLE jobs_upload
+(
+   id                 INT AUTO_INCREMENT PRIMARY KEY,
+   ref                VARCHAR(100) NOT NULL UNIQUE,
+   user_id            VARCHAR(250),
+   file_name          VARCHAR(250),
+   content_type       VARCHAR(250),
+   size_bytes         BIGINT,
+   storage_table_name VARCHAR(250),
+   storage_reference  VARCHAR(1000),
+   expires_at         TIMESTAMP WITH TIME ZONE,
+   job_id             INTEGER,
+   create_date        TIMESTAMP WITH TIME ZONE
+);
+
+CREATE TABLE demo_customer
+(
+   id             INT AUTO_INCREMENT PRIMARY KEY,
+   name           VARCHAR(250),
+   email          VARCHAR(250),
+   tier           VARCHAR(20),
+   lifetime_value INTEGER
+);
+      """;
+
+
+
+   /*******************************************************************************
+    ** not instantiable
+    *******************************************************************************/
+   private DemoSchema()
+   {
+   }
+
+}

src/demo/java/com/kingsrook/qbits/jobs/demo/JobsDemoServer.java

@@ -0,0 +1,339 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.jobs.demo;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qbits.jobs.JobsQBitConfig;
+import com.kingsrook.qbits.jobs.JobsQBitProducer;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobs;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
+import com.kingsrook.qqq.backend.core.jobs.DurableProcessMetaData;
+import com.kingsrook.qqq.backend.core.jobs.SystemScope;
+import com.kingsrook.qqq.backend.core.jobs.model.JobIdempotency;
+import com.kingsrook.qqq.backend.core.jobs.model.LaunchRequest;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QUser;
+import com.kingsrook.qqq.backend.core.modules.authentication.implementations.FullyAnonymousAuthenticationModule;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaUpdateStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import com.kingsrook.qqq.backend.core.utils.SleepUtils;
+import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager;
+import com.kingsrook.qqq.backend.module.rdbms.jdbc.QueryManager;
+import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
+import com.kingsrook.qqq.middleware.javalin.QApplicationJavalinServer;
+
+
+/*******************************************************************************
+ ** Local demo of the Jobs app (screenshots): an H2 database with the core jobs
+ ** tables, a customer table, real durable jobs running on this pod (one long
+ ** job with live progress, one queued behind it, one failed, one with row
+ ** errors, one awaiting confirmation) and 30 days of finished-job history.
+ **
+ ** Run: ./mvnw -Pdemo -Dqqq.snapshot.version=4.1.0-JOBS-SNAPSHOT test-compile exec:java
+ **        -Dexec.classpathScope=test -Dexec.mainClass=com.kingsrook.qbits.jobs.demo.JobsDemoServer
+ *******************************************************************************/
+public class JobsDemoServer extends AbstractQQQApplication
+{
+   static final String BACKEND   = "rdbms";
+   static final String CUSTOMER  = "demo_customer";
+   static final String RECALC    = "demoRecalculateTiers";
+   static final String NIGHTLY   = "demoNightlyExport";
+   static final String STORAGE   = "demo_jobs_storage";
+   static final String USER_ID   = "demo-user";
+
+
+
+   /*******************************************************************************
+    ** Anonymous, as a named user holding jobs_admin (demo only).
+    *******************************************************************************/
+   public static class DemoAuthenticationModule extends FullyAnonymousAuthenticationModule
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public QSession createSession(QInstance qInstance, Map<String, String> context)
+      {
+         QSession session = super.createSession(qInstance, context);
+         session.setUser(new QUser().withIdReference(USER_ID).withFullName("Demo User"));
+         session.withPermission("jobs_admin");
+         DurableJobsSecurity.applyToSession(session);
+         return (session);
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public QSession createAutomatedSessionForUser(QInstance qInstance, Serializable userId)
+      {
+         QSession session = new QSession().withUser(new QUser().withIdReference(String.valueOf(userId)).withFullName("Demo User"));
+         session.withPermission("jobs_admin");
+         return (session);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   static RDBMSBackendMetaData backend()
+   {
+      RDBMSBackendMetaData backend = new RDBMSBackendMetaData().withName(BACKEND).withVendor("h2").withHostName("mem").withDatabaseName("jobs_demo").withUsername("sa");
+      backend.setJdbcUrl(ConnectionManager.getJdbcUrl(backend) + ";DATABASE_TO_UPPER=FALSE;DB_CLOSE_DELAY=-1");
+      return (backend);
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   @Override
+   public QInstance defineQInstance() throws QException
+   {
+      QInstance instance = new QInstance();
+      instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS));
+      instance.addBackend(backend());
+      instance.addBackend(new com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
+      instance.addTable(new QTableMetaData().withName(STORAGE).withBackendName("memory").withPrimaryKeyField("reference")
+         .withField(new QFieldMetaData("reference", QFieldType.STRING))
+         .withField(new QFieldMetaData("contents", QFieldType.BLOB)));
+
+      instance.addTable(new QTableMetaData().withName(CUSTOMER).withLabel("Customer").withBackendName(BACKEND).withPrimaryKeyField("id")
+         .withRecordLabelFormatAndFields("%s", "name")
+         .withField(new QFieldMetaData("id", QFieldType.INTEGER))
+         .withField(new QFieldMetaData("name", QFieldType.STRING))
+         .withField(new QFieldMetaData("email", QFieldType.STRING))
+         .withField(new QFieldMetaData("tier", QFieldType.STRING))
+         .withField(new QFieldMetaData("lifetimeValue", QFieldType.INTEGER).withBackendName("lifetime_value")));
+      instance.addApp(new QAppMetaData().withName("crm").withLabel("CRM").withIcon(new QIcon("groups")).withChild(instance.getTable(CUSTOMER)));
+
+      QProcessMetaData recalc = StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName(RECALC).withLabel("Recalculate Customer Tiers").withTableName(CUSTOMER)
+         .withSourceTable(CUSTOMER).withDestinationTable(CUSTOMER)
+         .withExtractStepClass(ExtractViaQueryStep.class).withTransformStepClass(RecalcTransform.class).withLoadStepClass(LoadViaUpdateStep.class)
+         .getProcessMetaData();
+      recalc.withSupplementalMetaData(new DurableProcessMetaData().withChunkSize(500).withMaxConcurrentJobs(1).withIdempotency(JobIdempotency.NATURAL));
+      instance.addProcess(recalc);
+
+      QProcessMetaData nightly = new QProcessMetaData().withName(NIGHTLY).withLabel("Nightly Partner Export")
+         .withStepList(List.of(new QBackendStepMetaData().withName("export").withCode(new QCodeReference(FailingExport.class))));
+      nightly.withSupplementalMetaData(new DurableProcessMetaData().withEligible(true));
+      instance.addProcess(nightly);
+
+      new DurableJobsMetaDataProvider().defineAll(instance, BACKEND, DurableJobsMetaDataProvider.snakeCaseColumnNames());
+      instance.withSupplementalMetaData(new DurableJobsMetaData().withStorageTableName(STORAGE).withWorkerThreads(1).withPollIntervalMillis(500).withPodId("demo-pod-1"));
+
+      new JobsQBitProducer().withQBitConfig(new JobsQBitConfig()).produce(instance);
+      instance.setHomeAppName("jobs");
+      return (instance);
+   }
+
+
+
+   /*******************************************************************************
+    ** Doubles lifetime value and sets the tier; slow, so its job shows progress;
+    ** rejects customers with no email.
+    *******************************************************************************/
+   public static class RecalcTransform extends AbstractTransformStep
+   {
+      private final ProcessSummaryLine ok = new ProcessSummaryLine(Status.OK).withSingularPastMessage("customer was updated").withPluralPastMessage("customers were updated");
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+      {
+         for(QRecord record : input.getRecords())
+         {
+            SleepUtils.sleep(12, TimeUnit.MILLISECONDS);
+            int     value   = record.getValueInteger("lifetimeValue") == null ? 0 : record.getValueInteger("lifetimeValue");
+            QRecord updated = new QRecord().withValue("id", record.getValue("id")).withValue("lifetimeValue", value + 10)
+               .withValue("tier", value > 5000 ? "Gold" : value > 1000 ? "Silver" : "Bronze");
+            if(record.getValueString("email") == null)
+            {
+               updated.addError(new com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage("Customer has no email address"));
+            }
+            else
+            {
+               ok.incrementCountAndAddPrimaryKey(record.getValue("id"));
+            }
+            output.addRecord(updated);
+         }
+      }
+
+
+
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
+      {
+         return (new ArrayList<>(List.of(ok)));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An export whose partner endpoint is down.
+    *******************************************************************************/
+   public static class FailingExport implements BackendStep
+   {
+      /*******************************************************************************
+       **
+       *******************************************************************************/
+      @Override
+      public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+      {
+         throw (new QException("Partner SFTP host refused the connection (sftp.partner.example: connection timed out)"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Create the schema, customers, history, and live jobs.
+    *******************************************************************************/
+   static void seed(QInstance instance) throws Exception
+   {
+      try(var connection = ConnectionManager.getConnection(backend()))
+      {
+         for(String statement : DemoSchema.H2_DDL.split(";"))
+         {
+            if(!statement.isBlank())
+            {
+               QueryManager.executeUpdate(connection, statement);
+            }
+         }
+      }
+
+      QSession session = new DemoAuthenticationModule().createSession(instance, Map.of());
+      QContext.init(instance, session);
+      Random        random    = new Random(7);
+      List<QRecord> customers = new ArrayList<>();
+      for(int i = 1; i <= 12_000; i++)
+      {
+         customers.add(new QRecord().withValue("name", "Customer " + i).withValue("email", random.nextInt(250) == 0 ? null : "c" + i + "@example.com")
+            .withValue("tier", "Bronze").withValue("lifetimeValue", random.nextInt(9000)));
+      }
+      for(int i = 0; i < customers.size(); i += 1000)
+      {
+         new InsertAction().execute(new InsertInput(CUSTOMER).withRecords(customers.subList(i, i + 1000)));
+      }
+
+      DemoHistory.insert(random);
+
+      ///////////////////////////////////////////////////////////////////
+      // live jobs: a long one, one queued behind it (cap 1), a failure //
+      ///////////////////////////////////////////////////////////////////
+      DurableJobs.launch(new LaunchRequest(RECALC).withFilter(new QQueryFilter()).withLabel("Recalculate tiers: all customers").withExpectedRows(12_000L));
+      DurableJobs.launch(new LaunchRequest(RECALC).withFilter(new QQueryFilter()).withLabel("Recalculate tiers: quarterly re-run").withPriority(3));
+      DurableJobs.launch(new LaunchRequest(NIGHTLY).withLabel("Nightly partner export"));
+
+      QSession assistant = new DemoAuthenticationModule().createSession(instance, Map.of());
+      assistant.setClientChannel("assistant/turn-42");
+      QContext.init(instance, assistant);
+      DurableJobs.launch(new LaunchRequest(RECALC).withFilter(new QQueryFilter()).withLabel("Assistant: re-tier inactive customers"));
+      QContext.init(instance, session);
+
+      SystemScope.call(() ->
+      {
+         new InsertAction().execute(new InsertInput("jobs_worker").withRecord(new QRecord().withValue("podId", "demo-pod-2").withValue("threads", 2)
+            .withValue("draining", true).withValue("startedAt", Instant.now().minus(3, ChronoUnit.HOURS)).withValue("heartbeatAt", Instant.now().minusSeconds(8)).withValue("version", "0.1.0")));
+         return (null);
+      });
+   }
+
+
+
+   /*******************************************************************************
+    **
+    *******************************************************************************/
+   public static void main(String[] args) throws Exception
+   {
+      int port = args.length > 0 ? Integer.parseInt(args[0]) : 8000;
+      java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC")); // QQQ servers run in UTC
+
+      ///////////////////////////////////////////////////////////////////////
+      // the dispatcher keeps the first module per type: swap in the demo //
+      ///////////////////////////////////////////////////////////////////////
+      java.lang.reflect.Field modules = com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher.class.getDeclaredField("authenticationTypeToModuleClassNameMap");
+      modules.setAccessible(true);
+      @SuppressWarnings("unchecked")
+      Map<String, String> map = (Map<String, String>) modules.get(null);
+      map.put(QAuthenticationType.FULLY_ANONYMOUS.getName(), DemoAuthenticationModule.class.getName());
+
+      JobsDemoServer application = new JobsDemoServer();
+      QInstance      instance    = application.defineQInstance();
+      new QInstanceValidator().validate(instance);
+      seed(instance);
+      new QApplicationJavalinServer(application).withPort(port).withServeFrontendNext(true).start();
+      System.out.println("Jobs demo on http://localhost:" + port + "/app/jobs");
+   }
+
+}

src/main/java/com/kingsrook/qbits/example/ExampleAppQBitConfig.java

@@ -1,164 +0,0 @@
-/*******************************************************************************
- ** Configuration class for the Example Application QBit.
- **
- ** Application QBits support:
- ** - Backend configuration for data storage
- ** - Optional table prefixing for multi-instance deployment
- ** - Feature toggles for optional modules
- ** - Security key prefix for permission scoping
- ** - Integration hooks for other QBits
- *******************************************************************************/
-package com.kingsrook.qbits.example;
-
-
-import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
-import com.kingsrook.qqq.backend.core.utils.StringUtils;
-
-
-public class ExampleAppQBitConfig implements QBitConfig
-{
-   private String  backendName;
-   private String  tableNamePrefix;
-   private Boolean enableChildModule = true;
-   private String  securityKeyPrefix;
-
-   /////////////////////////////////////////////////////////////////////////
-   // Integration hooks - allow host app to inject custom behavior        //
-   /////////////////////////////////////////////////////////////////////////
-   private QCodeReference onEntityCreatedHandler;
-   private QCodeReference customValidationProvider;
-
-
-
-   /*******************************************************************************
-    ** Validate the configuration before the QBit is produced.
-    *******************************************************************************/
-   @Override
-   public void validate(QInstance qInstance, List<String> errors)
-   {
-      if(!StringUtils.hasContent(backendName))
-      {
-         errors.add("backendName is required for ExampleAppQBit");
-      }
-
-      if(qInstance.getBackend(backendName) == null)
-      {
-         errors.add("Backend not found: " + backendName);
-      }
-   }
-
-
-
-   /*******************************************************************************
-    ** Apply prefix to a table name if configured.
-    *******************************************************************************/
-   public String applyPrefix(String tableName)
-   {
-      if(StringUtils.hasContent(tableNamePrefix))
-      {
-         return tableNamePrefix + "_" + tableName;
-      }
-      return tableName;
-   }
-
-
-
-   /*******************************************************************************
-    ** Get the security key for permission checks.
-    *******************************************************************************/
-   public String getSecurityKey(String baseName)
-   {
-      if(StringUtils.hasContent(securityKeyPrefix))
-      {
-         return securityKeyPrefix + "." + baseName;
-      }
-      return baseName;
-   }
-
-
-
-   //////////////////////////////////////////////////////////////////////////////
-   // Getters and fluent setters                                               //
-   //////////////////////////////////////////////////////////////////////////////
-
-   public String getBackendName()
-   {
-      return backendName;
-   }
-
-
-   public ExampleAppQBitConfig withBackendName(String backendName)
-   {
-      this.backendName = backendName;
-      return this;
-   }
-
-
-   public String getTableNamePrefix()
-   {
-      return tableNamePrefix;
-   }
-
-
-   public ExampleAppQBitConfig withTableNamePrefix(String tableNamePrefix)
-   {
-      this.tableNamePrefix = tableNamePrefix;
-      return this;
-   }
-
-
-   public Boolean getEnableChildModule()
-   {
-      return enableChildModule;
-   }
-
-
-   public ExampleAppQBitConfig withEnableChildModule(Boolean enableChildModule)
-   {
-      this.enableChildModule = enableChildModule;
-      return this;
-   }
-
-
-   public String getSecurityKeyPrefix()
-   {
-      return securityKeyPrefix;
-   }
-
-
-   public ExampleAppQBitConfig withSecurityKeyPrefix(String securityKeyPrefix)
-   {
-      this.securityKeyPrefix = securityKeyPrefix;
-      return this;
-   }
-
-
-   public QCodeReference getOnEntityCreatedHandler()
-   {
-      return onEntityCreatedHandler;
-   }
-
-
-   public ExampleAppQBitConfig withOnEntityCreatedHandler(QCodeReference onEntityCreatedHandler)
-   {
-      this.onEntityCreatedHandler = onEntityCreatedHandler;
-      return this;
-   }
-
-
-   public QCodeReference getCustomValidationProvider()
-   {
-      return customValidationProvider;
-   }
-
-
-   public ExampleAppQBitConfig withCustomValidationProvider(QCodeReference customValidationProvider)
-   {
-      this.customValidationProvider = customValidationProvider;
-      return this;
-   }
-}

src/main/java/com/kingsrook/qbits/example/ExampleAppQBitProducer.java

@@ -1,175 +0,0 @@
-/*******************************************************************************
- ** Producer for the Example Application QBit.
- **
- ** Application QBits produce:
- ** - QBitMetaData for identification
- ** - QAppSection for UI navigation (REQUIRED)
- ** - Tables, processes, widgets, and other metadata
- ** - Security configuration
- *******************************************************************************/
-package com.kingsrook.qbits.example;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper;
-import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface;
-import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerOutput;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
-import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
-import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProducer;
-import com.kingsrook.qqq.backend.core.model.metadata.qbits.SourceQBitAware;
-import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
-import com.kingsrook.qbits.example.model.ExampleEntity;
-import com.kingsrook.qbits.example.model.ExampleChildEntity;
-
-
-public class ExampleAppQBitProducer implements QBitProducer
-{
-   public static final String GROUP_ID    = "com.kingsrook.qbits";
-   public static final String ARTIFACT_ID = "qbit-example-app";
-   public static final String VERSION     = "0.1.0";
-
-   private ExampleAppQBitConfig config;
-
-
-
-   /*******************************************************************************
-    ** Produce the QBit metadata into the QInstance.
-    *******************************************************************************/
-   @Override
-   public void produce(QInstance qInstance, String namespace) throws QException
-   {
-      /////////////////////////////////////////////////////////////////////////
-      // Validate configuration                                              //
-      /////////////////////////////////////////////////////////////////////////
-      List<String> errors = new ArrayList<>();
-      config.validate(qInstance, errors);
-      if(!errors.isEmpty())
-      {
-         throw new QException("Configuration errors: " + String.join(", ", errors));
-      }
-
-      /////////////////////////////////////////////////////////////////////////
-      // Create and register QBit identity                                   //
-      /////////////////////////////////////////////////////////////////////////
-      QBitMetaData qBitMetaData = new QBitMetaData()
-         .withGroupId(GROUP_ID)
-         .withArtifactId(ARTIFACT_ID)
-         .withVersion(VERSION)
-         .withNamespace(namespace)
-         .withConfig(config);
-      qInstance.addQBit(qBitMetaData);
-
-      /////////////////////////////////////////////////////////////////////////
-      // Discover and produce all component metadata                         //
-      /////////////////////////////////////////////////////////////////////////
-      List<MetaDataProducerInterface<?>> producers =
-         MetaDataProducerHelper.findProducers(getClass().getPackageName());
-
-      for(MetaDataProducerInterface<?> producer : producers)
-      {
-         ///////////////////////////////////////////////////////////////////////
-         // Skip disabled components                                         //
-         ///////////////////////////////////////////////////////////////////////
-         if(!isProducerEnabled(producer))
-         {
-            continue;
-         }
-
-         MetaDataProducerOutput output = producer.produce(qInstance);
-
-         ///////////////////////////////////////////////////////////////////////
-         // Apply table prefix if configured                                  //
-         ///////////////////////////////////////////////////////////////////////
-         if(output instanceof QTableMetaData table)
-         {
-            String prefixedName = config.applyPrefix(table.getName());
-            table.setName(prefixedName);
-            table.setBackendName(config.getBackendName());
-         }
-
-         ///////////////////////////////////////////////////////////////////////
-         // Mark scope for all SourceQBitAware outputs                        //
-         ///////////////////////////////////////////////////////////////////////
-         if(output instanceof SourceQBitAware sqa)
-         {
-            sqa.setSourceQBitName(qBitMetaData.getName());
-         }
-
-         output.addSelfToInstance(qInstance);
-      }
-
-      /////////////////////////////////////////////////////////////////////////
-      // Register QAppMetaData for UI navigation (REQUIRED for app QBits)    //
-      /////////////////////////////////////////////////////////////////////////
-      qInstance.addApp(produceApp(config, qBitMetaData.getName()));
-   }
-
-
-
-   /*******************************************************************************
-    ** Produce the QAppMetaData for UI navigation.
-    **
-    ** This is REQUIRED for Application QBits - it defines how users navigate
-    ** to the tables, processes, and widgets in the UI.
-    *******************************************************************************/
-   private QAppMetaData produceApp(ExampleAppQBitConfig config, String qbitName)
-   {
-      List<String> tables = new ArrayList<>();
-      tables.add(config.applyPrefix(ExampleEntity.TABLE_NAME));
-
-      if(Boolean.TRUE.equals(config.getEnableChildModule()))
-      {
-         tables.add(config.applyPrefix(ExampleChildEntity.TABLE_NAME));
-      }
-
-      QAppSection section = new QAppSection()
-         .withName(config.applyPrefix("exampleSection"))
-         .withLabel("Example Application")
-         .withIcon(new QIcon().withName("dashboard"))
-         .withTables(tables);
-
-      return new QAppMetaData()
-         .withName(config.applyPrefix("exampleApp"))
-         .withLabel("Example Application")
-         .withIcon(new QIcon().withName("dashboard"))
-         .withSection(section);
-   }
-
-
-
-   /*******************************************************************************
-    ** Check if a producer should be enabled based on configuration.
-    *******************************************************************************/
-   private boolean isProducerEnabled(MetaDataProducerInterface<?> producer)
-   {
-      /////////////////////////////////////////////////////////////////////////
-      // Skip child entity if module disabled                                //
-      /////////////////////////////////////////////////////////////////////////
-      if(!Boolean.TRUE.equals(config.getEnableChildModule()))
-      {
-         if(producer.getClass().getSimpleName().contains("ExampleChildEntity"))
-         {
-            return false;
-         }
-      }
-      return true;
-   }
-
-
-
-   //////////////////////////////////////////////////////////////////////////////
-   // Fluent setters                                                           //
-   //////////////////////////////////////////////////////////////////////////////
-
-   public ExampleAppQBitProducer withConfig(ExampleAppQBitConfig config)
-   {
-      this.config = config;
-      return this;
-   }
-}

src/main/java/com/kingsrook/qbits/example/model/ExampleChildEntity.java

@@ -1,159 +0,0 @@
-/*******************************************************************************
- ** Example child entity demonstrating parent-child relationships.
- **
- ** This entity shows:
- ** - Foreign key relationship to parent entity
- ** - Conditional inclusion via config toggle
- *******************************************************************************/
-package com.kingsrook.qbits.example.model;
-
-
-import java.time.Instant;
-import com.kingsrook.qqq.backend.core.model.data.QField;
-import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
-import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.QMetaDataProducingEntity;
-
-
-@QMetaDataProducingEntity(producePossibleValueSource = true)
-public class ExampleChildEntity extends QRecordEntity
-{
-   public static final String TABLE_NAME = "exampleChildEntity";
-
-   @QField(isPrimaryKey = true)
-   private Integer id;
-
-   @QField(isRequired = true, possibleValueSourceName = "exampleEntity")
-   private Integer exampleEntityId;
-
-   @QField(isRequired = true, maxLength = 100)
-   private String name;
-
-   @QField(maxLength = 500)
-   private String description;
-
-   @QField
-   private Integer sortOrder;
-
-   @QField
-   private Boolean isActive = true;
-
-   @QField
-   private Instant createDate;
-
-   @QField
-   private Instant modifyDate;
-
-
-
-   //////////////////////////////////////////////////////////////////////////////
-   // Fluent setters                                                           //
-   //////////////////////////////////////////////////////////////////////////////
-
-   public ExampleChildEntity withId(Integer id)
-   {
-      this.id = id;
-      return this;
-   }
-
-
-   public ExampleChildEntity withExampleEntityId(Integer exampleEntityId)
-   {
-      this.exampleEntityId = exampleEntityId;
-      return this;
-   }
-
-
-   public ExampleChildEntity withName(String name)
-   {
-      this.name = name;
-      return this;
-   }
-
-
-   public ExampleChildEntity withDescription(String description)
-   {
-      this.description = description;
-      return this;
-   }
-
-
-   public ExampleChildEntity withSortOrder(Integer sortOrder)
-   {
-      this.sortOrder = sortOrder;
-      return this;
-   }
-
-
-   public ExampleChildEntity withIsActive(Boolean isActive)
-   {
-      this.isActive = isActive;
-      return this;
-   }
-
-
-   public ExampleChildEntity withCreateDate(Instant createDate)
-   {
-      this.createDate = createDate;
-      return this;
-   }
-
-
-   public ExampleChildEntity withModifyDate(Instant modifyDate)
-   {
-      this.modifyDate = modifyDate;
-      return this;
-   }
-
-
-
-   //////////////////////////////////////////////////////////////////////////////
-   // Getters                                                                  //
-   //////////////////////////////////////////////////////////////////////////////
-
-   public Integer getId()
-   {
-      return id;
-   }
-
-
-   public Integer getExampleEntityId()
-   {
-      return exampleEntityId;
-   }
-
-
-   public String getName()
-   {
-      return name;
-   }
-
-
-   public String getDescription()
-   {
-      return description;
-   }
-
-
-   public Integer getSortOrder()
-   {
-      return sortOrder;
-   }
-
-
-   public Boolean getIsActive()
-   {
-      return isActive;
-   }
-
-
-   public Instant getCreateDate()
-   {
-      return createDate;
-   }
-
-
-   public Instant getModifyDate()
-   {
-      return modifyDate;
-   }
-}

src/main/java/com/kingsrook/qbits/example/model/ExampleEntity.java

@@ -1,145 +0,0 @@
-/*******************************************************************************
- ** Example entity for an Application QBit.
- **
- ** This entity demonstrates:
- ** - Standard QQQ entity patterns
- ** - PossibleValueSource generation
- ** - Fields with various types and constraints
- *******************************************************************************/
-package com.kingsrook.qbits.example.model;
-
-
-import java.time.Instant;
-import com.kingsrook.qqq.backend.core.model.data.QField;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.model.data.QRecordEntity;
-import com.kingsrook.qqq.backend.core.model.metadata.producers.annotations.QMetaDataProducingEntity;
-
-
-@QMetaDataProducingEntity(producePossibleValueSource = true)
-public class ExampleEntity extends QRecordEntity
-{
-   public static final String TABLE_NAME = "exampleEntity";
-
-   @QField(isPrimaryKey = true)
-   private Integer id;
-
-   @QField(isRequired = true, maxLength = 100)
-   private String name;
-
-   @QField(maxLength = 500)
-   private String description;
-
-   @QField(label = "Status")
-   private String status;
-
-   @QField
-   private Boolean isActive = true;
-
-   @QField
-   private Instant createDate;
-
-   @QField
-   private Instant modifyDate;
-
-
-
-   //////////////////////////////////////////////////////////////////////////////
-   // Fluent setters                                                           //
-   //////////////////////////////////////////////////////////////////////////////
-
-   public ExampleEntity withId(Integer id)
-   {
-      this.id = id;
-      return this;
-   }
-
-
-   public ExampleEntity withName(String name)
-   {
-      this.name = name;
-      return this;
-   }
-
-
-   public ExampleEntity withDescription(String description)
-   {
-      this.description = description;
-      return this;
-   }
-
-
-   public ExampleEntity withStatus(String status)
-   {
-      this.status = status;
-      return this;
-   }
-
-
-   public ExampleEntity withIsActive(Boolean isActive)
-   {
-      this.isActive = isActive;
-      return this;
-   }
-
-
-   public ExampleEntity withCreateDate(Instant createDate)
-   {
-      this.createDate = createDate;
-      return this;
-   }
-
-
-   public ExampleEntity withModifyDate(Instant modifyDate)
-   {
-      this.modifyDate = modifyDate;
-      return this;
-   }
-
-
-
-   //////////////////////////////////////////////////////////////////////////////
-   // Getters                                                                  //
-   //////////////////////////////////////////////////////////////////////////////
-
-   public Integer getId()
-   {
-      return id;
-   }
-
-
-   public String getName()
-   {
-      return name;
-   }
-
-
-   public String getDescription()
-   {
-      return description;
-   }
-
-
-   public String getStatus()
-   {
-      return status;
-   }
-
-
-   public Boolean getIsActive()
-   {
-      return isActive;
-   }
-
-
-   public Instant getCreateDate()
-   {
-      return createDate;
-   }
-
-
-   public Instant getModifyDate()
-   {
-      return modifyDate;
-   }
-}

src/main/java/com/kingsrook/qbits/example/processes/ExampleProcessMetaDataProducer.java

@@ -1,46 +0,0 @@
-/*******************************************************************************
- ** MetaData producer for the example process.
- *******************************************************************************/
-package com.kingsrook.qbits.example.processes;
-
-
-import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QFunctionInputMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
-import com.kingsrook.qbits.example.model.ExampleEntity;
-
-
-public class ExampleProcessMetaDataProducer implements MetaDataProducerInterface<QProcessMetaData>
-{
-   public static final String NAME = "exampleProcess";
-
-
-
-   /*******************************************************************************
-    ** Produce the process metadata.
-    *******************************************************************************/
-   @Override
-   public QProcessMetaData produce(QInstance qInstance) throws QException
-   {
-      return new QProcessMetaData()
-         .withName(NAME)
-         .withLabel("Example Process")
-         .withTableName(ExampleEntity.TABLE_NAME)
-         .withStepList(List.of(
-            new QBackendStepMetaData()
-               .withName("process")
-               .withCode(new QCodeReference(ExampleProcessStep.class))
-               .withInputData(new QFunctionInputMetaData()
-                  .withField(new QFieldMetaData("confirmAction", QFieldType.BOOLEAN)
-                     .withLabel("Confirm Action")
-                     .withDefaultValue(false)))
-         ));
-   }
-}

src/main/java/com/kingsrook/qbits/example/processes/ExampleProcessStep.java

@@ -1,67 +0,0 @@
-/*******************************************************************************
- ** Example process step demonstrating business logic implementation.
- **
- ** This shows:
- ** - Standard process step patterns
- ** - Input/output handling
- ** - Logging conventions
- *******************************************************************************/
-package com.kingsrook.qbits.example.processes;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.logging.QLogger;
-import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
-import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
-import com.kingsrook.qqq.backend.core.model.data.QRecord;
-import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
-import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
-
-
-public class ExampleProcessStep extends AbstractTransformStep
-{
-   private static final QLogger LOG = QLogger.getLogger(ExampleProcessStep.class);
-
-
-
-   /*******************************************************************************
-    ** Run the process step.
-    *******************************************************************************/
-   @Override
-   public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
-   {
-      List<QRecord> records = input.getRecords();
-      LOG.info("Processing records", logPair("count", records.size()));
-
-      for(QRecord record : records)
-      {
-         ///////////////////////////////////////////////////////////////////////
-         // Example business logic - update status                            //
-         ///////////////////////////////////////////////////////////////////////
-         String currentStatus = record.getValueString("status");
-         if("PENDING".equals(currentStatus))
-         {
-            record.setValue("status", "PROCESSED");
-         }
-
-         output.addRecord(record);
-      }
-
-      LOG.info("Processing complete", logPair("processed", records.size()));
-   }
-
-
-
-   /*******************************************************************************
-    ** Return the process summary. Override this method in subclasses to provide
-    ** meaningful summary lines for the result screen.
-    *******************************************************************************/
-   @Override
-   public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen)
-   {
-      return (new ArrayList<>());
-   }
-}

src/main/java/com/kingsrook/qbits/example/widgets/ExampleDashboardWidgetMetaDataProducer.java

@@ -1,35 +0,0 @@
-/*******************************************************************************
- ** MetaData producer for an example dashboard widget.
- **
- ** Widgets provide dashboard components for Application QBits.
- *******************************************************************************/
-package com.kingsrook.qbits.example.widgets;
-
-
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
-import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
-import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
-import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
-import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface;
-
-
-public class ExampleDashboardWidgetMetaDataProducer implements MetaDataProducerInterface<QWidgetMetaData>
-{
-   public static final String NAME = "exampleDashboardWidget";
-
-
-
-   /*******************************************************************************
-    ** Produce the widget metadata.
-    *******************************************************************************/
-   @Override
-   public QWidgetMetaData produce(QInstance qInstance) throws QException
-   {
-      return new QWidgetMetaData()
-         .withName(NAME)
-         .withLabel("Example Dashboard")
-         .withType(WidgetType.STATISTICS.getType())
-         .withCodeReference(new QCodeReference(ExampleDashboardWidgetRenderer.class));
-   }
-}

src/main/java/com/kingsrook/qbits/example/widgets/ExampleDashboardWidgetRenderer.java

@@ -1,33 +0,0 @@
-/*******************************************************************************
- ** Renderer for the example dashboard widget.
- **
- ** This class produces the data displayed in the widget.
- *******************************************************************************/
-package com.kingsrook.qbits.example.widgets;
-
-
-import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
-import com.kingsrook.qqq.backend.core.exceptions.QException;
-import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
-import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
-import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData;
-
-
-public class ExampleDashboardWidgetRenderer extends AbstractWidgetRenderer
-{
-   /*******************************************************************************
-    ** Render the widget data.
-    *******************************************************************************/
-   @Override
-   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
-   {
-      /////////////////////////////////////////////////////////////////////////
-      // Add chart data, counts, or other dashboard information here         //
-      /////////////////////////////////////////////////////////////////////////
-      StatisticsData data = new StatisticsData()
-         .withCount(0)
-         .withCountContext("Total items");
-
-      return (new RenderWidgetOutput(data));
-   }
-}

src/main/java/com/kingsrook/qbits/jobs/JobsCoreContract.java

@@ -0,0 +1,225 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsTables;
+import com.kingsrook.qqq.backend.core.jobs.effects.EffectStatus;
+import com.kingsrook.qqq.backend.core.jobs.effects.JobEffectType;
+import com.kingsrook.qqq.backend.core.jobs.model.ChunkStatus;
+import com.kingsrook.qqq.backend.core.jobs.model.JobStatus;
+import com.kingsrook.qqq.backend.core.jobs.runtime.JobSignals;
+import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
+
+
+/*******************************************************************************
+ ** Core identifiers and field names used by the Jobs app.
+ *******************************************************************************/
+public final class JobsCoreContract
+{
+   // Optional progress and log contract (core 4.1.0-RC.8).
+   public static final String JOBS_LOG = DurableJobsTables.TABLE_LOG;
+   public static final String CURRENT_PHASE = "currentPhase";
+   public static final String PERCENT_DONE = "percentDone";
+   public static final String PROGRESS_MESSAGE = "progressMessage";
+   public static final String BYTES_DONE = "bytesDone";
+   public static final String BYTES_TOTAL = "bytesTotal";
+   public static final String ETA_SECONDS = "etaSeconds";
+   public static final String LOG_SEQ = "logSeq";
+   public static final String LOG_DROPPED = "logDropped";
+   public static final String PROGRESS_JSON = "progressJson";
+   public static final String LEVEL = "level";
+   public static final String LOGGED_AT = "loggedAt";
+   public static final String PHASE = "phase";
+
+   public static final String JOBS_JOB = DurableJobsTables.TABLE_JOB;
+   public static final String JOBS_CHUNK = DurableJobsTables.TABLE_CHUNK;
+   public static final String JOBS_ROW_ERROR = DurableJobsTables.TABLE_ROW_ERROR;
+   public static final String JOBS_EVENT = DurableJobsTables.TABLE_EVENT;
+   public static final String JOBS_WORKER = DurableJobsTables.TABLE_WORKER;
+   public static final String JOBS_UPLOAD = DurableJobsTables.TABLE_UPLOAD;
+   public static final String JOBS_USER = DurableJobsTables.SECURITY_KEY_USER;
+   public static final String JOBS_USER_ALL_ACCESS = DurableJobsTables.SECURITY_KEY_USER_ALL_ACCESS;
+   public static final String JOBS_ADMIN = DurableJobsTables.PERMISSION_ADMIN;
+   public static final String ALERT_JOB_FAILED = JobSignals.ALERT_JOB_FAILED;
+   public static final String ALERT_JOB_STALLED = JobSignals.ALERT_JOB_STALLED;
+   public static final String HEARTBEAT_REAPER = JobSignals.HEARTBEAT_REAPER;
+   public static final String DURABLE_JOBS_METADATA = DurableJobsMetaData.NAME;
+   public static final String AUDIT = AuditsMetaDataProvider.TABLE_NAME_AUDIT;
+   public static final String ID = "id";
+   public static final String LABEL = "label";
+   public static final String PROCESS_NAME = "processName";
+   public static final String TABLE_NAME = "tableName";
+   public static final String MODE = "mode";
+   public static final String STATUS = "status";
+   public static final String STATUS_REASON = "statusReason";
+   public static final String PRIORITY = "priority";
+   public static final String USER_ID = "userId";
+   public static final String CLIENT_CHANNEL = "clientChannel";
+   public static final String CREDENTIAL_SCOPE_JSON = "credentialScopeJson";
+   public static final String SOURCE_TYPE = "sourceType";
+   public static final String SOURCE_JSON = "sourceJson";
+   public static final String PARAMS_JSON = "paramsJson";
+   public static final String IDEMPOTENCY = "idempotency";
+   public static final String SEMANTICS = "semantics";
+   public static final String CHUNK_SIZE = "chunkSize";
+   public static final String MAX_PARALLEL_CHUNKS = "maxParallelChunks";
+   public static final String ROWS_TOTAL = "rowsTotal";
+   public static final String ROWS_DONE = "rowsDone";
+   public static final String ROWS_OK = "rowsOk";
+   public static final String ROWS_FAILED = "rowsFailed";
+   public static final String CHUNKS_TOTAL = "chunksTotal";
+   public static final String CHUNKS_DONE = "chunksDone";
+   public static final String CHUNKS_FAILED = "chunksFailed";
+   public static final String SUMMARY_JSON = "summaryJson";
+   public static final String OUTPUT_STORAGE_JSON = "outputStorageJson";
+   public static final String PARENT_JOB_ID = "parentJobId";
+   public static final String SAVED_BULK_LOAD_PROFILE_ID = "savedBulkLoadProfileId";
+   public static final String NOTIFY_EMAIL = "notifyEmail";
+   public static final String QUEUED_AT = "queuedAt";
+   public static final String STARTED_AT = "startedAt";
+   public static final String FINISHED_AT = "finishedAt";
+   public static final String LAST_PROGRESS_AT = "lastProgressAt";
+   public static final String ERROR_MESSAGE = "errorMessage";
+   public static final String LEASE_VERSION = "leaseVersion";
+   public static final String CREATE_DATE = "createDate";
+   public static final String MODIFY_DATE = "modifyDate";
+   public static final String JOB_ID = "jobId";
+   public static final String SEQ = "seq";
+   public static final String RANGE_JSON = "rangeJson";
+   public static final String INPUT_STORAGE_REF = "inputStorageRef";
+   public static final String ATTEMPT = "attempt";
+   public static final String MAX_ATTEMPTS = "maxAttempts";
+   public static final String LEASE_OWNER = "leaseOwner";
+   public static final String LEASE_TOKEN = "leaseToken";
+   public static final String LEASE_EXPIRES_AT = "leaseExpiresAt";
+   public static final String HEARTBEAT_AT = "heartbeatAt";
+   public static final String ROWS_IN = "rowsIn";
+   public static final String CHECKPOINT_JSON = "checkpointJson";
+   public static final String CHUNK_ID = "chunkId";
+   public static final String ROW_NUMBER = "rowNumber";
+   public static final String RECORD_PK = "recordPk";
+   public static final String FIELD_NAME = "fieldName";
+   public static final String ERROR_CODE = "errorCode";
+   public static final String MESSAGE = "message";
+   public static final String RETRY_JOB_ID = "retryJobId";
+   public static final String RESOLVED = "resolved";
+   public static final String TYPE = "type";
+   public static final String ACTOR_USER_ID = "actorUserId";
+   public static final String POD = "pod";
+   public static final String DETAIL = "detail";
+   public static final String POD_ID = "podId";
+   public static final String THREADS = "threads";
+   public static final String DRAINING = "draining";
+   public static final String VERSION = "version";
+   public static final String REF = "ref";
+   public static final String FILE_NAME = "fileName";
+   public static final String CONTENT_TYPE = "contentType";
+   public static final String SIZE_BYTES = "sizeBytes";
+   public static final String STORAGE_TABLE_NAME = "storageTableName";
+   public static final String STORAGE_REFERENCE = "storageReference";
+   public static final String EXPIRES_AT = "expiresAt";
+   public static final String PLANNING = JobStatus.PLANNING.getPossibleValueId();
+   public static final String AWAITING_CONFIRMATION = JobStatus.AWAITING_CONFIRMATION.getPossibleValueId();
+   public static final String QUEUED = JobStatus.QUEUED.getPossibleValueId();
+   public static final String RUNNING = JobStatus.RUNNING.getPossibleValueId();
+   public static final String PAUSING = JobStatus.PAUSING.getPossibleValueId();
+   public static final String PAUSED = JobStatus.PAUSED.getPossibleValueId();
+   public static final String CANCELLING = JobStatus.CANCELLING.getPossibleValueId();
+   public static final String CANCELLED = JobStatus.CANCELLED.getPossibleValueId();
+   public static final String COMPLETED = JobStatus.COMPLETED.getPossibleValueId();
+   public static final String COMPLETED_WITH_ERRORS = JobStatus.COMPLETED_WITH_ERRORS.getPossibleValueId();
+   public static final String FAILED = JobStatus.FAILED.getPossibleValueId();
+   public static final String NEEDS_REVIEW = JobStatus.NEEDS_REVIEW.getPossibleValueId();
+   public static final String DONE = ChunkStatus.DONE.getPossibleValueId();
+
+   public static final String JOBS_EFFECT = DurableJobsTables.TABLE_EFFECT;
+   public static final String JOBS_EFFECT_STATUS = EffectStatus.NAME;
+   public static final String JOBS_EFFECT_TYPE = JobEffectType.NAME;
+   public static final String ALERT_EFFECT_UNRESOLVED = JobSignals.ALERT_EFFECT_UNRESOLVED;
+   public static final String EFFECT_NAME = "effectName";
+   public static final String NEEDS_REVIEW_FIELD = "needsReview";
+   public static final String ROW_KEY = "rowKey";
+   public static final String TARGET_NAME = "targetName";
+   public static final String EFFECT_TYPE = "effectType";
+   public static final String CHUNK_SEQ = "chunkSeq";
+   public static final String IDEMPOTENCY_KEY = "idempotencyKey";
+   public static final String PAYLOAD_JSON = "payloadJson";
+   public static final String PAYLOAD_REF = "payloadRef";
+   public static final String RECONCILE_ATTEMPTS = "reconcileAttempts";
+   public static final String NEXT_ATTEMPT_AT = "nextAttemptAt";
+   public static final String PROVIDER_REF = "providerRef";
+   public static final String LAST_ERROR = "lastError";
+   public static final String SENT_AT = "sentAt";
+   public static final String PENDING = EffectStatus.PENDING.getPossibleValueId();
+   public static final String SENDING = EffectStatus.SENDING.getPossibleValueId();
+   public static final String SENT = EffectStatus.SENT.getPossibleValueId();
+   public static final String UNKNOWN = EffectStatus.UNKNOWN.getPossibleValueId();
+   public static final String LOCAL_TX = JobEffectType.LOCAL_TX.getPossibleValueId();
+   public static final String IDEMPOTENT_EXTERNAL = JobEffectType.IDEMPOTENT_EXTERNAL.getPossibleValueId();
+   public static final String NON_IDEMPOTENT_EXTERNAL = JobEffectType.NON_IDEMPOTENT_EXTERNAL.getPossibleValueId();
+
+   public static final List<String> DATA_TABLES = List.of(JOBS_JOB, JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EFFECT, JOBS_EVENT, JOBS_WORKER);
+   public static final List<String> ALL_TABLES = List.of(JOBS_JOB, JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EVENT, JOBS_WORKER, JOBS_UPLOAD);
+   public static final List<String> RUNNING_STATUSES = List.of(RUNNING, PAUSING, CANCELLING, PLANNING);
+   public static final List<String> QUEUED_STATUSES = List.of(QUEUED, AWAITING_CONFIRMATION);
+   public static final List<String> ATTENTION_STATUSES = List.of(FAILED, NEEDS_REVIEW, COMPLETED_WITH_ERRORS);
+   public static final List<String> TERMINAL_STATUSES = List.of(COMPLETED, COMPLETED_WITH_ERRORS, FAILED, CANCELLED);
+
+   /*******************************************************************************
+    ** Constants only.
+    *******************************************************************************/
+   private JobsCoreContract()
+   {
+   }
+
+   /*******************************************************************************
+    ** The core's inferred child join name.
+    *******************************************************************************/
+   public static String childJoin(String child)
+   {
+      return (DurableJobsMetaDataProvider.childJoinName(child));
+   }
+
+   /*******************************************************************************
+    ** The display label of a job status, as core's JobStatus labels it (the
+    ** record screens show the same words). Unknown values pass through.
+    *******************************************************************************/
+   public static String statusLabel(String status)
+   {
+      if(status == null)
+      {
+         return ("Unknown");
+      }
+      try
+      {
+         return (JobStatus.valueOf(status).getPossibleValueLabel());
+      }
+      catch(IllegalArgumentException e)
+      {
+         return (status);
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/JobsIntegrations.java

@@ -0,0 +1,63 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.util.List;
+
+
+/*******************************************************************************
+ ** Opt-in table registrations, without dependencies on the consuming qbits.
+ *******************************************************************************/
+public final class JobsIntegrations
+{
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private JobsIntegrations()
+   {
+   }
+
+   /*******************************************************************************
+    ** Main table publishing events.
+    *******************************************************************************/
+   public static List<String> esbTableNames()
+   {
+      return (List.of(JobsCoreContract.JOBS_JOB));
+   }
+
+   /*******************************************************************************
+    ** Main searchable table.
+    *******************************************************************************/
+   public static List<String> quickSearchTables()
+   {
+      return (List.of(JobsCoreContract.JOBS_JOB));
+   }
+
+   /*******************************************************************************
+    ** Reportable job data, including the optional effects outbox.
+    *******************************************************************************/
+   public static List<String> dataAnalysisTables()
+   {
+      return (JobsCoreContract.DATA_TABLES);
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/JobsQBitConfig.java

@@ -0,0 +1,268 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
+import org.quartz.CronExpression;
+
+
+/*******************************************************************************
+ ** Host configuration for the Jobs app and its optional scheduled retention.
+ *******************************************************************************/
+public class JobsQBitConfig implements QBitConfig
+{
+   private String appName = "jobs";
+   private String parentAppName;
+   private int retentionDays = 30;
+   private int inputRetentionDays = 7;
+   private String purgeCronExpression = "0 30 3 * * ?";
+   private int homeRefreshSeconds = 5;
+   private int recordRefreshSeconds = 3;
+   private String schedulerName = null;
+
+   /*******************************************************************************
+    ** Get the optional host parent app.
+    *******************************************************************************/
+   public String getParentAppName()
+   {
+      return (parentAppName);
+   }
+
+   /*******************************************************************************
+    ** Set the optional host parent app.
+    *******************************************************************************/
+   public void setParentAppName(String parentAppName)
+   {
+      this.parentAppName = parentAppName;
+   }
+
+   /*******************************************************************************
+    ** Place Jobs under an existing host app, such as Operations under Admin.
+    *******************************************************************************/
+   public JobsQBitConfig withParentAppName(String parentAppName)
+   {
+      this.parentAppName = parentAppName;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Report invalid settings before adding any metadata.
+    *******************************************************************************/
+   @Override
+   public void validate(QInstance instance, List<String> errors)
+   {
+      assertCondition(appName != null && appName.matches("[A-Za-z][A-Za-z0-9_]*"), "appName must be a valid name", errors);
+      assertCondition(parentAppName == null || (!parentAppName.equals(appName) && instance.getApp(parentAppName) != null), "parentAppName must name an existing, different app", errors);
+      assertCondition(retentionDays > 0, "retentionDays must be positive", errors);
+      assertCondition(inputRetentionDays > 0 && inputRetentionDays <= retentionDays, "inputRetentionDays must be positive and no greater than retentionDays", errors);
+      assertCondition(homeRefreshSeconds > 0 && recordRefreshSeconds > 0, "refresh seconds must be positive", errors);
+      assertCondition(purgeCronExpression != null && CronExpression.isValidExpression(purgeCronExpression), "purgeCronExpression must be a valid Quartz cron", errors);
+      assertCondition(schedulerName == null || !schedulerName.isBlank(), "schedulerName must be null or nonblank", errors);
+   }
+
+   /*******************************************************************************
+    ** The permission a user needs to see the Jobs app and its home widgets,
+    ** "<appName>.hasAccess" (default "jobs.hasAccess"). Hosts must grant it.
+    *******************************************************************************/
+   public String getHasAccessPermissionName()
+   {
+      return (appName + ".hasAccess");
+   }
+
+   /*******************************************************************************
+    ** Get appName.
+    *******************************************************************************/
+   public String getAppName()
+   {
+      return (appName);
+   }
+
+   /*******************************************************************************
+    ** Set appName.
+    *******************************************************************************/
+   public void setAppName(String appName)
+   {
+      this.appName = appName;
+   }
+
+   /*******************************************************************************
+    ** Configure appName.
+    *******************************************************************************/
+   public JobsQBitConfig withAppName(String appName)
+   {
+      this.appName = appName;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Get retentionDays.
+    *******************************************************************************/
+   public int getRetentionDays()
+   {
+      return (retentionDays);
+   }
+
+   /*******************************************************************************
+    ** Set retentionDays.
+    *******************************************************************************/
+   public void setRetentionDays(int retentionDays)
+   {
+      this.retentionDays = retentionDays;
+   }
+
+   /*******************************************************************************
+    ** Configure retentionDays.
+    *******************************************************************************/
+   public JobsQBitConfig withRetentionDays(int retentionDays)
+   {
+      this.retentionDays = retentionDays;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Get inputRetentionDays.
+    *******************************************************************************/
+   public int getInputRetentionDays()
+   {
+      return (inputRetentionDays);
+   }
+
+   /*******************************************************************************
+    ** Set inputRetentionDays.
+    *******************************************************************************/
+   public void setInputRetentionDays(int inputRetentionDays)
+   {
+      this.inputRetentionDays = inputRetentionDays;
+   }
+
+   /*******************************************************************************
+    ** Configure inputRetentionDays.
+    *******************************************************************************/
+   public JobsQBitConfig withInputRetentionDays(int inputRetentionDays)
+   {
+      this.inputRetentionDays = inputRetentionDays;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Get purgeCronExpression.
+    *******************************************************************************/
+   public String getPurgeCronExpression()
+   {
+      return (purgeCronExpression);
+   }
+
+   /*******************************************************************************
+    ** Set purgeCronExpression.
+    *******************************************************************************/
+   public void setPurgeCronExpression(String purgeCronExpression)
+   {
+      this.purgeCronExpression = purgeCronExpression;
+   }
+
+   /*******************************************************************************
+    ** Configure purgeCronExpression.
+    *******************************************************************************/
+   public JobsQBitConfig withPurgeCronExpression(String purgeCronExpression)
+   {
+      this.purgeCronExpression = purgeCronExpression;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Get homeRefreshSeconds.
+    *******************************************************************************/
+   public int getHomeRefreshSeconds()
+   {
+      return (homeRefreshSeconds);
+   }
+
+   /*******************************************************************************
+    ** Set homeRefreshSeconds.
+    *******************************************************************************/
+   public void setHomeRefreshSeconds(int homeRefreshSeconds)
+   {
+      this.homeRefreshSeconds = homeRefreshSeconds;
+   }
+
+   /*******************************************************************************
+    ** Configure homeRefreshSeconds.
+    *******************************************************************************/
+   public JobsQBitConfig withHomeRefreshSeconds(int homeRefreshSeconds)
+   {
+      this.homeRefreshSeconds = homeRefreshSeconds;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Get recordRefreshSeconds.
+    *******************************************************************************/
+   public int getRecordRefreshSeconds()
+   {
+      return (recordRefreshSeconds);
+   }
+
+   /*******************************************************************************
+    ** Set recordRefreshSeconds.
+    *******************************************************************************/
+   public void setRecordRefreshSeconds(int recordRefreshSeconds)
+   {
+      this.recordRefreshSeconds = recordRefreshSeconds;
+   }
+
+   /*******************************************************************************
+    ** Configure recordRefreshSeconds.
+    *******************************************************************************/
+   public JobsQBitConfig withRecordRefreshSeconds(int recordRefreshSeconds)
+   {
+      this.recordRefreshSeconds = recordRefreshSeconds;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Get schedulerName.
+    *******************************************************************************/
+   public String getSchedulerName()
+   {
+      return (schedulerName);
+   }
+
+   /*******************************************************************************
+    ** Set schedulerName.
+    *******************************************************************************/
+   public void setSchedulerName(String schedulerName)
+   {
+      this.schedulerName = schedulerName;
+   }
+
+   /*******************************************************************************
+    ** Configure schedulerName.
+    *******************************************************************************/
+   public JobsQBitConfig withSchedulerName(String schedulerName)
+   {
+      this.schedulerName = schedulerName;
+      return (this);
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/JobsQBitProducer.java

@@ -0,0 +1,299 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import com.kingsrook.qbits.jobs.processes.JobsPurgeFinishedStep;
+import com.kingsrook.qbits.jobs.widgets.JobsChildRecordListRenderer;
+import com.kingsrook.qbits.jobs.widgets.JobsWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
+import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
+import com.kingsrook.qqq.backend.core.alerts.AlertDefaultMonitor;
+import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
+import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
+import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ALERT_EFFECT_UNRESOLVED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ALERT_JOB_FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ALERT_JOB_STALLED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ALL_TABLES;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.AUDIT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.HEARTBEAT_REAPER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ADMIN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_LOG;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ROW_ERROR;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LEVEL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LOGGED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.MESSAGE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PHASE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROW_NUMBER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SEQ;
+
+
+/*******************************************************************************
+ ** Jobs app entry point. Core must define its durable jobs tables first.
+ *******************************************************************************/
+public class JobsQBitProducer implements QBitMetaDataProducer<JobsQBitConfig>
+{
+   public static final String QBIT_NAME = "com.kingsrook.qbits:qbit-jobs:jobs";
+   public static final String PURGE_PROCESS = "jobsPurgeFinished";
+   public static final List<String> HOME_WIDGETS = List.of("jobsHomeSummary", "jobsHomeActive", "jobsHomeThroughput", "jobsHomeAttention", "jobsHomeWorkers", "jobsHomeHistory");
+   private JobsQBitConfig qBitConfig = new JobsQBitConfig();
+
+   /*******************************************************************************
+    ** Configure this instance of the app.
+    *******************************************************************************/
+   public JobsQBitProducer withQBitConfig(JobsQBitConfig config)
+   {
+      qBitConfig = config;
+      return (this);
+   }
+
+   /*******************************************************************************
+    ** Host settings.
+    *******************************************************************************/
+   @Override
+   public JobsQBitConfig getQBitConfig()
+   {
+      return (qBitConfig);
+   }
+
+   /*******************************************************************************
+    ** Stable namespace, independent of the host's navigation label.
+    *******************************************************************************/
+   @Override
+   public String getNamespace()
+   {
+      return ("jobs");
+   }
+
+   /*******************************************************************************
+    ** Artifact identifier with the Maven-filtered version.
+    *******************************************************************************/
+   @Override
+   public QBitMetaData getQBitMetaData()
+   {
+      return (new QBitMetaData().withGroupId("com.kingsrook.qbits").withArtifactId("qbit-jobs")
+         .withVersion(version()).withNamespace(getNamespace()).withConfig(qBitConfig));
+   }
+
+   /*******************************************************************************
+    ** Read a packaged resource, never a filesystem file.
+    *******************************************************************************/
+   private static String version()
+   {
+      try(InputStream stream = JobsQBitProducer.class.getResourceAsStream("/jobs.properties"))
+      {
+         Properties properties = new Properties();
+         if(stream != null)
+         {
+            properties.load(stream);
+         }
+         return (properties.getProperty("version", "unknown"));
+      }
+      catch(IOException e)
+      {
+         throw (new IllegalStateException("Cannot read qbit-jobs version resource", e));
+      }
+   }
+
+   /*******************************************************************************
+    ** Validate first, then register presentation, alerts and retention. No tables
+    ** are added. Return empty output because registration has already occurred.
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance instance) throws QException
+   {
+      if(ALL_TABLES.stream().anyMatch(table -> instance.getTable(table) == null))
+      {
+         throw (new QException("Durable jobs are not enabled: call DurableJobsMetaDataProvider.defineAll(...) before producing qbit-jobs"));
+      }
+      if(qBitConfig == null)
+      {
+         throw (new QException("JobsQBitConfig is required"));
+      }
+      qBitConfig.validate(instance);
+      instance.addQBit(getQBitMetaData());
+      registerWidget(instance, "jobsHomeSummary", "Overview", WidgetType.MULTI_STATISTICS, qBitConfig.getHomeRefreshSeconds());
+      registerWidget(instance, "jobsHomeActive", "Running and queued", WidgetType.TABLE, qBitConfig.getHomeRefreshSeconds());
+      registerWidget(instance, "jobsHomeThroughput", "Rows per hour", WidgetType.STACKED_BAR_CHART, qBitConfig.getHomeRefreshSeconds());
+      registerWidget(instance, "jobsHomeAttention", "Needs attention", WidgetType.TABLE, qBitConfig.getHomeRefreshSeconds());
+      QWidgetMetaData workers = registerWidget(instance, "jobsHomeWorkers", "Workers", WidgetType.TABLE, qBitConfig.getHomeRefreshSeconds());
+      workers.withPermissionRules(adminRules());
+      registerWidget(instance, "jobsHomeHistory", "Job history", WidgetType.STACKED_BAR_CHART, qBitConfig.getHomeRefreshSeconds());
+      for(String name : HOME_WIDGETS)
+      {
+         if(!"jobsHomeWorkers".equals(name))
+         {
+            instance.getWidget(name).setPermissionRules(appAccessRules());
+         }
+      }
+      QAppMetaData app = new QAppMetaData().withName(qBitConfig.getAppName()).withLabel("Jobs").withIcon(new QIcon("pending_actions"))
+         .withChild(instance.getTable(JOBS_JOB)).withWidgets(new ArrayList<>(HOME_WIDGETS))
+         .withPermissionRules(appAccessRules());
+      app.setParentAppName(qBitConfig.getParentAppName());
+      instance.addApp(app);
+      if(qBitConfig.getParentAppName() != null)
+      {
+         instance.getApp(qBitConfig.getParentAppName()).withChild(app);
+      }
+      registerWidget(instance, "jobsJobProgress", "Live Progress", WidgetType.MULTI_STATISTICS, qBitConfig.getRecordRefreshSeconds());
+      addRecordSection(instance, "jobsJobProgress", "Live Progress");
+      if(instance.getTable(JOBS_LOG) != null && instance.getJoin(JobsCoreContract.childJoin(JOBS_LOG)) != null)
+      {
+         child(instance, JOBS_LOG, "jobsJobLog", "Log", ID, false);
+      }
+      child(instance, JOBS_CHUNK, "jobsJobChunks", "Chunks", SEQ, true);
+      child(instance, JOBS_ROW_ERROR, "jobsJobRowErrors", "Row Errors", ROW_NUMBER, true);
+      if(instance.getTable(JOBS_EFFECT) != null && instance.getJoin(JobsCoreContract.childJoin(JOBS_EFFECT)) != null)
+      {
+         child(instance, JOBS_EFFECT, "jobsJobEffects", "Effects", ID, true);
+      }
+      child(instance, JOBS_EVENT, "jobsJobEvents", "Events", ID, false);
+      if(instance.getTable(AUDIT) != null)
+      {
+         registerWidget(instance, "jobsJobAudit", "Changes made by this job", WidgetType.STATISTICS, qBitConfig.getRecordRefreshSeconds());
+         addRecordSection(instance, "jobsJobAudit", "Changes made by this job");
+      }
+      registerRetention(instance);
+      registerAlerts(instance);
+      return (new MetaDataProducerMultiOutput());
+   }
+
+   /*******************************************************************************
+    ** Every panel uses flexible grid sizing and typed data.
+    *******************************************************************************/
+   private QWidgetMetaData registerWidget(QInstance instance, String name, String label, WidgetType type, int refresh) throws QException
+   {
+      QWidgetMetaData widget = new QWidgetMetaData().withName(name).withLabel(label).withType(type.getType()).withGridColumns(12)
+         .withCodeReference(new QCodeReference(JobsWidgetRenderer.class)).withDefaultValue("refreshSeconds", refresh);
+      instance.addWidget(widget);
+      return (widget);
+   }
+
+   /*******************************************************************************
+    ** Use the core builder, retaining parent/child security and ordering.
+    *******************************************************************************/
+   private void child(QInstance instance, String table, String name, String label, String order, boolean ascending) throws QException
+   {
+      QWidgetMetaData widget = ChildRecordListRenderer.widgetMetaDataBuilder(instance.getJoin(JobsCoreContract.childJoin(table)))
+         .withOrderBys(List.of(new QFilterOrderBy(order, ascending))).withMaxRows(25).getWidgetMetaData()
+         .withName(name).withLabel(label).withGridColumns(12).withCodeReference(new QCodeReference(JobsChildRecordListRenderer.class))
+         .withDefaultValue("refreshSeconds", qBitConfig.getRecordRefreshSeconds());
+      if(JOBS_LOG.equals(table))
+      {
+         widget.withDefaultValue(ChildRecordListRenderer.KEY_ONLY_INCLUDE_FIELD_NAMES, new ArrayList<>(List.of(LEVEL, LOGGED_AT, PHASE, MESSAGE)));
+      }
+      instance.addWidget(widget);
+      addRecordSection(instance, name, label);
+   }
+
+   /*******************************************************************************
+    ** Append record panels without replacing core's fields or action sections.
+    *******************************************************************************/
+   private void addRecordSection(QInstance instance, String widgetName, String label)
+   {
+      instance.getTable(JOBS_JOB).withSection(new QFieldSection().withName(widgetName).withLabel(label)
+         .withTier(Tier.T2).withWidgetName(widgetName).withGridColumns(12));
+   }
+
+   /*******************************************************************************
+    ** The Jobs app and its home widgets require "<appName>.hasAccess"; users
+    ** without it never see the app or its widgets (hidden, not locked).
+    *******************************************************************************/
+   private QPermissionRules appAccessRules()
+   {
+      return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
+         .withPermissionBaseName(qBitConfig.getAppName()).withDenyBehavior(DenyBehavior.HIDDEN));
+   }
+
+   /*******************************************************************************
+    ** Match the exact core permission, without a .hasAccess suffix.
+    *******************************************************************************/
+   private static QPermissionRules adminRules()
+   {
+      return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
+         .withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(JOBS_ADMIN)));
+   }
+
+   /*******************************************************************************
+    ** Hidden system maintenance; a scheduler is explicitly selected by the host.
+    *******************************************************************************/
+   private void registerRetention(QInstance instance) throws QException
+   {
+      QProcessMetaData process = new QProcessMetaData().withName(PURGE_PROCESS).withLabel("Purge finished jobs")
+         .withIsHidden(true).withPermissionRules(adminRules()).withSourceQBitName(QBIT_NAME)
+         .withStep(new QBackendStepMetaData().withName("purge").withCode(new QCodeReference(JobsPurgeFinishedStep.class)));
+      if(qBitConfig.getSchedulerName() != null)
+      {
+         process.withSchedule(new QScheduleMetaData().withSchedulerName(qBitConfig.getSchedulerName())
+            .withCronExpression(qBitConfig.getPurgeCronExpression()).withCronTimeZoneId("UTC"));
+      }
+      instance.addProcess(process);
+   }
+
+   /*******************************************************************************
+    ** Type declarations consumed by qbit-alerts; core publishes the signals.
+    *******************************************************************************/
+   private void registerAlerts(QInstance instance)
+   {
+      AlertsInstanceMetaData.ofOrWithNew(instance)
+         .withAlertType(new QAlertTypeMetaData().withName(ALERT_JOB_FAILED).withLabel("Job failed").withDefaultSeverity(AlertSeverity.ERROR)
+            .withSubjectTableName(JOBS_JOB).withSourceQBitName(QBIT_NAME).withHowToFix("Inspect the job error and row errors, correct the cause, then retry failed work."))
+         .withAlertType(new QAlertTypeMetaData().withName(ALERT_JOB_STALLED).withLabel("Job stalled").withDefaultSeverity(AlertSeverity.WARNING)
+            .withSubjectTableName(JOBS_JOB).withSourceQBitName(QBIT_NAME).withHowToFix("Check worker heartbeats, storage access and chunk leases; restore workers before retrying."))
+         .withAlertType(new QAlertTypeMetaData().withName(ALERT_EFFECT_UNRESOLVED).withLabel("Job effect needs review").withDefaultSeverity(AlertSeverity.WARNING)
+            .withSubjectTableName(JOBS_JOB).withSourceQBitName(QBIT_NAME).withHowToFix("Review the effect and confirm its outcome with the provider before retrying."))
+         .withAlertType(new QAlertTypeMetaData().withName(HEARTBEAT_REAPER).withLabel("Job reaper stopped").withDefaultSeverity(AlertSeverity.ERROR)
+            .withSourceQBitName(QBIT_NAME).withVisibilityPermission(JOBS_ADMIN).withDescription("No pod is reaping leases.")
+            .withHowToFix("Restore a durable jobs runtime so expired chunk leases are reclaimed.")
+            .withDefaultMonitor(AlertDefaultMonitor.heartbeat(HEARTBEAT_REAPER, HEARTBEAT_REAPER, 60, 120).withScheduleSeconds(60)));
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/processes/JobsPurgeFinishedStep.java

@@ -0,0 +1,301 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs.processes;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.jobs.JobsQBitConfig;
+import com.kingsrook.qbits.jobs.JobsQBitProducer;
+import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FINISHED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.INPUT_STORAGE_REF;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ADMIN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_LOG;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ROW_ERROR;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_UPLOAD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW_FIELD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STORAGE_REFERENCE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STORAGE_TABLE_NAME;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.TERMINAL_STATUSES;
+
+
+/*******************************************************************************
+ ** Retention for core's standard tables. Batches use increasing primary keys so
+ ** updates and deletes cannot cause skipped rows. Only this step uses system access.
+ *******************************************************************************/
+public class JobsPurgeFinishedStep implements BackendStep
+{
+   private static final int BATCH_SIZE = 200;
+   private final JobsStorage storage;
+
+   /*******************************************************************************
+    ** Framework constructor.
+    *******************************************************************************/
+   public JobsPurgeFinishedStep()
+   {
+      this(new JobsStorage());
+   }
+
+   /*******************************************************************************
+    ** Storage seam for testing deletion failures and retries.
+    *******************************************************************************/
+   public JobsPurgeFinishedStep(JobsStorage storage)
+   {
+      this.storage = storage;
+   }
+
+   /*******************************************************************************
+    ** Restore the caller even when storage or a table action fails.
+    *******************************************************************************/
+   @Override
+   public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+   {
+      if(QContext.getQSession() == null || !QContext.getQSession().hasPermission(JOBS_ADMIN))
+      {
+         throw (new QException("jobsPurgeFinished requires jobs_admin or a system session"));
+      }
+      JobsQBitConfig config = (JobsQBitConfig) QContext.getQInstance().getQBits().get(JobsQBitProducer.QBIT_NAME).getConfig();
+      QSession caller = QContext.getQSession();
+      int[] counts = new int[4];
+      try
+      {
+         QContext.setQSession(new QSystemUserSession());
+         Instant now = Instant.now();
+         purge(config, now, counts);
+         cleanInputs(config, now, counts);
+      }
+      finally
+      {
+         QContext.setQSession(caller);
+      }
+      ArrayList<ProcessSummaryLineInterface> summary = new ArrayList<>();
+      summary.add(new ProcessSummaryLine(Status.INFO, counts[0], "jobs purged"));
+      summary.add(new ProcessSummaryLine(Status.INFO, counts[1], "chunk objects deleted"));
+      summary.add(new ProcessSummaryLine(Status.INFO, counts[2], "input objects deleted"));
+      if(counts[3] > 0)
+      {
+         summary.add(new ProcessSummaryLine(Status.INFO, counts[3], "jobs kept: effects still unsettled"));
+      }
+      output.addValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY, summary);
+   }
+
+   /*******************************************************************************
+    ** Remove old final jobs, deleting children before their parent.
+    *******************************************************************************/
+   private void purge(JobsQBitConfig config, Instant now, int[] counts) throws QException
+   {
+      int lastId = 0;
+      while(true)
+      {
+         QQueryFilter filter = new QQueryFilter().withCriteria(STATUS, QCriteriaOperator.IN, TERMINAL_STATUSES)
+            .withCriteria(FINISHED_AT, QCriteriaOperator.LESS_THAN, now.minus(config.getRetentionDays(), ChronoUnit.DAYS));
+         List<QRecord> jobs = batch(JOBS_JOB, filter, lastId);
+         if(jobs.isEmpty())
+         {
+            return;
+         }
+         for(QRecord job : jobs)
+         {
+            lastId = job.getValueInteger(ID);
+            if(hasUnsettledEffects(lastId))
+            {
+               counts[3]++;
+               continue;
+            }
+            if(QContext.getQInstance().getTable(JOBS_EFFECT) != null)
+            {
+               removeChildren(JOBS_EFFECT, lastId, counts);
+            }
+            if(QContext.getQInstance().getTable(JOBS_LOG) != null)
+            {
+               removeChildren(JOBS_LOG, lastId, counts);
+            }
+            removeChildren(JOBS_ROW_ERROR, lastId, counts);
+            removeChildren(JOBS_EVENT, lastId, counts);
+            removeChildren(JOBS_CHUNK, lastId, counts);
+            removeChildren(JOBS_UPLOAD, lastId, counts);
+            deleteRecord(JOBS_JOB, job);
+            counts[0]++;
+         }
+      }
+   }
+
+   /*******************************************************************************
+    ** Preserve unresolved outbox work, including unknown or missing statuses.
+    *******************************************************************************/
+   private boolean hasUnsettledEffects(int jobId) throws QException
+   {
+      if(QContext.getQInstance().getTable(JOBS_EFFECT) == null)
+      {
+         return (false);
+      }
+      QQueryFilter unsettled = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+         .withCriteria(STATUS, QCriteriaOperator.NOT_IN, List.of(SENT, FAILED))
+         .withCriteria(STATUS, QCriteriaOperator.IS_BLANK)
+         .withCriteria(NEEDS_REVIEW_FIELD, QCriteriaOperator.EQUALS, true);
+      QQueryFilter filter = new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.EQUALS, jobId).withSubFilter(unsettled).withLimit(1);
+      return (!new QueryAction().execute(new QueryInput().withTableName(JOBS_EFFECT).withFilter(filter)).getRecords().isEmpty());
+   }
+
+   /*******************************************************************************
+    ** Page all children, including uploads, before removing a job.
+    *******************************************************************************/
+   private void removeChildren(String table, int jobId, int[] counts) throws QException
+   {
+      long lastId = 0;
+      while(true)
+      {
+         List<QRecord> records = batch(table, new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.EQUALS, jobId), lastId);
+         if(records.isEmpty())
+         {
+            return;
+         }
+         for(QRecord record : records)
+         {
+            lastId = record.getValueLong(ID);
+            if(JOBS_CHUNK.equals(table) && storage.delete(storage.chunkStorageTable(), record.getValueString(INPUT_STORAGE_REF)))
+            {
+               counts[1]++;
+            }
+            if(JOBS_UPLOAD.equals(table) && storage.delete(record.getValueString(STORAGE_TABLE_NAME), record.getValueString(STORAGE_REFERENCE)))
+            {
+               counts[2]++;
+            }
+            deleteRecord(table, record);
+         }
+      }
+   }
+
+   /*******************************************************************************
+    ** Keep successful job history while releasing its old input objects.
+    *******************************************************************************/
+   private void cleanInputs(JobsQBitConfig config, Instant now, int[] counts) throws QException
+   {
+      int lastId = 0;
+      while(true)
+      {
+         QQueryFilter filter = new QQueryFilter().withCriteria(STATUS, QCriteriaOperator.EQUALS, COMPLETED)
+            .withCriteria(ROWS_FAILED, QCriteriaOperator.EQUALS, 0)
+            .withCriteria(FINISHED_AT, QCriteriaOperator.LESS_THAN, now.minus(config.getInputRetentionDays(), ChronoUnit.DAYS));
+         List<QRecord> jobs = batch(JOBS_JOB, filter, lastId);
+         if(jobs.isEmpty())
+         {
+            return;
+         }
+         for(QRecord job : jobs)
+         {
+            lastId = job.getValueInteger(ID);
+            if(!hasUnsettledEffects(lastId))
+            {
+               cleanJobInputs(lastId, counts);
+            }
+         }
+      }
+   }
+
+   /*******************************************************************************
+    ** Null a reference only after storage reports successful deletion.
+    *******************************************************************************/
+   private void cleanJobInputs(int jobId, int[] counts) throws QException
+   {
+      int lastId = 0;
+      while(true)
+      {
+         List<QRecord> chunks = batch(JOBS_CHUNK, new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.EQUALS, jobId)
+            .withCriteria(INPUT_STORAGE_REF, QCriteriaOperator.IS_NOT_BLANK), lastId);
+         if(chunks.isEmpty())
+         {
+            return;
+         }
+         for(QRecord chunk : chunks)
+         {
+            lastId = chunk.getValueInteger(ID);
+            if(storage.delete(storage.chunkStorageTable(), chunk.getValueString(INPUT_STORAGE_REF)))
+            {
+               QRecord update = new QRecord().withValue(ID, lastId).withValue(INPUT_STORAGE_REF, null);
+               List<QRecord> results = new UpdateAction().execute(new UpdateInput().withTableName(JOBS_CHUNK).withRecord(update)).getRecords();
+               if(results.isEmpty() || results.stream().anyMatch(record -> record.getErrors() != null && !record.getErrors().isEmpty()))
+               {
+                  throw (new QException("Could not clear deleted chunk input reference"));
+               }
+               counts[2]++;
+            }
+         }
+      }
+   }
+
+   /*******************************************************************************
+    ** Keyset pagination remains correct when rows are deleted or updated.
+    *******************************************************************************/
+   private List<QRecord> batch(String table, QQueryFilter filter, long lastId) throws QException
+   {
+      return (new QueryAction().execute(new QueryInput().withTableName(table).withFilter(filter.withCriteria(ID, QCriteriaOperator.GREATER_THAN, lastId)
+         .withOrderBy(new QFilterOrderBy(ID)).withLimit(BATCH_SIZE))).getRecords());
+   }
+
+   /*******************************************************************************
+    ** Treat action-level record errors as failures, never as a successful purge.
+    *******************************************************************************/
+   private void deleteRecord(String table, QRecord record) throws QException
+   {
+      DeleteOutput result = new DeleteAction().execute(new DeleteInput().withTableName(table).withPrimaryKey(record.getValue(ID)));
+      if(result.getRecordsWithErrors() != null && !result.getRecordsWithErrors().isEmpty())
+      {
+         throw (new QException("Could not purge record from " + table));
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/processes/JobsStorage.java

@@ -0,0 +1,58 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs.processes;
+
+
+import com.kingsrook.qqq.backend.core.actions.tables.StorageAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
+import com.kingsrook.qqq.backend.core.model.actions.tables.storage.StorageInput;
+
+
+/*******************************************************************************
+ ** Object cleanup through core storage actions and durable jobs configuration.
+ *******************************************************************************/
+public class JobsStorage
+{
+   /*******************************************************************************
+    ** Look up the host's durable jobs storage table.
+    *******************************************************************************/
+   public String chunkStorageTable() throws QException
+   {
+      DurableJobsMetaData metadata = DurableJobsMetaData.of(QContext.getQInstance());
+      return (metadata == null ? null : metadata.getStorageTableName());
+   }
+
+   /*******************************************************************************
+    ** Return true only after successful deletion; failures abort retention for retry.
+    *******************************************************************************/
+   public boolean delete(String table, String reference) throws QException
+   {
+      if(reference == null || reference.isBlank() || table == null || table.isBlank())
+      {
+         return (false);
+      }
+      new StorageAction().deleteObject(new StorageInput(table).withReference(reference));
+      return (true);
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/widgets/JobsChartData.java

@@ -0,0 +1,42 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs.widgets;
+
+
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+
+
+/*******************************************************************************
+ ** ChartData payload with the stacked-bar type used by this app's metadata.
+ *******************************************************************************/
+public class JobsChartData extends ChartData
+{
+   /*******************************************************************************
+    ** Match the stacked-bar widget metadata.
+    *******************************************************************************/
+   @Override
+   public String getType()
+   {
+      return (WidgetType.STACKED_BAR_CHART.getType());
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/widgets/JobsChildRecordListRenderer.java

@@ -0,0 +1,47 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs.widgets;
+
+
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+
+
+/*******************************************************************************
+ ** Core child lists with the Jobs session keys ensured before reading.
+ *******************************************************************************/
+public class JobsChildRecordListRenderer extends ChildRecordListRenderer
+{
+   /*******************************************************************************
+    ** Keep the core child-list behavior and viewer's session.
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      DurableJobsSecurity.ensureSessionKeys(QContext.getQSession());
+      return (super.render(input));
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/widgets/JobsFormat.java

@@ -0,0 +1,157 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs.widgets;
+
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+
+
+/*******************************************************************************
+ ** Human-readable values for the Jobs widgets: counts with separators, relative
+ ** times within a day ("4m ago", "in 3m") and absolute UTC times beyond it
+ ** ("Oct 14, 08:46 UTC"). Never raw ISO timestamps.
+ *******************************************************************************/
+public final class JobsFormat
+{
+   /** The single-character ellipsis appended to clipped text. */
+   public static final String ELLIPSIS = String.valueOf((char) 0x2026);
+
+   private static final DateTimeFormatter ABSOLUTE = DateTimeFormatter.ofPattern("MMM d, HH:mm 'UTC'", Locale.US).withZone(ZoneOffset.UTC);
+
+
+
+   /*******************************************************************************
+    ** Static helpers only.
+    *******************************************************************************/
+   private JobsFormat()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Binary byte units, with one decimal place above bytes.
+    *******************************************************************************/
+   public static String bytes(long value)
+   {
+      double amount = Math.max(0, value);
+      String[] units = {"B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"};
+      int unit = 0;
+      while(amount >= 1024 && unit < units.length - 1)
+      {
+         amount /= 1024;
+         unit++;
+      }
+      return (unit == 0 ? count((long) amount) + " B" : String.format(Locale.US, "%.1f %s", amount, units[unit]));
+   }
+
+
+
+   /*******************************************************************************
+    ** A count with thousands separators.
+    *******************************************************************************/
+   public static String count(long value)
+   {
+      return (String.format(Locale.US, "%,d", value));
+   }
+
+
+
+   /*******************************************************************************
+    ** Relative within 24 hours of now, absolute UTC otherwise; "Unknown" for null.
+    *******************************************************************************/
+   public static String time(Instant value, Instant now)
+   {
+      if(value == null)
+      {
+         return ("Unknown");
+      }
+      Duration between = Duration.between(value, now);
+      if(between.abs().compareTo(Duration.ofHours(24)) >= 0)
+      {
+         return (ABSOLUTE.format(value));
+      }
+      return (relative(value, now));
+   }
+
+
+
+   /*******************************************************************************
+    ** Always relative ("12s ago", "3d ago", "in 8m"); "Unknown" for null.
+    *******************************************************************************/
+   public static String relative(Instant value, Instant now)
+   {
+      if(value == null)
+      {
+         return ("Unknown");
+      }
+      long seconds = Duration.between(value, now).getSeconds();
+      if(seconds < 0)
+      {
+         return ("in " + span(-seconds));
+      }
+      return (seconds < 5 ? "just now" : span(seconds) + " ago");
+   }
+
+
+
+   /*******************************************************************************
+    ** A duration in its largest whole unit: 45s, 8m, 2h, 3d.
+    *******************************************************************************/
+   public static String span(long seconds)
+   {
+      long value = Math.max(0, seconds);
+      if(value < 60)
+      {
+         return (value + "s");
+      }
+      if(value < 3600)
+      {
+         return (value / 60 + "m");
+      }
+      if(value < 86_400)
+      {
+         return (value / 3600 + "h");
+      }
+      return (value / 86_400 + "d");
+   }
+
+
+
+   /*******************************************************************************
+    ** The first max characters of a single-line plain text, with an ellipsis
+    ** when clipped; empty for null.
+    *******************************************************************************/
+   public static String clip(String text, int max)
+   {
+      if(text == null)
+      {
+         return ("");
+      }
+      String line = text.replaceAll("\\s+", " ").trim();
+      return (line.length() <= max ? line : line.substring(0, max - 1) + ELLIPSIS);
+   }
+}

src/main/java/com/kingsrook/qbits/jobs/widgets/JobsWidgetRenderer.java

@@ -0,0 +1,918 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs.widgets;
+
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.BlockLink;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.progressbar.ProgressBarBlockData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.progressbar.ProgressBarValues;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBlockData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextStyles;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextValues;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ATTENTION_STATUSES;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.AUDIT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.BYTES_DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.BYTES_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED_WITH_ERRORS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CURRENT_PHASE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.DRAINING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ERROR_MESSAGE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ETA_SECONDS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FINISHED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.HEARTBEAT_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ADMIN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_WORKER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LABEL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LAST_PROGRESS_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LOG_DROPPED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.MESSAGE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW_FIELD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PENDING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PERCENT_DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.POD_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PRIORITY;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PROCESS_NAME;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PROGRESS_JSON;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PROGRESS_MESSAGE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.QUEUED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.QUEUED_STATUSES;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_IN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_OK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.RUNNING_STATUSES;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SENDING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STARTED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS_REASON;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.TERMINAL_STATUSES;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.THREADS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.UNKNOWN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.statusLabel;
+
+
+/*******************************************************************************
+ ** Typed dashboard and record panels, always queried in the viewer's context.
+ ** Multi-line table cells are composite widget blocks (text with a record link,
+ ** progress bars); each row also carries its plain values under readable keys.
+ ** No HTML anywhere. The widget label is the card title, so the data itself
+ ** carries no title.
+ *******************************************************************************/
+public class JobsWidgetRenderer extends AbstractWidgetRenderer
+{
+   public static final String JOB_CELL      = "jobCell";
+   public static final String PROGRESS_CELL = "progressCell";
+   public static final String STATUS_CELL   = "statusCell";
+   public static final String POD_CELL      = "podCell";
+   public static final String CAPACITY_CELL = "capacityCell";
+
+   private static final String SMALL = "13";
+
+
+
+   /*******************************************************************************
+    ** Dispatch the named panel without changing the current session.
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      DurableJobsSecurity.ensureSessionKeys(QContext.getQSession());
+      Instant now = Instant.now();
+      QWidgetData data = switch(input.getWidgetMetaData().getName())
+      {
+         case "jobsHomeSummary" -> summary(now);
+         case "jobsHomeActive" -> active(now);
+         case "jobsHomeThroughput" -> throughput(now);
+         case "jobsHomeAttention" -> attention(now);
+         case "jobsHomeWorkers" -> workers(now);
+         case "jobsHomeHistory" -> history(now);
+         case "jobsJobProgress" -> progress(record(input), now);
+         case "jobsJobAudit" -> audit(record(input));
+         default -> throw new QException("Unknown Jobs widget: " + input.getWidgetMetaData().getName());
+      };
+      return (new RenderWidgetOutput(data));
+   }
+
+
+
+   /*******************************************************************************
+    ** Read through QueryAction so core's record locks apply, including child joins.
+    *******************************************************************************/
+   public static List<QRecord> query(String table, QQueryFilter filter) throws QException
+   {
+      DurableJobsSecurity.ensureSessionKeys(QContext.getQSession());
+      return (new QueryAction().execute(new QueryInput().withTableName(table).withFilter(filter)
+         .withShouldFetchHeavyFields(true)).getRecords());
+   }
+
+
+
+   /*******************************************************************************
+    ** The label of a process, falling back to its name when it is not defined here.
+    *******************************************************************************/
+   public static String processLabel(String processName)
+   {
+      if(processName == null)
+      {
+         return ("Unknown process");
+      }
+      QProcessMetaData process = QContext.getQInstance().getProcess(processName);
+      return (process == null || process.getLabel() == null ? processName : process.getLabel());
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve the same id parameter used by core's ChildRecordListRenderer.
+    ** Verify ownership before querying audit records, which may have broader locks.
+    *******************************************************************************/
+   private QRecord record(RenderWidgetInput input) throws QException
+   {
+      String primaryKey = QContext.getQInstance().getTable(JOBS_JOB).getPrimaryKeyField();
+      String id         = input.getQueryParams().getOrDefault(ID, input.getQueryParams().get(primaryKey));
+      if(id == null || id.isBlank())
+      {
+         return (null);
+      }
+      List<QRecord> records = query(JOBS_JOB, new QQueryFilter().withCriteria(primaryKey, QCriteriaOperator.EQUALS, id).withLimit(1));
+      return (records.isEmpty() ? null : records.getFirst());
+   }
+
+
+
+   /*******************************************************************************
+    ** Restrict to the named statuses.
+    *******************************************************************************/
+   private static QQueryFilter statuses(List<String> values)
+   {
+      return (new QQueryFilter().withCriteria(STATUS, QCriteriaOperator.IN, values));
+   }
+
+
+
+   /*******************************************************************************
+    ** Recent finish filter shared by attention, history and summary.
+    *******************************************************************************/
+   private static QQueryFilter finishedSince(Instant since)
+   {
+      return (new QQueryFilter().withCriteria(FINISHED_AT, QCriteriaOperator.GREATER_THAN_OR_EQUALS, since));
+   }
+
+
+
+   /*******************************************************************************
+    ** Two groups: what is happening now (each linking to the matching jobs), and
+    ** the last 24 hours. Every statistic shows its label once and a whole number.
+    *******************************************************************************/
+   private MultiStatisticsData summary(Instant now) throws QException
+   {
+      Instant weekAgo = now.minus(7, ChronoUnit.DAYS);
+      QQueryFilter filter = new QQueryFilter().withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+         .withSubFilter(statuses(RUNNING_STATUSES)).withSubFilter(statuses(QUEUED_STATUSES))
+         .withSubFilter(finishedSince(weekAgo))
+         .withSubFilter(new QQueryFilter().withCriteria(LAST_PROGRESS_AT, QCriteriaOperator.GREATER_THAN_OR_EQUALS, now.minus(1, ChronoUnit.HOURS)));
+      long running   = 0;
+      long queued    = 0;
+      long attention = 0;
+      long completed = 0;
+      long rows      = 0;
+      long hourRows  = 0;
+      for(QRecord job : query(JOBS_JOB, filter))
+      {
+         String  status = job.getValueString(STATUS);
+         boolean live   = RUNNING_STATUSES.contains(status);
+         boolean day    = since(job, FINISHED_AT, now.minus(1, ChronoUnit.DAYS));
+         running += live ? 1 : 0;
+         queued += QUEUED_STATUSES.contains(status) ? 1 : 0;
+         attention += ATTENTION_STATUSES.contains(status) && since(job, FINISHED_AT, weekAgo) ? 1 : 0;
+         completed += (COMPLETED.equals(status) || COMPLETED_WITH_ERRORS.equals(status)) && day ? 1 : 0;
+         rows += live || day ? number(job, ROWS_DONE) : 0;
+         hourRows += since(job, LAST_PROGRESS_AT, now.minus(1, ChronoUnit.HOURS)) ? number(job, ROWS_DONE) : 0;
+      }
+
+      MultiStatisticsData.StatisticsGroupData nowGroup = new MultiStatisticsData.StatisticsGroupData().withHeader("Now").withStatisticList(List.of(
+         statistic("Running", running, jobsUrl(statuses(RUNNING_STATUSES))),
+         statistic("Queued", queued, jobsUrl(statuses(QUEUED_STATUSES))),
+         statistic("Needs attention", attention, jobsUrl(statuses(ATTENTION_STATUSES).withSubFilter(finishedSince(weekAgo))))));
+      MultiStatisticsData.StatisticsGroupData dayGroup = new MultiStatisticsData.StatisticsGroupData().withHeader("Last 24 hours").withStatisticList(List.of(
+         statistic("Completed", completed, null),
+         statistic("Rows processed", rows, null),
+         statistic("Rows/min (last hour)", Math.round(hourRows / 60.0), null)));
+      return (new MultiStatisticsData(null, List.of(nowGroup, dayGroup)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Active jobs, ordered by priority then queued time and capped at 25.
+    ** Three columns, so it reads at phone width: job, progress, status.
+    *******************************************************************************/
+   private TableData active(Instant now) throws QException
+   {
+      List<String> activeStatuses = new ArrayList<>(RUNNING_STATUSES);
+      activeStatuses.addAll(QUEUED_STATUSES);
+      List<Map<String, Object>> rows = new ArrayList<>();
+      for(QRecord job : query(JOBS_JOB, statuses(activeStatuses).withOrderBy(new QFilterOrderBy(PRIORITY, false))
+         .withOrderBy(new QFilterOrderBy(QUEUED_AT)).withLimit(25)))
+      {
+         String  status  = job.getValueString(STATUS);
+         String  reason  = job.getValueString(STATUS_REASON);
+         boolean waiting = QUEUED_STATUSES.contains(status) && reason != null && !reason.isBlank();
+         String  state   = waiting ? reason : statusLabel(status);
+         String  percent = percent(job);
+         String  rowText = count(job, ROWS_DONE) + " / " + total(job, ROWS_TOTAL);
+         String  rate    = job.getValue(STARTED_AT) == null ? "Not started" : JobsFormat.count(Math.round(rate(job, now))) + " rows/min";
+         String  eta     = eta(job, now);
+         String  started = job.getValue(STARTED_AT) == null ? "Not started" : "Started " + JobsFormat.time(job.getValueInstant(STARTED_AT), now);
+
+         Map<String, Object> row = row("Job", job.getValueString(LABEL), "Process", processLabel(job.getValueString(PROCESS_NAME)),
+            "Status", state, "Progress", percent, "Rows", rowText, "Rate", rate, "ETA", eta, "Started", started, "Path", jobPath(job));
+
+         CompositeWidgetData progress = column();
+         if(percentValue(job) != null)
+         {
+            progress.withBlock(new ProgressBarBlockData().withValues(new ProgressBarValues().withPercent(percentValue(job)).withValue(percent)));
+         }
+         else
+         {
+            progress.withBlock(text("Unknown total", null, null));
+         }
+         progress.withBlock(small(rowText + " rows"));
+         for(String detail : progressDetails(job))
+         {
+            progress.withBlock(small(detail));
+         }
+         row.put("Phase", phaseLabel(job));
+         row.put("Message", job.getValueString(PROGRESS_MESSAGE));
+         row.put("Bytes", bytes(job));
+
+         row.put(JOB_CELL, jobCell(job, null));
+         row.put(PROGRESS_CELL, progress);
+         row.put(STATUS_CELL, column().withBlock(text(state, "semibold", null))
+            .withBlock(small(RUNNING_STATUSES.contains(status) ? rate + ", ETA " + eta : started)));
+         rows.add(row);
+      }
+      return (table("No jobs are running", rows, col("Job", JOB_CELL), col("Progress", PROGRESS_CELL), col("Status", STATUS_CELL)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Failures and partial successes in the last seven days: failed and
+    ** needs-review first, then completed with errors, newest first within each.
+    *******************************************************************************/
+   private TableData attention(Instant now) throws QException
+   {
+      List<QRecord> jobs = new ArrayList<>(query(JOBS_JOB, statuses(ATTENTION_STATUSES).withSubFilter(finishedSince(now.minus(7, ChronoUnit.DAYS)))
+         .withOrderBy(new QFilterOrderBy(FINISHED_AT, false)).withLimit(200)));
+      jobs.sort(Comparator.comparingInt((QRecord job) -> COMPLETED_WITH_ERRORS.equals(job.getValueString(STATUS)) ? 1 : 0)
+         .thenComparing((QRecord job) -> job.getValueInstant(FINISHED_AT), Comparator.nullsLast(Comparator.reverseOrder())));
+
+      List<QRecord> listedJobs = jobs.subList(0, Math.min(25, jobs.size()));
+      Map<Integer, Integer> reviewCounts = new HashMap<>();
+      if(!listedJobs.isEmpty() && QContext.getQInstance().getTable(JOBS_EFFECT) != null)
+      {
+         QQueryFilter filter = new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.IN, listedJobs.stream().map(job -> job.getValueInteger(ID)).toList())
+            .withCriteria(NEEDS_REVIEW_FIELD, QCriteriaOperator.EQUALS, true);
+         for(QRecord effect : query(JOBS_EFFECT, filter))
+         {
+            reviewCounts.merge(effect.getValueInteger(JOB_ID), 1, Integer::sum);
+         }
+      }
+      List<Map<String, Object>> rows = new ArrayList<>();
+      for(QRecord job : listedJobs)
+      {
+         String error    = JobsFormat.clip(job.getValueString(ERROR_MESSAGE), 120);
+         String failed   = count(job, ROWS_FAILED) + (number(job, ROWS_FAILED) == 1 ? " row failed" : " rows failed");
+         String finished = "Finished " + JobsFormat.time(job.getValueInstant(FINISHED_AT), now);
+         String status   = statusLabel(job.getValueString(STATUS));
+         Map<String, Object> row = row("Job", job.getValueString(LABEL), "Process", processLabel(job.getValueString(PROCESS_NAME)),
+            "Status", status, "Rows failed", number(job, ROWS_FAILED), "Error", error, "Finished", finished, "Path", jobPath(job));
+         row.put(JOB_CELL, jobCell(job, error.isEmpty() ? null : error));
+         CompositeWidgetData statusCell = column().withBlock(text(status, "semibold", COMPLETED_WITH_ERRORS.equals(job.getValueString(STATUS)) ? "WARNING" : "ERROR"));
+         if(number(job, ROWS_FAILED) > 0)
+         {
+            statusCell.withBlock(small(failed));
+         }
+         int effectsNeedingReview = reviewCounts.getOrDefault(job.getValueInteger(ID), 0);
+         if(effectsNeedingReview > 0)
+         {
+            statusCell.withBlock(small(effectsNeedingReview + " effects need review"));
+         }
+         row.put(STATUS_CELL, statusCell.withBlock(small(finished)));
+         rows.add(row);
+      }
+      return (table("No jobs need attention", rows, col("Job", JOB_CELL), col("Status", STATUS_CELL)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Defense in depth: direct renderer calls also require jobs_admin.
+    *******************************************************************************/
+   private TableData workers(Instant now) throws QException
+   {
+      List<Map<String, Object>> rows  = new ArrayList<>();
+      boolean                   admin = QContext.getQSession() != null && QContext.getQSession().hasPermission(JOBS_ADMIN);
+      if(admin)
+      {
+         for(QRecord worker : query(JOBS_WORKER, new QQueryFilter().withOrderBy(new QFilterOrderBy(POD_ID))))
+         {
+            long   threads   = number(worker, THREADS);
+            String capacity  = threads + (threads == 1 ? " thread" : " threads");
+            String draining  = Boolean.TRUE.equals(worker.getValueBoolean(DRAINING)) ? "Yes" : "No";
+            String heartbeat = JobsFormat.relative(worker.getValueInstant(HEARTBEAT_AT), now);
+            String started   = "Started " + JobsFormat.time(worker.getValueInstant(STARTED_AT), now);
+            Map<String, Object> row = row("Pod", worker.getValueString(POD_ID), "Threads", capacity, "Draining", draining,
+               "Last heartbeat", heartbeat, "Started", started);
+            row.put(POD_CELL, column().withBlock(text(worker.getValueString(POD_ID), "semibold", null)).withBlock(small(started)));
+            row.put(CAPACITY_CELL, column().withBlock(text(capacity, null, null)).withBlock(small("Draining: " + draining)));
+            rows.add(row);
+         }
+      }
+      TableData table = table(admin ? "No workers have registered" : "Requires jobs_admin permission", rows,
+         col("Pod", POD_CELL), col("Capacity", CAPACITY_CELL), new TableData.Column("default", "Last heartbeat", "Last heartbeat", null, "left"));
+      return (table);
+   }
+
+
+
+   /*******************************************************************************
+    ** Aggregate chunks into 24 UTC hour buckets and the top five processes,
+    ** named by their labels. Job process names come from one query.
+    *******************************************************************************/
+   private ChartData throughput(Instant now) throws QException
+   {
+      Instant       start  = now.truncatedTo(ChronoUnit.HOURS).minus(23, ChronoUnit.HOURS);
+      List<QRecord> chunks = query(JOBS_CHUNK, statuses(List.of(DONE)).withSubFilter(finishedSince(start)));
+      Set<Serializable> jobIds = new HashSet<>();
+      chunks.forEach(chunk -> jobIds.add(chunk.getValue(JOB_ID)));
+      Map<Serializable, String> processes = new HashMap<>();
+      if(!jobIds.isEmpty())
+      {
+         for(QRecord job : query(JOBS_JOB, new QQueryFilter().withCriteria(ID, QCriteriaOperator.IN, jobIds)))
+         {
+            processes.put(job.getValue(ID), processLabel(job.getValueString(PROCESS_NAME)));
+         }
+      }
+      Map<String, Long> totals = new HashMap<>();
+      for(QRecord chunk : chunks)
+      {
+         String process = processes.get(chunk.getValue(JOB_ID));
+         if(process != null)
+         {
+            totals.merge(process, number(chunk, ROWS_IN), Long::sum);
+         }
+      }
+      List<String> top = totals.keySet().stream().sorted(Comparator.<String>comparingLong(totals::get).reversed().thenComparing(Comparator.naturalOrder())).limit(5).toList();
+      Map<String, long[]> series = new LinkedHashMap<>();
+      top.forEach(process -> series.put(process, new long[24]));
+      for(QRecord chunk : chunks)
+      {
+         String process = processes.get(chunk.getValue(JOB_ID));
+         if(process != null)
+         {
+            String group  = top.contains(process) ? process : "Other";
+            int    bucket = (int) Duration.between(start, chunk.getValueInstant(FINISHED_AT)).toHours();
+            if(bucket >= 0 && bucket < 24)
+            {
+               series.computeIfAbsent(group, key -> new long[24])[bucket] += number(chunk, ROWS_IN);
+            }
+         }
+      }
+      List<String>      labels = new ArrayList<>();
+      DateTimeFormatter hour   = DateTimeFormatter.ofPattern("HH:mm", Locale.US);
+      for(int i = 0; i < 24; i++)
+      {
+         labels.add(start.plus(i, ChronoUnit.HOURS).atOffset(ZoneOffset.UTC).format(hour));
+      }
+      return (chart("Rows processed per hour, last 24 hours (UTC)", labels, series));
+   }
+
+
+
+   /*******************************************************************************
+    ** Thirty UTC dates including today, split by final status.
+    *******************************************************************************/
+   private ChartData history(Instant now) throws QException
+   {
+      LocalDate           first  = now.atOffset(ZoneOffset.UTC).toLocalDate().minusDays(29);
+      Map<String, long[]> series = new LinkedHashMap<>();
+      TERMINAL_STATUSES.forEach(status -> series.put(statusLabel(status), new long[30]));
+      for(QRecord job : query(JOBS_JOB, statuses(TERMINAL_STATUSES).withSubFilter(finishedSince(first.atStartOfDay().toInstant(ZoneOffset.UTC)))))
+      {
+         int bucket = (int) ChronoUnit.DAYS.between(first, job.getValueInstant(FINISHED_AT).atOffset(ZoneOffset.UTC).toLocalDate());
+         if(bucket >= 0 && bucket < 30)
+         {
+            series.get(statusLabel(job.getValueString(STATUS)))[bucket]++;
+         }
+      }
+      List<String>      labels = new ArrayList<>();
+      DateTimeFormatter day    = DateTimeFormatter.ofPattern("MMM d", Locale.US);
+      for(int i = 0; i < 30; i++)
+      {
+         labels.add(first.plusDays(i).format(day));
+      }
+      return (chart("Jobs finished per day, last 30 days (UTC)", labels, series));
+   }
+
+
+
+   /*******************************************************************************
+    ** Record progress: rows, chunks, speed and optional effects, scoped to the viewer.
+    *******************************************************************************/
+   private MultiStatisticsData progress(QRecord job, Instant now) throws QException
+   {
+      if(job == null)
+      {
+         return (new MultiStatisticsData(null, List.of()));
+      }
+      String status  = job.getValueString(STATUS);
+      String percent = percent(job);
+
+      List<MultiStatisticsData.StatisticsGroupData.Statistic> rowStats = new ArrayList<>();
+      rowStats.add(statistic("Succeeded", number(job, ROWS_OK), null));
+      rowStats.add(statistic("Failed", number(job, ROWS_FAILED), null));
+      if(job.getValue(ROWS_TOTAL) != null)
+      {
+         rowStats.add(statistic("Total", number(job, ROWS_TOTAL), null));
+      }
+      MultiStatisticsData.StatisticsGroupData rows = new MultiStatisticsData.StatisticsGroupData().withHeader("Rows")
+         .withSubheader(percentValue(job) == null ? "Total not known yet" : percent + " done").withStatisticList(rowStats);
+
+      List<MultiStatisticsData.StatisticsGroupData.Statistic> chunkStats = new ArrayList<>();
+      chunkStats.add(statistic("Done", number(job, CHUNKS_DONE), null));
+      chunkStats.add(statistic("Failed", number(job, CHUNKS_FAILED), null));
+      if(job.getValue(CHUNKS_TOTAL) != null)
+      {
+         chunkStats.add(statistic("Total", number(job, CHUNKS_TOTAL), null));
+      }
+      MultiStatisticsData.StatisticsGroupData chunks = new MultiStatisticsData.StatisticsGroupData().withHeader("Chunks").withStatisticList(chunkStats);
+
+      String speedNote;
+      if(job.getValue(FINISHED_AT) != null)
+      {
+         speedNote = statusLabel(status) + " " + JobsFormat.time(job.getValueInstant(FINISHED_AT), now);
+      }
+      else if(job.getValue(ETA_SECONDS) != null || RUNNING_STATUSES.contains(status))
+      {
+         speedNote = "ETA " + eta(job, now);
+      }
+      else
+      {
+         speedNote = statusLabel(status);
+      }
+      MultiStatisticsData.StatisticsGroupData speed = new MultiStatisticsData.StatisticsGroupData().withHeader("Speed").withSubheader(speedNote)
+         .withStatisticList(List.of(statistic("Rows per minute", Math.round(rate(job, now)), null)));
+      List<MultiStatisticsData.StatisticsGroupData> groups = new ArrayList<>(List.of(rows, chunks, speed));
+      List<String> details = progressDetails(job);
+      if(!details.isEmpty())
+      {
+         groups.addFirst(new MultiStatisticsData.StatisticsGroupData().withHeader("Progress")
+            .withSubheader(String.join(" | ", details)).withStatisticList(List.of()));
+      }
+      if(number(job, LOG_DROPPED) > 0)
+      {
+         groups.add(new MultiStatisticsData.StatisticsGroupData().withHeader("Log")
+            .withSubheader(JobsFormat.count(number(job, LOG_DROPPED)) + " earlier lines not kept").withStatisticList(List.of()));
+      }
+      if(QContext.getQInstance().getTable(JOBS_EFFECT) != null)
+      {
+         List<QRecord> effects = query(JOBS_EFFECT, new QQueryFilter().withCriteria(JOB_ID, QCriteriaOperator.EQUALS, job.getValue(ID)));
+         if(!effects.isEmpty())
+         {
+            long sent = 0;
+            long pending = 0;
+            long failed = 0;
+            long review = 0;
+            for(QRecord effect : effects)
+            {
+               if(Boolean.TRUE.equals(effect.getValueBoolean(NEEDS_REVIEW_FIELD)))
+               {
+                  review++;
+               }
+               else
+               {
+                  String effectStatus = effect.getValueString(STATUS);
+                  sent += SENT.equals(effectStatus) ? 1 : 0;
+                  failed += FAILED.equals(effectStatus) ? 1 : 0;
+                  pending += PENDING.equals(effectStatus) || SENDING.equals(effectStatus) || UNKNOWN.equals(effectStatus) ? 1 : 0;
+               }
+            }
+            groups.add(new MultiStatisticsData.StatisticsGroupData().withHeader("Effects").withStatisticList(List.of(
+               statistic("Sent", sent, null), statistic("Pending", pending, null),
+               statistic("Failed", failed, null), statistic("Needs review", review, null))));
+         }
+      }
+      return (new MultiStatisticsData(null, groups));
+   }
+
+
+
+   /*******************************************************************************
+    ** Link encoding follows core ChildRecordListRenderer's filtered table link.
+    *******************************************************************************/
+   private StatisticsData audit(QRecord job) throws QException
+   {
+      StatisticsData data = new StatisticsData().withCount(0).withCountContext("Job not found or not accessible");
+      data.setLabel("Changes made by this job");
+      if(job != null && QContext.getQInstance().getTable(AUDIT) != null)
+      {
+         QQueryFilter filter = new QQueryFilter().withCriteria(MESSAGE, QCriteriaOperator.CONTAINS, "job:" + job.getValue(ID) + "/");
+         int          count  = new CountAction().execute(new CountInput(AUDIT).withFilter(filter)).getCount();
+         data.withCount(count).withCountContext("Audit rows").withCountURL(tableUrl(AUDIT, filter));
+      }
+      return (data);
+   }
+
+
+
+   /*******************************************************************************
+    ** A table's path with a filter, as core's filtered table links encode it.
+    *******************************************************************************/
+   private static String tableUrl(String table, QQueryFilter filter) throws QException
+   {
+      String path = QContext.getQInstance().getTablePath(table);
+      return ((path == null ? "/" + table : path) + "?filter=" + URLEncoder.encode(JsonUtils.toJson(filter), StandardCharsets.UTF_8));
+   }
+
+
+
+   /*******************************************************************************
+    ** The jobs table filtered to a summary statistic's jobs.
+    *******************************************************************************/
+   private static String jobsUrl(QQueryFilter filter) throws QException
+   {
+      return (tableUrl(JOBS_JOB, filter));
+   }
+
+
+
+   /*******************************************************************************
+    ** A table with no inner title (the widget label is the title), at most 25 rows.
+    *******************************************************************************/
+   private static TableData table(String empty, List<Map<String, Object>> rows, TableData.Column... columns)
+   {
+      return (new TableData(null, Arrays.asList(columns), rows).withNoRowsFoundHTML(empty).withRowsPerPage(25).withHidePaginationDropdown(true));
+   }
+
+
+
+   /*******************************************************************************
+    ** A flexible-width composite (block) column.
+    *******************************************************************************/
+   private static TableData.Column col(String header, String accessor)
+   {
+      return (new TableData.Column("composite", header, accessor, null, "left"));
+   }
+
+
+
+   /*******************************************************************************
+    ** Preserve key order.
+    *******************************************************************************/
+   private static Map<String, Object> row(Object... pairs)
+   {
+      Map<String, Object> row = new LinkedHashMap<>();
+      for(int i = 0; i < pairs.length; i += 2)
+      {
+         row.put((String) pairs[i], pairs[i + 1]);
+      }
+      return (row);
+   }
+
+
+
+   /*******************************************************************************
+    ** The job cell: its label linked to the job record, the process label under
+    ** it, and optionally a note (the error) on its own line.
+    *******************************************************************************/
+   private static CompositeWidgetData jobCell(QRecord job, String note) throws QException
+   {
+      String label = Objects.toString(job.getValueString(LABEL), "Job " + job.getValue(ID));
+      CompositeWidgetData cell = column()
+         .withBlock(text(label, "semibold", null).withLink(new BlockLink(jobPath(job))))
+         .withBlock(small(processLabel(job.getValueString(PROCESS_NAME))));
+      if(note != null)
+      {
+         cell.withBlock(text(note, null, "ERROR").withStyles(new TextStyles().withSize(SMALL).withColor("ERROR")));
+      }
+      return (cell);
+   }
+
+
+
+   /*******************************************************************************
+    ** A vertical stack of blocks.
+    *******************************************************************************/
+   private static CompositeWidgetData column()
+   {
+      return (new CompositeWidgetData().withLayout(CompositeWidgetData.Layout.FLEX_COLUMN));
+   }
+
+
+
+   /*******************************************************************************
+    ** A text block, with optional weight and standard color.
+    *******************************************************************************/
+   private static TextBlockData text(String value, String weight, String color)
+   {
+      TextStyles styles = new TextStyles();
+      if(weight != null)
+      {
+         styles.withWeight(weight);
+      }
+      if(color != null)
+      {
+         styles.withColor(color);
+      }
+      return (new TextBlockData().withValues(new TextValues(value)).withStyles(styles));
+   }
+
+
+
+   /*******************************************************************************
+    ** A small muted secondary line.
+    *******************************************************************************/
+   private static TextBlockData small(String value)
+   {
+      return (new TextBlockData().withValues(new TextValues(value)).withStyles(new TextStyles().withSize(SMALL).withColor("MUTED")));
+   }
+
+
+
+   /*******************************************************************************
+    ** One whole-number statistic (rates are rounded; huge values are capped).
+    *******************************************************************************/
+   private static MultiStatisticsData.StatisticsGroupData.Statistic statistic(String label, long value, String url)
+   {
+      return (new MultiStatisticsData.StatisticsGroupData.Statistic(label, (int) Math.min(Integer.MAX_VALUE, value), url));
+   }
+
+
+
+   /*******************************************************************************
+    ** Typed stacked chart with deterministic series colors and no inner title.
+    *******************************************************************************/
+   private static ChartData chart(String description, List<String> labels, Map<String, long[]> series)
+   {
+      String[]                     colors   = {"#2563eb", "#16a34a", "#d97706", "#dc2626", "#7c3aed", "#64748b"};
+      List<ChartData.Data.Dataset> datasets = new ArrayList<>();
+      for(Map.Entry<String, long[]> entry : series.entrySet())
+      {
+         List<Number> values = Arrays.stream(entry.getValue()).mapToObj(value -> (Number) value).toList();
+         datasets.add(new ChartData.Data.Dataset().withLabel(entry.getKey()).withData(values).withBackgroundColor(colors[datasets.size() % colors.length]));
+      }
+      return (new JobsChartData().withDescription(description).withChartData(new ChartData.Data().withLabels(labels).withDatasets(datasets)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Null counters mean no recorded progress.
+    *******************************************************************************/
+   private static long number(QRecord record, String field)
+   {
+      Long value = record.getValueLong(field);
+      return (value == null ? 0 : value);
+   }
+
+
+
+   /*******************************************************************************
+    ** A counter with thousands separators.
+    *******************************************************************************/
+   private static String count(QRecord record, String field)
+   {
+      return (JobsFormat.count(number(record, field)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Inclusive lower bound, with unfinished jobs excluded.
+    *******************************************************************************/
+   private static boolean since(QRecord record, String field, Instant cutoff)
+   {
+      Instant value = record.getValueInstant(field);
+      return (value != null && !value.isBefore(cutoff));
+   }
+
+
+
+   /*******************************************************************************
+    ** Distinguish an unknown total from a known zero.
+    *******************************************************************************/
+   private static String total(QRecord job, String field)
+   {
+      return (job.getValue(field) == null ? "Unknown" : count(job, field));
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve the phase label from the optional snapshot; old or malformed data
+    ** still displays the reported phase name.
+    *******************************************************************************/
+   private static String phaseLabel(QRecord job)
+   {
+      String current = job.getValueString(CURRENT_PHASE);
+      String json = job.getValueString(PROGRESS_JSON);
+      if(current != null && json != null)
+      {
+         try
+         {
+            JsonNode snapshot = JsonUtils.toObject(json, JsonNode.class);
+            for(JsonNode phase : snapshot == null ? List.<JsonNode>of() : snapshot.path("phases"))
+            {
+               if(current.equals(phase.path("name").asText()) && !phase.path("label").asText().isBlank())
+               {
+                  return (phase.path("label").asText());
+               }
+            }
+         }
+         catch(IOException ignored)
+         {
+            // The denormalized progress remains usable without a valid snapshot.
+         }
+      }
+      return (current);
+   }
+
+   /*******************************************************************************
+    ** Optional byte counts, retaining unknown totals rather than inventing zero.
+    *******************************************************************************/
+   private static String bytes(QRecord job)
+   {
+      String done = job.getValue(BYTES_DONE) == null ? null : JobsFormat.bytes(number(job, BYTES_DONE));
+      String total = job.getValue(BYTES_TOTAL) == null ? null : JobsFormat.bytes(number(job, BYTES_TOTAL));
+      return (total == null ? done : done == null ? total + " total" : done + " / " + total);
+   }
+
+   /*******************************************************************************
+    ** Plain typed text shared by the home and record progress views.
+    *******************************************************************************/
+   private static List<String> progressDetails(QRecord job)
+   {
+      List<String> details = new ArrayList<>();
+      for(String detail : new String[] {phaseLabel(job), job.getValueString(PROGRESS_MESSAGE), bytes(job)})
+      {
+         if(detail != null && !detail.isBlank())
+         {
+            details.add(detail);
+         }
+      }
+      return (details);
+   }
+
+   /*******************************************************************************
+    ** Explicit percent, else rows then chunks; null when no total is known.
+    *******************************************************************************/
+   private static BigDecimal percentValue(QRecord job)
+   {
+      if(job.getValue(PERCENT_DONE) != null)
+      {
+         return (BigDecimal.valueOf(Math.max(0, Math.min(100, number(job, PERCENT_DONE)))));
+      }
+      long total = number(job, ROWS_TOTAL);
+      long done  = number(job, ROWS_DONE);
+      if(total <= 0)
+      {
+         total = number(job, CHUNKS_TOTAL);
+         done = number(job, CHUNKS_DONE);
+      }
+      return (total <= 0 ? null : BigDecimal.valueOf(Math.min(100, Math.round(100.0 * done / total))));
+   }
+
+
+
+   /*******************************************************************************
+    ** Percent done as text ("45%"), or "Unknown".
+    *******************************************************************************/
+   private static String percent(QRecord job)
+   {
+      BigDecimal value = percentValue(job);
+      return (value == null ? "Unknown" : value.toPlainString() + "%");
+   }
+
+
+
+   /*******************************************************************************
+    ** Lifetime processing rate in rows per minute, frozen at finishedAt.
+    *******************************************************************************/
+   private static double rate(QRecord job, Instant now)
+   {
+      Instant start   = job.getValueInstant(STARTED_AT);
+      Instant finish  = job.getValueInstant(FINISHED_AT);
+      long    seconds = start == null ? 0 : Duration.between(start, finish == null ? now : finish).getSeconds();
+      return (seconds <= 0 ? 0 : number(job, ROWS_DONE) * 60.0 / seconds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Relative ETA ("in 8m") for a running job with a rate and a known total.
+    *******************************************************************************/
+   private static String eta(QRecord job, Instant now)
+   {
+      if(job.getValue(ETA_SECONDS) != null)
+      {
+         return ("in " + JobsFormat.span(number(job, ETA_SECONDS)));
+      }
+      double rate = rate(job, now);
+      if(!RUNNING_STATUSES.contains(job.getValueString(STATUS)) || rate <= 0 || job.getValue(ROWS_TOTAL) == null)
+      {
+         return ("Unknown");
+      }
+      long seconds = (long) Math.ceil(Math.max(0, number(job, ROWS_TOTAL) - number(job, ROWS_DONE)) * 60.0 / rate);
+      return ("in " + JobsFormat.span(seconds));
+   }
+
+
+
+   /*******************************************************************************
+    ** The job's record path, from the instance's table path (frontends map it).
+    *******************************************************************************/
+   private static String jobPath(QRecord job) throws QException
+   {
+      String path = QContext.getQInstance().getTablePath(JOBS_JOB);
+      return ((path == null ? "/" + JOBS_JOB : path) + "/" + job.getValue(ID));
+   }
+}

src/main/resources/jobs.properties

@@ -0,0 +1 @@
+version=${project.version}

src/test/java/com/kingsrook/qbits/jobs/BaseJobsTest.java

@@ -0,0 +1,263 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import com.kingsrook.qqq.backend.core.actions.dashboard.RenderWidgetAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.audits.AuditsMetaDataProvider;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.model.session.QUser;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.AUDIT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.AWAITING_CONFIRMATION;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CANCELLED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CANCELLING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED_WITH_ERRORS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.DETAIL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.DRAINING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.EFFECT_NAME;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.EFFECT_TYPE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ERROR_MESSAGE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FINISHED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.HEARTBEAT_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.IDEMPOTENCY_KEY;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.IDEMPOTENT_EXTERNAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.INPUT_STORAGE_REF;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ADMIN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ROW_ERROR;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_UPLOAD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_WORKER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LABEL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LAST_PROGRESS_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.MESSAGE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW_FIELD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PAUSED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PAUSING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PLANNING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.POD_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PRIORITY;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PROCESS_NAME;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.QUEUED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.QUEUED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.REF;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_IN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_OK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROW_NUMBER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.RUNNING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SEQ;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STARTED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STORAGE_REFERENCE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STORAGE_TABLE_NAME;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.TERMINAL_STATUSES;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.THREADS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.TYPE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.USER_ID;
+
+
+/*******************************************************************************
+ ** Memory instance and fixtures covering every job status and two owners.
+ *******************************************************************************/
+public class BaseJobsTest
+{
+   protected QInstance instance;
+   protected JobsQBitConfig config;
+   protected Instant now;
+   protected static final String USER_A = "user-a";
+   protected static final String USER_B = "user-b";
+
+   /*******************************************************************************
+    ** Fresh store and instance.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      MemoryRecordStore.fullReset();
+      instance = new QInstance();
+      instance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
+      instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK));
+      new DurableJobsMetaDataProvider().defineAll(instance, "memory", null);
+      instance.withSupplementalMetaData(new DurableJobsMetaData());
+      new AuditsMetaDataProvider().defineAll(instance, "memory", null);
+      config = new JobsQBitConfig();
+      new JobsQBitProducer().withQBitConfig(config).produce(instance);
+      new QInstanceEnricher(instance).enrich();
+      QContext.init(instance, new QSystemUserSession(instance));
+      now = Instant.now();
+   }
+
+   /*******************************************************************************
+    ** Clear thread locals and memory between tests.
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      QContext.clear();
+      MemoryRecordStore.fullReset();
+   }
+
+   /*******************************************************************************
+    ** Start without pre-applied jobs keys.
+    *******************************************************************************/
+   protected QSession login(String user, boolean admin)
+   {
+      QSession session = new QSession().withUser(new QUser().withIdReference(user));
+      if(admin)
+      {
+         session.withPermission(JOBS_ADMIN);
+      }
+      QContext.setQSession(session);
+      return (session);
+   }
+
+   /*******************************************************************************
+    ** Create data through core actions.
+    *******************************************************************************/
+   protected QRecord insert(String table, QRecord record) throws QException
+   {
+      QRecord inserted = new InsertAction().execute(new InsertInput().withTableName(table).withRecord(record)).getRecords().getFirst();
+      Assertions.assertThat(inserted.getErrors()).isNullOrEmpty();
+      return (inserted);
+   }
+
+   /*******************************************************************************
+    ** Job with known counters.
+    *******************************************************************************/
+   protected QRecord job(String user, String status, Instant finished) throws QException
+   {
+      return (insert(JOBS_JOB, new QRecord().withValue(USER_ID, user).withValue(LABEL, user + " " + status)
+         .withValue(PROCESS_NAME, "importOrders").withValue(STATUS, status).withValue(PRIORITY, 5)
+         .withValue(QUEUED_AT, now.minus(11, ChronoUnit.MINUTES)).withValue(STARTED_AT, now.minus(10, ChronoUnit.MINUTES))
+         .withValue(FINISHED_AT, finished).withValue(LAST_PROGRESS_AT, now.minus(2, ChronoUnit.MINUTES))
+         .withValue(ROWS_TOTAL, 1000).withValue(ROWS_DONE, 450).withValue(ROWS_OK, 440).withValue(ROWS_FAILED, 10)
+         .withValue(CHUNKS_TOTAL, 10).withValue(CHUNKS_DONE, 4).withValue(ERROR_MESSAGE, "x".repeat(140))));
+   }
+
+   /*******************************************************************************
+    ** Related rows for join and retention tests.
+    *******************************************************************************/
+   protected void children(QRecord job) throws QException
+   {
+      Serializable id = job.getValue(ID);
+      insert(JOBS_CHUNK, new QRecord().withValue(JOB_ID, id).withValue(SEQ, 1).withValue(STATUS, DONE)
+         .withValue(ROWS_IN, 100).withValue(FINISHED_AT, now.minus(2, ChronoUnit.HOURS)).withValue(INPUT_STORAGE_REF, "chunk/" + id));
+      insert(JOBS_ROW_ERROR, new QRecord().withValue(JOB_ID, id).withValue(ROW_NUMBER, 5).withValue(MESSAGE, "Bad row"));
+      insert(JOBS_EVENT, new QRecord().withValue(JOB_ID, id).withValue(TYPE, COMPLETED).withValue(DETAIL, "Finished"));
+      insert(JOBS_UPLOAD, new QRecord().withValue(JOB_ID, id).withValue(USER_ID, job.getValue(USER_ID))
+         .withValue(REF, "upload/" + id).withValue(STORAGE_TABLE_NAME, "objects").withValue(STORAGE_REFERENCE, "input/" + id));
+      insert(AUDIT, new QRecord().withValue(MESSAGE, "job:" + id + "/chunk:1 changed a row"));
+   }
+
+   /*******************************************************************************
+    ** An outbox entry with a unique idempotency key, independent of other fixtures.
+    *******************************************************************************/
+   protected QRecord effect(QRecord job, String status, boolean needsReview) throws QException
+   {
+      return (insert(JOBS_EFFECT, new QRecord().withValue(JOB_ID, job.getValue(ID)).withValue(EFFECT_NAME, "Send notification")
+         .withValue(STATUS, status).withValue(NEEDS_REVIEW_FIELD, needsReview).withValue(EFFECT_TYPE, IDEMPOTENT_EXTERNAL)
+         .withValue(IDEMPOTENCY_KEY, UUID.randomUUID().toString())));
+   }
+
+   /*******************************************************************************
+    ** Every status plus another owner's running job and one worker.
+    *******************************************************************************/
+   protected List<QRecord> seed() throws QException
+   {
+      List<String> statuses = List.of(PLANNING, AWAITING_CONFIRMATION, QUEUED, RUNNING, PAUSING, PAUSED, CANCELLING,
+         CANCELLED, COMPLETED, COMPLETED_WITH_ERRORS, FAILED, NEEDS_REVIEW);
+      List<QRecord> jobs = new ArrayList<>();
+      for(String status : statuses)
+      {
+         boolean finished = TERMINAL_STATUSES.contains(status) || NEEDS_REVIEW.equals(status);
+         QRecord job = job(USER_A, status, finished ? now.minus(30, ChronoUnit.MINUTES) : null);
+         children(job);
+         jobs.add(job);
+      }
+      children(job(USER_B, RUNNING, null));
+      insert(JOBS_WORKER, new QRecord().withValue(POD_ID, "pod-1").withValue(THREADS, 4).withValue(DRAINING, false)
+         .withValue(HEARTBEAT_AT, now.minusSeconds(12)).withValue(STARTED_AT, now.minus(1, ChronoUnit.DAYS)));
+      return (jobs);
+   }
+
+   /*******************************************************************************
+    ** Use the widget action so metadata default query parameters are applied.
+    *******************************************************************************/
+   protected QWidgetData render(String name, String id) throws QException
+   {
+      RenderWidgetInput input = new RenderWidgetInput().withWidgetMetaData(instance.getWidget(name));
+      if(id != null)
+      {
+         input.addQueryParam("id", id);
+      }
+      return (new RenderWidgetAction().execute(input).getWidgetData());
+   }
+
+   /*******************************************************************************
+    ** Read fixtures as the current session.
+    *******************************************************************************/
+   protected List<QRecord> records(String table) throws QException
+   {
+      return (new QueryAction().execute(new QueryInput().withTableName(table)).getRecords());
+   }
+}

src/test/java/com/kingsrook/qbits/jobs/JobsAppPermissionTest.java

@@ -0,0 +1,109 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.actions.metadata.MetaDataAction;
+import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
+import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataInput;
+import com.kingsrook.qqq.backend.core.model.actions.metadata.MetaDataOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.DenyBehavior;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** The Jobs app and its home widgets are hidden from users without
+ ** "jobs.hasAccess"; Workers additionally needs jobs_admin.
+ *******************************************************************************/
+class JobsAppPermissionTest extends BaseJobsTest
+{
+   private static final List<String> NON_ADMIN_WIDGETS = JobsQBitProducer.HOME_WIDGETS.stream().filter(w -> !"jobsHomeWorkers".equals(w)).toList();
+
+
+
+   /*******************************************************************************
+    ** The permission name is public and follows the configured app name.
+    *******************************************************************************/
+   @Test
+   void permissionNameAndRules()
+   {
+      assertThat(config.getHasAccessPermissionName()).isEqualTo("jobs.hasAccess");
+      assertThat(new JobsQBitConfig().withAppName("bg").getHasAccessPermissionName()).isEqualTo("bg.hasAccess");
+      assertThat(instance.getApp("jobs").getPermissionRules().getLevel()).isEqualTo(PermissionLevel.HAS_ACCESS_PERMISSION);
+      assertThat(instance.getApp("jobs").getPermissionRules().getPermissionBaseName()).isEqualTo("jobs");
+      assertThat(instance.getApp("jobs").getPermissionRules().getDenyBehavior()).isEqualTo(DenyBehavior.HIDDEN);
+      assertThat(instance.getWidget("jobsJobProgress").getPermissionRules().getLevel()).isNotEqualTo(PermissionLevel.HAS_ACCESS_PERMISSION);
+      for(String name : NON_ADMIN_WIDGETS)
+      {
+         assertThat(instance.getWidget(name).getPermissionRules().getPermissionBaseName()).isEqualTo("jobs");
+         assertThat(instance.getWidget(name).getPermissionRules().getDenyBehavior()).isEqualTo(DenyBehavior.HIDDEN);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** No permissions: no app, no home widgets, every render denied.
+    *******************************************************************************/
+   @Test
+   void noPermissionsSeesNothing() throws Exception
+   {
+      instance.setEnforcePermissionsInCoreActions(true);
+      login("nobody", false);
+      MetaDataOutput output = new MetaDataAction().execute(new MetaDataInput());
+      assertThat(output.getApps()).doesNotContainKey("jobs");
+      assertThat(output.getWidgets()).doesNotContainKeys(JobsQBitProducer.HOME_WIDGETS.toArray(String[]::new));
+      for(String name : JobsQBitProducer.HOME_WIDGETS)
+      {
+         assertThatThrownBy(() -> render(name, null)).isInstanceOf(QPermissionDeniedException.class);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** jobs.hasAccess: app and the five widgets, Workers only with jobs_admin.
+    *******************************************************************************/
+   @Test
+   void hasAccessSeesAppAndWidgets() throws Exception
+   {
+      instance.setEnforcePermissionsInCoreActions(true);
+      login("user", false).withPermission(config.getHasAccessPermissionName());
+      MetaDataOutput output = new MetaDataAction().execute(new MetaDataInput());
+      assertThat(output.getApps()).containsKey("jobs");
+      assertThat(output.getWidgets()).containsKeys(NON_ADMIN_WIDGETS.toArray(String[]::new));
+      assertThat(output.getWidgets()).doesNotContainKey("jobsHomeWorkers");
+      for(String name : NON_ADMIN_WIDGETS)
+      {
+         assertThat(render(name, null)).isNotNull();
+      }
+      assertThatThrownBy(() -> render("jobsHomeWorkers", null)).isInstanceOf(QPermissionDeniedException.class);
+
+      login("admin", true).withPermission(config.getHasAccessPermissionName());
+      assertThat(new MetaDataAction().execute(new MetaDataInput()).getWidgets()).containsKey("jobsHomeWorkers");
+      assertThat(render("jobsHomeWorkers", null)).isNotNull();
+   }
+}

src/test/java/com/kingsrook/qbits/jobs/JobsFormatTest.java

@@ -0,0 +1,79 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import com.kingsrook.qbits.jobs.widgets.JobsFormat;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Human-readable counts, times and labels.
+ *******************************************************************************/
+class JobsFormatTest
+{
+   private static final Instant NOW = Instant.parse("2026-10-14T08:46:30Z");
+
+
+
+   /*******************************************************************************
+    ** Relative within a day, absolute UTC beyond it, never ISO.
+    *******************************************************************************/
+   @Test
+   void times()
+   {
+      assertThat(JobsFormat.time(null, NOW)).isEqualTo("Unknown");
+      assertThat(JobsFormat.time(NOW.minusSeconds(2), NOW)).isEqualTo("just now");
+      assertThat(JobsFormat.time(NOW.minusSeconds(45), NOW)).isEqualTo("45s ago");
+      assertThat(JobsFormat.time(NOW.minus(4, ChronoUnit.MINUTES), NOW)).isEqualTo("4m ago");
+      assertThat(JobsFormat.time(NOW.minus(2, ChronoUnit.HOURS), NOW)).isEqualTo("2h ago");
+      assertThat(JobsFormat.time(NOW.plus(3, ChronoUnit.MINUTES), NOW)).isEqualTo("in 3m");
+      assertThat(JobsFormat.time(NOW.minus(24, ChronoUnit.HOURS), NOW)).isEqualTo("Oct 13, 08:46 UTC");
+      assertThat(JobsFormat.time(NOW.plus(2, ChronoUnit.DAYS), NOW)).isEqualTo("Oct 16, 08:46 UTC");
+      assertThat(JobsFormat.relative(NOW.minus(3, ChronoUnit.DAYS), NOW)).isEqualTo("3d ago");
+      assertThat(JobsFormat.relative(null, NOW)).isEqualTo("Unknown");
+      assertThat(JobsFormat.span(-5)).isEqualTo("0s");
+   }
+
+
+
+   /*******************************************************************************
+    ** Counts, clipping and status labels.
+    *******************************************************************************/
+   @Test
+   void countsClipsAndLabels()
+   {
+      assertThat(JobsFormat.count(1234567)).isEqualTo("1,234,567");
+      assertThat(JobsFormat.clip(null, 10)).isEmpty();
+      assertThat(JobsFormat.clip("  short\n text ", 20)).isEqualTo("short text");
+      assertThat(JobsFormat.clip("abcdefghijkl", 5)).isEqualTo("abcd" + JobsFormat.ELLIPSIS).hasSize(5);
+      assertThat(JobsCoreContract.statusLabel(JobsCoreContract.COMPLETED_WITH_ERRORS)).isEqualTo("Completed With Errors");
+      assertThat(JobsCoreContract.statusLabel(JobsCoreContract.AWAITING_CONFIRMATION)).isEqualTo("Awaiting Confirmation");
+      assertThat(JobsCoreContract.statusLabel(JobsCoreContract.NEEDS_REVIEW)).isEqualTo("Needs Review");
+      assertThat(JobsCoreContract.statusLabel(JobsCoreContract.PAUSED)).isEqualTo("Paused");
+      assertThat(JobsCoreContract.statusLabel("SOMETHING_NEW")).isEqualTo("SOMETHING_NEW");
+      assertThat(JobsCoreContract.statusLabel(null)).isEqualTo("Unknown");
+   }
+}

src/test/java/com/kingsrook/qbits/jobs/JobsProgressTest.java

@@ -0,0 +1,290 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import com.kingsrook.qbits.jobs.processes.JobsPurgeFinishedStep;
+import com.kingsrook.qbits.jobs.widgets.JobsFormat;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.security.RecordSecurityLock;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.BYTES_DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.BYTES_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CURRENT_PHASE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ETA_SECONDS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_LOG;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_USER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LEVEL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LOGGED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LOG_DROPPED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LOG_SEQ;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.MESSAGE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PERCENT_DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PHASE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PROGRESS_JSON;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PROGRESS_MESSAGE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.RUNNING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.USER_ID;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** Optional RC.8 progress and logs, including the actual core log permission checker.
+ *******************************************************************************/
+class JobsProgressTest extends BaseJobsTest
+{
+   /*******************************************************************************
+    ** Add optional fields and a log using the real kernel permission checker by
+    ** name, preserving the default suite's coverage of older kernel metadata.
+    *******************************************************************************/
+   private void enableProgress() throws Exception
+   {
+      for(String field : List.of(CURRENT_PHASE, PROGRESS_MESSAGE, PROGRESS_JSON))
+      {
+         instance.getTable(JOBS_JOB).withField(new QFieldMetaData(field, QFieldType.TEXT));
+      }
+      for(String field : List.of(PERCENT_DONE, LOG_DROPPED))
+      {
+         instance.getTable(JOBS_JOB).withField(new QFieldMetaData(field, QFieldType.INTEGER));
+      }
+      for(String field : List.of(BYTES_DONE, BYTES_TOTAL, ETA_SECONDS, LOG_SEQ))
+      {
+         instance.getTable(JOBS_JOB).withField(new QFieldMetaData(field, QFieldType.LONG));
+      }
+      instance.setEnforcePermissionsInCoreActions(true);
+      //////////////////////////////////////////////////////////////////////
+      // core 4.1.0-RC.8 defines jobs_log; older metadata gets this mirror //
+      //////////////////////////////////////////////////////////////////////
+      if(instance.getTable(JOBS_LOG) != null)
+      {
+         return;
+      }
+      instance.addTable(new QTableMetaData().withName(JOBS_LOG).withBackendName("memory").withPrimaryKeyField(ID)
+         .withField(new QFieldMetaData(ID, QFieldType.LONG))
+         .withField(new QFieldMetaData(JOB_ID, QFieldType.INTEGER))
+         .withField(new QFieldMetaData(LEVEL, QFieldType.STRING))
+         .withField(new QFieldMetaData(LOGGED_AT, QFieldType.DATE_TIME))
+         .withField(new QFieldMetaData(PHASE, QFieldType.STRING))
+         .withField(new QFieldMetaData(MESSAGE, QFieldType.STRING))
+         .withPermissionRules(new QPermissionRules().withCustomPermissionChecker(new QCodeReference(
+            Class.forName("com.kingsrook.qqq.backend.core.jobs.progress.JobLogPermissions"))))
+         .withRecordSecurityLock(new RecordSecurityLock().withSecurityKeyType(JOBS_USER).withFieldName(JOBS_JOB + "." + USER_ID)
+            .withJoinNameChain(List.of(JobsCoreContract.childJoin(JOBS_LOG)))));
+      instance.addJoin(new QJoinMetaData().withLeftTable(JOBS_JOB).withRightTable(JOBS_LOG).withType(JoinType.ONE_TO_MANY)
+         .withJoinOn(new JoinOn(ID, JOB_ID)).withInferredName());
+      clearPresentation();
+      new JobsQBitProducer().withQBitConfig(config).produce(instance);
+      new QInstanceEnricher(instance).enrich();
+   }
+
+   /*******************************************************************************
+    ** Rich fields override counters, resolve labels, and remain typed and flexible.
+    *******************************************************************************/
+   @Test
+   void richProgressOnHomeAndRecord() throws Exception
+   {
+      enableProgress();
+      QRecord job = job(USER_A, RUNNING, null);
+      update(job, new QRecord().withValue(CURRENT_PHASE, "upload").withValue(PERCENT_DONE, 73)
+         .withValue(PROGRESS_MESSAGE, "Uploading manifest").withValue(BYTES_DONE, 1536L).withValue(BYTES_TOTAL, 1048576L)
+         .withValue(ETA_SECONDS, 90L).withValue(LOG_DROPPED, 12)
+         .withValue(PROGRESS_JSON, "{\"phases\":[{\"name\":\"upload\",\"label\":\"Upload backup\"}]}"));
+      login(USER_A, false).withPermission(config.getHasAccessPermissionName());
+      TableData home = (TableData) render("jobsHomeActive", null);
+      assertThat(home.getRows()).singleElement().satisfies(row -> assertThat(row)
+         .containsEntry("Progress", "73%").containsEntry("Phase", "Upload backup")
+         .containsEntry("Message", "Uploading manifest").containsEntry("Bytes", "1.5 KiB / 1.0 MiB").containsEntry("ETA", "in 1m"));
+      assertThat(home.getColumns()).hasSize(3).allSatisfy(column -> assertThat(column.getWidth()).isNull());
+      MultiStatisticsData record = (MultiStatisticsData) render("jobsJobProgress", job.getValueString(ID));
+      assertThat(record.getStatisticsGroupData()).extracting(MultiStatisticsData.StatisticsGroupData::getSubheader)
+         .contains("Upload backup | Uploading manifest | 1.5 KiB / 1.0 MiB", "73% done", "ETA in 1m", "12 earlier lines not kept");
+      login(USER_B, false).withPermission(config.getHasAccessPermissionName());
+      assertThat(((TableData) render("jobsHomeActive", null)).getRows()).isEmpty();
+      assertThat(((MultiStatisticsData) render("jobsJobProgress", job.getValueString(ID))).getStatisticsGroupData()).isEmpty();
+   }
+
+   /*******************************************************************************
+    ** Null percent uses rows then chunks, zero is explicit, malformed JSON is safe.
+    *******************************************************************************/
+   @Test
+   void fallbacksAndPartialProgress() throws Exception
+   {
+      enableProgress();
+      QRecord job = job(USER_A, RUNNING, null);
+      update(job, new QRecord().withValue(CURRENT_PHASE, "copy").withValue(PROGRESS_JSON, "bad json").withValue(BYTES_TOTAL, 1024L));
+      assertThat(((TableData) render("jobsHomeActive", null)).getRows().getFirst())
+         .containsEntry("Progress", "45%").containsEntry("Phase", "copy").containsEntry("Bytes", "1.0 KiB total");
+      update(job, new QRecord().withValue(ROWS_TOTAL, null).withValue(PROGRESS_JSON, "null"));
+      assertThat(((TableData) render("jobsHomeActive", null)).getRows().getFirst()).containsEntry("Progress", "40%");
+      update(job, new QRecord().withValue(PERCENT_DONE, 0).withValue(BYTES_TOTAL, null).withValue(BYTES_DONE, 0L));
+      assertThat(((TableData) render("jobsHomeActive", null)).getRows().getFirst()).containsEntry("Progress", "0%").containsEntry("Bytes", "0 B");
+      update(job, new QRecord().withValue(PERCENT_DONE, null).withValue(CHUNKS_TOTAL, null));
+      assertThat(((TableData) render("jobsHomeActive", null)).getRows().getFirst()).containsEntry("Progress", "Unknown");
+      assertThat(JobsFormat.bytes(1073741824L)).isEqualTo("1.0 GiB");
+      assertThat(JobsFormat.bytes(Long.MAX_VALUE)).isEqualTo("8.0 EiB");
+   }
+
+   /*******************************************************************************
+    ** The log is ordered, full width and owner scoped, with no user mutations.
+    *******************************************************************************/
+   @Test
+   void logPanelAndPermissions() throws Exception
+   {
+      enableProgress();
+      QRecord job = job(USER_A, RUNNING, null);
+      log(job, 1000000002L);
+      log(job, 1000000001L);
+      assertThat(instance.getWidget("jobsJobLog").getGridColumns()).isEqualTo(12);
+      assertThat(instance.getTable(JOBS_JOB).getSections()).extracting(QFieldSection::getWidgetName)
+         .containsSubsequence("jobsJobProgress", "jobsJobLog", "jobsJobEvents");
+      login(USER_A, false).withPermission("jobs_job.read");
+      ChildRecordListData data = (ChildRecordListData) render("jobsJobLog", job.getValueString(ID));
+      assertThat(data.getQueryOutput().getRecords()).extracting(record -> record.getValueLong(ID)).containsExactly(1000000002L, 1000000001L);
+      assertThat(data.getOnlyIncludeFieldNames()).containsExactly(LEVEL, LOGGED_AT, PHASE, MESSAGE);
+      assertThat(data.getCanAddChildRecord()).isFalse();
+      assertThatThrownBy(() -> new DeleteAction().execute(new DeleteInput().withTableName(JOBS_LOG).withPrimaryKey(1000000001L))).hasMessageContaining("read-only");
+      assertThatThrownBy(() -> log(job, 1000000003L)).hasMessageContaining("read-only");
+      login("admin", true).withPermission("jobs_job.read");
+      assertThatThrownBy(() -> new UpdateAction().execute(new UpdateInput().withTableName(JOBS_LOG)
+         .withRecord(new QRecord().withValue(ID, 1000000001L).withValue(MESSAGE, "changed")))).hasMessageContaining("read-only");
+      login(USER_B, false).withPermission("jobs_job.read");
+      assertThatThrownBy(() -> render("jobsJobLog", job.getValueString(ID))).hasMessageContaining("Could not find");
+   }
+
+   /*******************************************************************************
+    ** The system purge crosses a full batch with LONG IDs; active logs remain.
+    *******************************************************************************/
+   @Test
+   void purgeLogBatchesWithSystemPermission() throws Exception
+   {
+      enableProgress();
+      QRecord old = job(USER_A, COMPLETED, now.minus(31, ChronoUnit.DAYS));
+      QRecord active = job(USER_B, RUNNING, null);
+      for(long id = 1000000000L; id < 1000000205L; id++)
+      {
+         log(old, id);
+      }
+      log(active, 1000000300L);
+      QRecord held = job(USER_A, COMPLETED, now.minus(31, ChronoUnit.DAYS));
+      effect(held, JobsCoreContract.PENDING, false);
+      log(held, 1000000301L);
+      var caller = login("admin", true);
+      new JobsPurgeFinishedStep().run(new RunBackendStepInput(), new RunBackendStepOutput());
+      assertThat(QContext.getQSession()).isSameAs(caller);
+      QContext.setQSession(new QSystemUserSession());
+      assertThat(records(JOBS_LOG)).extracting(record -> record.getValue(JOB_ID)).containsExactlyInAnyOrder(active.getValue(ID), held.getValue(ID));
+      assertThat(records(JOBS_JOB)).extracting(record -> record.getValue(ID)).containsExactlyInAnyOrder(active.getValue(ID), held.getValue(ID));
+   }
+
+   /*******************************************************************************
+    ** Log metadata is optional; hosts choose the Admin/Operations parent.
+    *******************************************************************************/
+   @Test
+   void optionalLogAndHostNavigation() throws Exception
+   {
+      assertThat(instance.getWidget("jobsJobLog")).as("core RC.8 defines jobs_log, so the Log list is produced").isNotNull();
+      enableProgress();
+      instance.getJoins().remove(JobsCoreContract.childJoin(JOBS_LOG));
+      instance.getWidgets().remove("jobsJobLog");
+      clearPresentation();
+      new JobsQBitProducer().produce(instance);
+      assertThat(instance.getWidget("jobsJobLog")).isNull();
+      QAppMetaData admin = new QAppMetaData().withName("admin").withLabel("Admin");
+      QAppMetaData operations = new QAppMetaData().withName("operations").withLabel("Operations");
+      operations.setParentAppName("admin");
+      instance.addApp(admin.withChild(operations));
+      instance.addApp(operations);
+      config.withAppName("backgroundJobs").withParentAppName("operations");
+      clearPresentation();
+      new JobsQBitProducer().withQBitConfig(config).produce(instance);
+      assertThat(instance.getApp("backgroundJobs").getParentAppName()).isEqualTo("operations");
+      assertThat(operations.getChildren()).extracting(child -> child.getName()).containsExactly("backgroundJobs");
+      config.setParentAppName("missing");
+      assertThatThrownBy(() -> config.validate(instance)).hasMessageContaining("parentAppName");
+      config.setParentAppName("backgroundJobs");
+      assertThatThrownBy(() -> config.validate(instance)).hasMessageContaining("parentAppName");
+      config.setParentAppName(null);
+      config.validate(instance);
+   }
+
+   /*******************************************************************************
+    ** Rebuild presentation against changed optional metadata in these tests.
+    *******************************************************************************/
+   private void clearPresentation()
+   {
+      instance.getQBits().clear();
+      instance.getWidgets().clear();
+      instance.getApps().remove("jobs");
+      instance.getProcesses().remove(JobsQBitProducer.PURGE_PROCESS);
+      instance.getTable(JOBS_JOB).getSections().removeIf(section -> section.getWidgetName() != null);
+   }
+
+   /*******************************************************************************
+    ** Persist progress through core actions.
+    *******************************************************************************/
+   private void update(QRecord job, QRecord values) throws Exception
+   {
+      var output = new UpdateAction().execute(new UpdateInput().withTableName(JOBS_JOB).withRecord(values.withValue(ID, job.getValue(ID))));
+      assertThat(output.getRecords().getFirst().getErrors()).isNullOrEmpty();
+   }
+
+   /*******************************************************************************
+    ** Explicit LONG cursors make ordering assertions deterministic. Core memory
+    ** sequence bookkeeping currently restricts inserts to the integer range.
+    *******************************************************************************/
+   private void log(QRecord job, long id) throws Exception
+   {
+      insert(JOBS_LOG, new QRecord().withValue(ID, id).withValue(JOB_ID, job.getValue(ID)).withValue(LEVEL, "INFO")
+         .withValue(LOGGED_AT, now).withValue(PHASE, "upload").withValue(MESSAGE, "Wrote manifest"));
+   }
+}

src/test/java/com/kingsrook/qbits/jobs/JobsQBitProducerTest.java

@@ -0,0 +1,266 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qqq.backend.core.alerts.AlertMonitorKind;
+import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
+import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
+import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaDataProvider;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ALERT_EFFECT_UNRESOLVED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ALERT_JOB_FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ALERT_JOB_STALLED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ATTEMPT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNK_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.HEARTBEAT_REAPER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT_STATUS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT_TYPE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ROW_ERROR;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_UPLOAD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_WORKER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LEASE_VERSION;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW_FIELD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.RECONCILE_ATTEMPTS;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** Metadata contract, configuration validation and integration registrations.
+ *******************************************************************************/
+class JobsQBitProducerTest extends BaseJobsTest
+{
+   /*******************************************************************************
+    ** The real kernel configuration passes startup validation without waivers.
+    *******************************************************************************/
+   @Test
+   void durableJobsMetadataValidates()
+   {
+      QInstanceValidator validator = new QInstanceValidator();
+      DurableJobsMetaData.of(instance).validate(instance, validator);
+      assertThat(validator.getErrors()).isEmpty();
+   }
+
+   /*******************************************************************************
+    ** Fail before registering anything when the kernel has not been enabled.
+    *******************************************************************************/
+   @Test
+   void missingTablesFailClearly()
+   {
+      QInstance empty = new QInstance();
+      assertThatThrownBy(() -> new JobsQBitProducer().produce(empty)).isInstanceOf(QException.class)
+         .hasMessage("Durable jobs are not enabled: call DurableJobsMetaDataProvider.defineAll(...) before producing qbit-jobs");
+      assertThat(empty.getQBits()).isEmpty();
+      instance.getTables().remove(JOBS_UPLOAD);
+      assertThatThrownBy(() -> new JobsQBitProducer().produce(instance)).hasMessageContaining("Durable jobs are not enabled");
+   }
+
+   /*******************************************************************************
+    ** Only the main table appears in navigation; every panel uses typed data.
+    *******************************************************************************/
+   @Test
+   void navigationWidgetsAndRecordSections()
+   {
+      assertThat(instance.getTables()).hasSize(12);
+      assertThat(instance.getApp("jobs").getChildren()).extracting(child -> child.getName()).containsExactly(JOBS_JOB);
+      assertThat(instance.getApp("jobs").getWidgets()).containsExactlyElementsOf(JobsQBitProducer.HOME_WIDGETS);
+      assertThat(instance.getWidgets()).hasSize(13);
+      assertThat(instance.getWidgets().values()).allSatisfy(widget -> assertThat(widget.getType()).isNotEqualTo(WidgetType.HTML.getType()));
+      assertThat(instance.getTable(JOBS_JOB).getSections()).extracting(QFieldSection::getWidgetName)
+         .containsSubsequence("jobsJobProgress", "jobsJobLog", "jobsJobChunks", "jobsJobRowErrors", "jobsJobEffects", "jobsJobEvents", "jobsJobAudit");
+      for(String name : JobsQBitProducer.HOME_WIDGETS)
+      {
+         assertThat(instance.getWidget(name).getDefaultValues()).containsEntry("refreshSeconds", 5);
+      }
+      assertThat(instance.getWidget("jobsJobProgress").getDefaultValues()).containsEntry("refreshSeconds", 3);
+      assertThat(instance.getProcess(JobsQBitProducer.PURGE_PROCESS).getIsHidden()).isTrue();
+      assertThat(instance.getProcess(JobsQBitProducer.PURGE_PROCESS).getSchedule()).isNull();
+      assertThat(instance.getWidget("jobsHomeWorkers").getPermissionRules().getCustomPermissionChecker()).isNotNull();
+      QBitMetaData metadata = instance.getQBits().get(JobsQBitProducer.QBIT_NAME);
+      assertThat(metadata.getGroupId()).isEqualTo("com.kingsrook.qbits");
+      assertThat(metadata.getArtifactId()).isEqualTo("qbit-jobs");
+      assertThat(metadata.getNamespace()).isEqualTo("jobs");
+      assertThat(metadata.getVersion()).matches("\\d+\\.\\d+\\.\\d+(-SNAPSHOT|-RC\\.\\d+)?");
+   }
+
+   /*******************************************************************************
+    ** Optional audit and scheduler behavior, including configured refresh values.
+    *******************************************************************************/
+   @Test
+   void schedulerAndAuditAreOptional() throws Exception
+   {
+      QInstance other = new QInstance();
+      new DurableJobsMetaDataProvider().defineAll(other, "memory", null);
+      other.withSupplementalMetaData(new DurableJobsMetaData());
+      JobsQBitConfig custom = new JobsQBitConfig().withAppName("backgroundJobs").withSchedulerName("scheduler")
+         .withPurgeCronExpression("0 0 4 * * ?").withHomeRefreshSeconds(9).withRecordRefreshSeconds(8);
+      new JobsQBitProducer().withQBitConfig(custom).produce(other);
+      assertThat(other.getWidget("jobsJobAudit")).isNull();
+      assertThat(other.getWidgets()).hasSize(12);
+      assertThat(other.getApp("backgroundJobs")).isNotNull();
+      QScheduleMetaData schedule = other.getProcess(JobsQBitProducer.PURGE_PROCESS).getSchedule();
+      assertThat(schedule.getSchedulerName()).isEqualTo("scheduler");
+      assertThat(schedule.getCronExpression()).isEqualTo("0 0 4 * * ?");
+      assertThat(schedule.getCronTimeZoneId()).isEqualTo("UTC");
+      assertThat(other.getWidget("jobsHomeSummary").getDefaultValues()).containsEntry("refreshSeconds", 9);
+      assertThat(other.getWidget("jobsJobProgress").getDefaultValues()).containsEntry("refreshSeconds", 8);
+   }
+
+   /*******************************************************************************
+    ** Match the kernel's failure, stall and heartbeat signal identifiers.
+    *******************************************************************************/
+   @Test
+   void alertTypes()
+   {
+      QAlertTypeMetaData failed = QAlertTypeMetaData.of(instance, ALERT_JOB_FAILED);
+      assertThat(failed.getDefaultSeverity()).isEqualTo(AlertSeverity.ERROR);
+      assertThat(failed.getSubjectTableName()).isEqualTo(JOBS_JOB);
+      assertThat(failed.getHowToFix()).isNotBlank();
+      assertThat(QAlertTypeMetaData.of(instance, ALERT_JOB_STALLED).getDefaultSeverity()).isEqualTo(AlertSeverity.WARNING);
+      QAlertTypeMetaData effect = QAlertTypeMetaData.of(instance, ALERT_EFFECT_UNRESOLVED);
+      assertThat(effect.getLabel()).isEqualTo("Job effect needs review");
+      assertThat(effect.getDefaultSeverity()).isEqualTo(AlertSeverity.WARNING);
+      assertThat(effect.getSubjectTableName()).isEqualTo(JOBS_JOB);
+      assertThat(effect.getHowToFix()).isNotBlank();
+      QAlertTypeMetaData reaper = QAlertTypeMetaData.of(instance, HEARTBEAT_REAPER);
+      assertThat(reaper.getLabel()).isEqualTo("Job reaper stopped");
+      assertThat(reaper.getDefaultMonitors()).singleElement().satisfies(monitor ->
+      {
+         assertThat(monitor.getKind()).isEqualTo(AlertMonitorKind.HEARTBEAT);
+         assertThat(monitor.getDefinition()).containsEntry("heartbeatKey", HEARTBEAT_REAPER);
+      });
+   }
+
+   /*******************************************************************************
+    ** Configuration rejects unsafe retention and malformed schedules.
+    *******************************************************************************/
+   @Test
+   void validateConfiguration() throws Exception
+   {
+      assertThatThrownBy(() -> new JobsQBitProducer().withQBitConfig(null).produce(instance)).hasMessageContaining("JobsQBitConfig is required");
+      JobsQBitConfig bad = new JobsQBitConfig().withAppName(" ").withRetentionDays(0).withInputRetentionDays(-1)
+         .withPurgeCronExpression("not cron").withHomeRefreshSeconds(0).withRecordRefreshSeconds(0).withSchedulerName("");
+      List<String> errors = new ArrayList<>();
+      bad.validate(instance, errors);
+      assertThat(errors).hasSize(6);
+      assertThatThrownBy(() -> new JobsQBitProducer().withQBitConfig(bad).produce(instance)).isInstanceOf(QException.class);
+      config.setAppName("jobs2");
+      config.setRetentionDays(20);
+      config.setInputRetentionDays(3);
+      config.setHomeRefreshSeconds(7);
+      config.setRecordRefreshSeconds(4);
+      config.setPurgeCronExpression("0 30 3 * * ?");
+      config.setSchedulerName("daily");
+      assertThat(config.getAppName()).isEqualTo("jobs2");
+      assertThat(config.getRetentionDays()).isEqualTo(20);
+      assertThat(config.getInputRetentionDays()).isEqualTo(3);
+      assertThat(config.getHomeRefreshSeconds()).isEqualTo(7);
+      assertThat(config.getRecordRefreshSeconds()).isEqualTo(4);
+      assertThat(config.getSchedulerName()).isEqualTo("daily");
+      assertThat(config.getPurgeCronExpression()).isEqualTo("0 30 3 * * ?");
+      config.validate(instance);
+      assertThat(new JobsQBitProducer().withQBitConfig(config).getQBitConfig()).isSameAs(config);
+   }
+
+   /*******************************************************************************
+    ** Integration hooks expose only intended standard tables.
+    *******************************************************************************/
+   @Test
+   void integrationLists()
+   {
+      assertThat(JobsIntegrations.esbTableNames()).containsExactly(JOBS_JOB);
+      assertThat(JobsIntegrations.quickSearchTables()).containsExactly(JOBS_JOB);
+      assertThat(JobsIntegrations.dataAnalysisTables()).containsExactly(JOBS_JOB, JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EFFECT, JOBS_EVENT, JOBS_WORKER);
+   }
+
+   /*******************************************************************************
+    ** The jobs alert types pass the alerts validator (dotted, no underscores).
+    *******************************************************************************/
+   @Test
+   void alertTypesValidate()
+   {
+      AlertsInstanceMetaData alerts = new AlertsInstanceMetaData();
+      alerts.withAlertType(QAlertTypeMetaData.of(instance, ALERT_JOB_FAILED));
+      alerts.withAlertType(QAlertTypeMetaData.of(instance, ALERT_JOB_STALLED));
+      alerts.withAlertType(QAlertTypeMetaData.of(instance, ALERT_EFFECT_UNRESOLVED));
+      QInstanceValidator validator = new QInstanceValidator();
+      alerts.validate(instance, validator);
+      assertThat(validator.getErrors()).isEmpty();
+   }
+
+   /*******************************************************************************
+    ** Hosts can omit the effects panel by removing its table or join.
+    *******************************************************************************/
+   @Test
+   void effectsTableAndJoinAreOptional() throws Exception
+   {
+      QInstance withoutTable = new QInstance();
+      new DurableJobsMetaDataProvider().defineAll(withoutTable, "memory", null);
+      withoutTable.withSupplementalMetaData(new DurableJobsMetaData());
+      withoutTable.getTables().remove(JOBS_EFFECT);
+      new JobsQBitProducer().produce(withoutTable);
+      assertThat(withoutTable.getWidget("jobsJobEffects")).isNull();
+      QInstance withoutJoin = new QInstance();
+      new DurableJobsMetaDataProvider().defineAll(withoutJoin, "memory", null);
+      withoutJoin.withSupplementalMetaData(new DurableJobsMetaData());
+      withoutJoin.getJoins().remove(JobsCoreContract.childJoin(JOBS_EFFECT));
+      new JobsQBitProducer().produce(withoutJoin);
+      assertThat(withoutJoin.getWidget("jobsJobEffects")).isNull();
+   }
+
+   /*******************************************************************************
+    ** Outbox metadata carries the core field defaults, references and labels.
+    *******************************************************************************/
+   @Test
+   void effectsCoreContract() throws Exception
+   {
+      assertThat(instance.getTable(JOBS_EFFECT).getFields()).hasSize(26);
+      assertThat(instance.getTable(JOBS_EFFECT).getField(JOB_ID).getPossibleValueSourceName()).isEqualTo(JOBS_JOB);
+      assertThat(instance.getTable(JOBS_EFFECT).getField(CHUNK_ID).getPossibleValueSourceName()).isEqualTo(JOBS_CHUNK);
+      assertThat(instance.getTable(JOBS_EFFECT).getField(NEEDS_REVIEW_FIELD).getDefaultValue()).isEqualTo(false);
+      for(String field : List.of(ATTEMPT, RECONCILE_ATTEMPTS, LEASE_VERSION))
+      {
+         assertThat(instance.getTable(JOBS_EFFECT).getField(field).getDefaultValue()).isEqualTo(0);
+      }
+      assertThat(instance.getPossibleValueSource(JOBS_EFFECT_STATUS).getEnumValues()).extracting(value -> value.getLabel())
+         .containsExactly("Pending", "Sending", "Sent", "Failed", "Unknown");
+      assertThat(instance.getPossibleValueSource(JOBS_EFFECT_TYPE).getEnumValues()).extracting(value -> value.getLabel())
+         .containsExactly("Local Transaction", "Idempotent External", "Non-Idempotent External");
+   }
+}

src/test/java/com/kingsrook/qbits/jobs/JobsRetentionTest.java

@@ -0,0 +1,351 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.jobs.processes.JobsPurgeFinishedStep;
+import com.kingsrook.qbits.jobs.processes.JobsStorage;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsMetaData;
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.AUDIT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED_WITH_ERRORS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.DURABLE_JOBS_METADATA;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.INPUT_STORAGE_REF;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ROW_ERROR;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_UPLOAD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW_FIELD;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PENDING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.RUNNING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SENDING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SEQ;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.TERMINAL_STATUSES;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.UNKNOWN;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** Retention boundaries, pagination, input cleanup, failure recovery and summaries.
+ *******************************************************************************/
+class JobsRetentionTest extends BaseJobsTest
+{
+   /*******************************************************************************
+    ** Only old terminal jobs and their children are purged; audit remains intact.
+    *******************************************************************************/
+   @Test
+   void purgesOldTerminalJobsAndIsIdempotent() throws Exception
+   {
+      for(String status : TERMINAL_STATUSES)
+      {
+         children(job(USER_A, status, now.minus(31, ChronoUnit.DAYS)));
+      }
+      children(job(USER_B, RUNNING, now.minus(40, ChronoUnit.DAYS)));
+      children(job(USER_A, COMPLETED, now.minus(2, ChronoUnit.DAYS)));
+      children(job(USER_A, NEEDS_REVIEW, now.minus(40, ChronoUnit.DAYS)));
+      List<Integer> auditIds = records(AUDIT).stream().map(record -> record.getValueInteger(ID)).toList();
+      RecordingStorage storage = new RecordingStorage();
+      QSession caller = login("admin", true);
+      List<ProcessSummaryLine> summary = run(new JobsPurgeFinishedStep(storage));
+      assertThat(QContext.getQSession()).isSameAs(caller);
+      assertThat(summary).extracting(ProcessSummaryLine::getCount).containsExactly(4, 4, 4);
+      DurableJobsSecurity.ensureSessionKeys(caller);
+      assertThat(records(JOBS_JOB)).hasSize(3).extracting(record -> record.getValueString(STATUS)).containsExactly(RUNNING, COMPLETED, NEEDS_REVIEW);
+      for(String table : List.of(JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EVENT, JOBS_UPLOAD))
+      {
+         assertThat(records(table)).hasSize(3);
+      }
+      assertThat(records(AUDIT)).extracting(record -> record.getValueInteger(ID)).containsAll(auditIds);
+      assertThat(storage.deleted).hasSize(8);
+      assertThat(run(new JobsPurgeFinishedStep(storage))).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 0);
+   }
+
+   /*******************************************************************************
+    ** Clean successful inputs expire independently of their retained job history.
+    *******************************************************************************/
+   @Test
+   void cleansOnlyOldSuccessfulInputs() throws Exception
+   {
+      QRecord clean = job(USER_A, COMPLETED, now.minus(8, ChronoUnit.DAYS));
+      clearFailures(clean);
+      children(clean);
+      children(job(USER_A, COMPLETED, now.minus(8, ChronoUnit.DAYS)));
+      QRecord recent = job(USER_A, COMPLETED, now.minus(6, ChronoUnit.DAYS));
+      clearFailures(recent);
+      children(recent);
+      children(job(USER_A, COMPLETED_WITH_ERRORS, now.minus(8, ChronoUnit.DAYS)));
+      RecordingStorage storage = new RecordingStorage();
+      assertThat(run(new JobsPurgeFinishedStep(storage))).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 1);
+      assertThat(records(JOBS_JOB)).hasSize(4);
+      assertThat(records(JOBS_CHUNK).getFirst().getValue(INPUT_STORAGE_REF)).isNull();
+      assertThat(records(JOBS_CHUNK).subList(1, 4)).allSatisfy(chunk -> assertThat(chunk.getValue(INPUT_STORAGE_REF)).isNotNull());
+      assertThat(run(new JobsPurgeFinishedStep(storage))).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 0);
+      assertThat(storage.deleted).containsExactly("objects:chunk/" + clean.getValue(ID));
+   }
+
+   /*******************************************************************************
+    ** Both parent and child batches span more than 200 records without skips.
+    *******************************************************************************/
+   @Test
+   void pagesJobsAndChildren() throws Exception
+   {
+      for(int i = 0; i < 205; i++)
+      {
+         job(USER_A, FAILED, now.minus(40, ChronoUnit.DAYS));
+      }
+      QRecord old = records(JOBS_JOB).getFirst();
+      for(int i = 0; i < 205; i++)
+      {
+         insert(JOBS_CHUNK, new QRecord().withValue(JOB_ID, old.getValue(ID)).withValue(SEQ, i).withValue(INPUT_STORAGE_REF, "old/" + i));
+      }
+      QRecord clean = job(USER_B, COMPLETED, now.minus(8, ChronoUnit.DAYS));
+      clearFailures(clean);
+      for(int i = 0; i < 205; i++)
+      {
+         insert(JOBS_CHUNK, new QRecord().withValue(JOB_ID, clean.getValue(ID)).withValue(SEQ, i).withValue(INPUT_STORAGE_REF, "clean/" + i));
+      }
+      RecordingStorage storage = new RecordingStorage();
+      assertThat(run(new JobsPurgeFinishedStep(storage))).extracting(ProcessSummaryLine::getCount).containsExactly(205, 205, 205);
+      assertThat(records(JOBS_JOB)).hasSize(1);
+      assertThat(records(JOBS_CHUNK)).hasSize(205).allSatisfy(chunk -> assertThat(chunk.getValue(INPUT_STORAGE_REF)).isNull());
+      assertThat(storage.deleted).hasSize(410);
+   }
+
+   /*******************************************************************************
+    ** Core storage deletes input objects before retention clears their references.
+    *******************************************************************************/
+   @Test
+   void releasedCoreDeletesObjects() throws Exception
+   {
+      QRecord clean = job(USER_A, COMPLETED, now.minus(8, ChronoUnit.DAYS));
+      clearFailures(clean);
+      children(clean);
+      instance.addTable(new QTableMetaData().withName("objects").withBackendName("memory").withPrimaryKeyField(ID)
+         .withField(new QFieldMetaData(ID, QFieldType.STRING)).withField(new QFieldMetaData("data", QFieldType.BLOB)));
+      instance.withSupplementalMetaData(new DurableJobsMetaData().withStorageTableName("objects"));
+      QSession caller = QContext.getQSession();
+      assertThatThrownBy(() -> run(new JobsPurgeFinishedStep())).isInstanceOf(QException.class);
+      assertThat(QContext.getQSession()).isSameAs(caller);
+      assertThat(records(JOBS_CHUNK).getFirst().getValue(INPUT_STORAGE_REF)).isNotNull();
+      insert("objects", new QRecord().withValue(ID, "chunk/" + clean.getValue(ID)).withValue("data", new byte[] {1}));
+      assertThat(run(new JobsPurgeFinishedStep())).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 1);
+      assertThat(records(JOBS_CHUNK).getFirst().getValue(INPUT_STORAGE_REF)).isNull();
+      JobsStorage storage = new JobsStorage();
+      assertThat(storage.chunkStorageTable()).isEqualTo("objects");
+      assertThat(records("objects")).isEmpty();
+      assertThatThrownBy(() -> storage.delete("objects", "missing-object")).isInstanceOf(QException.class);
+      assertThat(storage.delete("", "object")).isFalse();
+      assertThat(storage.delete("objects", " ")).isFalse();
+      assertThat(storage.delete(null, "object")).isFalse();
+      assertThat(storage.delete("objects", null)).isFalse();
+   }
+
+   /*******************************************************************************
+    ** A storage failure preserves the parent and always restores the caller.
+    *******************************************************************************/
+   @Test
+   void storageFailureRestoresSession() throws Exception
+   {
+      children(job(USER_A, FAILED, now.minus(40, ChronoUnit.DAYS)));
+      QSession caller = login("admin", true);
+      RecordingStorage storage = new RecordingStorage();
+      storage.fail = true;
+      assertThatThrownBy(() -> run(new JobsPurgeFinishedStep(storage))).hasMessageContaining("storage unavailable");
+      assertThat(QContext.getQSession()).isSameAs(caller);
+      DurableJobsSecurity.ensureSessionKeys(caller);
+      assertThat(records(JOBS_JOB)).hasSize(1);
+      assertThat(records(JOBS_CHUNK)).hasSize(1);
+      storage.fail = false;
+      assertThat(run(new JobsPurgeFinishedStep(storage))).extracting(ProcessSummaryLine::getCount).containsExactly(1, 1, 1);
+   }
+
+   /*******************************************************************************
+    ** Hidden does not mean authorized: direct invocation checks permission too.
+    *******************************************************************************/
+   @Test
+   void normalViewerCannotPurge()
+   {
+      QSession viewer = login(USER_A, false);
+      assertThatThrownBy(() -> run(new JobsPurgeFinishedStep())).hasMessageContaining("requires jobs_admin");
+      assertThat(QContext.getQSession()).isSameAs(viewer);
+   }
+
+   /*******************************************************************************
+    ** No supplemental metadata means object cleanup is skipped.
+    *******************************************************************************/
+   @Test
+   void absentStorageMetadata() throws Exception
+   {
+      instance.getSupplementalMetaData().remove(DURABLE_JOBS_METADATA);
+      assertThat(new JobsStorage().chunkStorageTable()).isNull();
+      assertThat(run(new JobsPurgeFinishedStep())).extracting(ProcessSummaryLine::getCount).containsExactly(0, 0, 0);
+   }
+
+   /*******************************************************************************
+    ** Unsettled effects preserve every child and input; settled effects are paged away.
+    *******************************************************************************/
+   @Test
+   void effectsBlockPurgeUntilSettled() throws Exception
+   {
+      List<QRecord> kept = new ArrayList<>();
+      for(String status : List.of(PENDING, SENDING, UNKNOWN))
+      {
+         QRecord job = job(USER_A, COMPLETED, now.minus(40, ChronoUnit.DAYS));
+         clearFailures(job);
+         children(job);
+         effect(job, status, false);
+         kept.add(job);
+      }
+      QRecord review = job(USER_A, FAILED, now.minus(40, ChronoUnit.DAYS));
+      children(review);
+      effect(review, FAILED, true);
+      kept.add(review);
+      QRecord settled = job(USER_B, COMPLETED, now.minus(40, ChronoUnit.DAYS));
+      children(settled);
+      for(int i = 0; i < 205; i++)
+      {
+         effect(settled, i % 2 == 0 ? SENT : FAILED, false);
+      }
+      RecordingStorage storage = new RecordingStorage();
+      QSession caller = login("admin", true);
+      List<ProcessSummaryLine> summary = run(new JobsPurgeFinishedStep(storage));
+      assertThat(QContext.getQSession()).isSameAs(caller);
+      assertThat(summary).extracting(ProcessSummaryLine::getCount).containsExactly(1, 1, 1, 4);
+      assertThat(summary.getLast().getMessage()).isEqualTo("jobs kept: effects still unsettled");
+      DurableJobsSecurity.ensureSessionKeys(caller);
+      assertThat(records(JOBS_JOB)).extracting(record -> record.getValue(ID)).containsExactlyElementsOf(kept.stream().map(record -> record.getValue(ID)).toList());
+      assertThat(records(JOBS_EFFECT)).hasSize(4);
+      for(String table : List.of(JOBS_CHUNK, JOBS_ROW_ERROR, JOBS_EVENT, JOBS_UPLOAD))
+      {
+         assertThat(records(table)).hasSize(4);
+      }
+      assertThat(records(JOBS_CHUNK)).allSatisfy(chunk -> assertThat(chunk.getValue(INPUT_STORAGE_REF)).isNotNull());
+      assertThat(storage.deleted).containsExactly("objects:chunk/" + settled.getValue(ID), "objects:input/" + settled.getValue(ID));
+      for(QRecord effect : records(JOBS_EFFECT))
+      {
+         new UpdateAction().execute(new UpdateInput().withTableName(JOBS_EFFECT).withRecord(new QRecord()
+            .withValue(ID, effect.getValue(ID)).withValue(STATUS, SENT).withValue(NEEDS_REVIEW_FIELD, false)));
+      }
+      assertThat(run(new JobsPurgeFinishedStep(storage))).extracting(ProcessSummaryLine::getCount).containsExactly(4, 4, 4);
+      assertThat(records(JOBS_JOB)).isEmpty();
+      assertThat(records(JOBS_EFFECT)).isEmpty();
+   }
+
+   /*******************************************************************************
+    ** Retention tolerates a host omitting the effects table.
+    *******************************************************************************/
+   @Test
+   void purgeWithoutEffectsTable() throws Exception
+   {
+      instance.getTables().remove(JOBS_EFFECT);
+      children(job(USER_A, FAILED, now.minus(40, ChronoUnit.DAYS)));
+      assertThat(run(new JobsPurgeFinishedStep(new RecordingStorage()))).extracting(ProcessSummaryLine::getCount).containsExactly(1, 1, 1);
+      assertThat(records(JOBS_JOB)).isEmpty();
+   }
+
+   /*******************************************************************************
+    ** Mark the test job successful with no failed rows.
+    *******************************************************************************/
+   private void clearFailures(QRecord job) throws QException
+   {
+      new UpdateAction().execute(new UpdateInput().withTableName(JOBS_JOB)
+         .withRecord(new QRecord().withValue(ID, job.getValue(ID)).withValue(ROWS_FAILED, 0)));
+   }
+
+   /*******************************************************************************
+    ** The scheduled step carries standard ProcessSummaryLines in its output.
+    *******************************************************************************/
+   @SuppressWarnings("unchecked")
+   private List<ProcessSummaryLine> run(JobsPurgeFinishedStep step) throws QException
+   {
+      RunBackendStepOutput output = new RunBackendStepOutput();
+      step.run(new RunBackendStepInput(), output);
+      return ((List<ProcessSummaryLine>) output.getValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY));
+   }
+
+   /*******************************************************************************
+    ** Test storage records deletions and simulates failures without using disk.
+    *******************************************************************************/
+   private static class RecordingStorage extends JobsStorage
+   {
+      private final List<String> deleted = new ArrayList<>();
+      private boolean fail;
+
+      /*******************************************************************************
+       ** Configured storage table.
+       *******************************************************************************/
+      @Override
+      public String chunkStorageTable()
+      {
+         return ("objects");
+      }
+
+      /*******************************************************************************
+       ** Record calls and verify they execute as system.
+       *******************************************************************************/
+      @Override
+      public boolean delete(String table, String reference) throws QException
+      {
+         assertThat(QContext.getQSession()).isInstanceOf(QSystemUserSession.class);
+         if(fail)
+         {
+            throw (new QException("storage unavailable"));
+         }
+         if(reference == null)
+         {
+            return (false);
+         }
+         deleted.add(table + ":" + reference);
+         return (true);
+      }
+   }
+
+}

src/test/java/com/kingsrook/qbits/jobs/JobsSecurityTest.java

@@ -0,0 +1,76 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import com.kingsrook.qqq.backend.core.jobs.DurableJobsSecurity;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.model.session.QUser;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_ADMIN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_USER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_USER_ALL_ACCESS;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Core-compatible security key derivation and null/system session behavior.
+ *******************************************************************************/
+class JobsSecurityTest
+{
+   /*******************************************************************************
+    ** Reapplying keys derives ownership and admin bypass from the actual session.
+    *******************************************************************************/
+   @Test
+   void derivesKeysFromUserAndPermission()
+   {
+      QSession session = new QSession().withUser(new QUser().withIdReference("user"))
+         .withSecurityKeyValue(JOBS_USER, "wrong").withSecurityKeyValue(JOBS_USER_ALL_ACCESS, true);
+      DurableJobsSecurity.applyToSession(session);
+      assertThat(session.getSecurityKeyValues(JOBS_USER)).containsExactly("user");
+      assertThat(session.getSecurityKeyValues(JOBS_USER_ALL_ACCESS)).isEmpty();
+      session.withPermission(JOBS_ADMIN);
+      DurableJobsSecurity.applyToSession(session);
+      assertThat(session.getSecurityKeyValues(JOBS_USER_ALL_ACCESS)).containsExactly(true);
+      DurableJobsSecurity.ensureSessionKeys(session);
+      assertThat(session.getSecurityKeyValues(JOBS_USER)).containsExactly("user");
+   }
+
+   /*******************************************************************************
+    ** Null, anonymous and system users are handled without granting extra access.
+    *******************************************************************************/
+   @Test
+   void nullAnonymousAndSystem()
+   {
+      DurableJobsSecurity.applyToSession(null);
+      DurableJobsSecurity.ensureSessionKeys(null);
+      DurableJobsSecurity.applyToSession(new QSystemUserSession());
+      DurableJobsSecurity.ensureSessionKeys(new QSystemUserSession());
+      assertThat(DurableJobsSecurity.userId(null)).isNull();
+      QSession anonymous = new QSession();
+      DurableJobsSecurity.ensureSessionKeys(anonymous);
+      assertThat(DurableJobsSecurity.userId(anonymous)).isNull();
+      assertThat(anonymous.getSecurityKeyValues(JOBS_USER)).isEmpty();
+      DurableJobsSecurity.ensureSessionKeys(new QSession().withUser(new QUser().withIdReference("")));
+   }
+}

src/test/java/com/kingsrook/qbits/jobs/JobsWidgetRendererTest.java

@@ -0,0 +1,565 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.jobs;
+
+
+import java.io.Serializable;
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.jobs.widgets.JobsFormat;
+import com.kingsrook.qbits.jobs.widgets.JobsWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChildRecordListData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.CompositeWidgetData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.progressbar.ProgressBarBlockData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.blocks.text.TextBlockData;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import org.junit.jupiter.api.Test;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.CHUNKS_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.COMPLETED_WITH_ERRORS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.DONE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.FINISHED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_CHUNK;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EFFECT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_EVENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_JOB;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOBS_USER;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.JOB_ID;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.LAST_PROGRESS_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.NEEDS_REVIEW;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PENDING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PRIORITY;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.PROCESS_NAME;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.QUEUED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_FAILED;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_IN;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.ROWS_TOTAL;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.RUNNING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SENDING;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SENT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.SEQ;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STARTED_AT;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.STATUS_REASON;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.TYPE;
+import static com.kingsrook.qbits.jobs.JobsCoreContract.UNKNOWN;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** Real widget rendering, temporal buckets, empty states and owner isolation.
+ *******************************************************************************/
+class JobsWidgetRendererTest extends BaseJobsTest
+{
+   /*******************************************************************************
+    ** Summary: two groups, each statistic labelled once, counts reflect status
+    ** groups and the viewer's records, and the Now statistics link to the jobs.
+    *******************************************************************************/
+   @Test
+   void summaryCountsAndOwnerIsolation() throws Exception
+   {
+      seed();
+      QSession            viewer = login(USER_A, false);
+      MultiStatisticsData data   = (MultiStatisticsData) render("jobsHomeSummary", null);
+      assertThat(data.getTitle()).isNull();
+      assertThat(data.getStatisticsGroupData()).extracting(MultiStatisticsData.StatisticsGroupData::getHeader).containsExactly("Now", "Last 24 hours");
+      assertThat(data.getStatisticsGroupData()).allSatisfy(group -> assertThat(group.getStatisticList()).hasSize(3));
+      assertThat(count(data, "Running")).isEqualTo(4);
+      assertThat(count(data, "Queued")).isEqualTo(2);
+      assertThat(count(data, "Needs attention")).isEqualTo(3);
+      assertThat(count(data, "Completed")).isEqualTo(2);
+      assertThat(count(data, "Rows processed")).isEqualTo(4050);
+      assertThat(count(data, "Rows/min (last hour)")).isEqualTo(90);
+      assertThat(URLDecoder.decode(stat(data, "Needs attention").getUrl(), StandardCharsets.UTF_8)).contains(JOBS_JOB, "FAILED", "NEEDS_REVIEW");
+      assertThat(stat(data, "Completed").getUrl()).isNull();
+      assertThat(QContext.getQSession()).isSameAs(viewer);
+      assertThat(viewer.getSecurityKeyValues(JOBS_USER)).containsExactly(USER_A);
+      login(USER_B, false);
+      assertThat(count((MultiStatisticsData) render("jobsHomeSummary", null), "Running")).isEqualTo(1);
+      login("admin", true);
+      assertThat(count((MultiStatisticsData) render("jobsHomeSummary", null), "Running")).isEqualTo(5);
+   }
+
+
+
+   /*******************************************************************************
+    ** Old completions do not count in current summary windows.
+    *******************************************************************************/
+   @Test
+   void summaryExcludesOldData() throws Exception
+   {
+      QRecord old = job(USER_A, COMPLETED, now.minus(8, ChronoUnit.DAYS));
+      update(JOBS_JOB, old.getValue(ID), LAST_PROGRESS_AT, now.minus(2, ChronoUnit.HOURS));
+      job(USER_A, FAILED, now.minus(8, ChronoUnit.DAYS));
+      login(USER_A, false);
+      MultiStatisticsData data = (MultiStatisticsData) render("jobsHomeSummary", null);
+      assertThat(count(data, "Completed")).isZero();
+      assertThat(count(data, "Needs attention")).isZero();
+      assertThat(count(data, "Rows processed")).isZero();
+      assertThat(count(data, "Rows/min (last hour)")).isEqualTo(8);
+   }
+
+
+
+   /*******************************************************************************
+    ** Three block columns; priority ordering; labels, links, bars and honest
+    ** rate/ETA values; plain values kept on each row.
+    *******************************************************************************/
+   @Test
+   void activeTable() throws Exception
+   {
+      instance.addProcess(new QProcessMetaData().withName("importOrders").withLabel("Import orders"));
+      List<QRecord> jobs   = seed();
+      QRecord       queued = jobs.get(2);
+      update(JOBS_JOB, queued.getValue(ID), PRIORITY, 99);
+      update(JOBS_JOB, queued.getValue(ID), STATUS_REASON, "Waiting for capacity");
+      login(USER_A, false);
+      TableData table = (TableData) render("jobsHomeActive", null);
+      assertThat(table.getLabel()).isNull();
+      assertThat(table.getColumns()).extracting(TableData.Column::getHeader).containsExactly("Job", "Progress", "Status");
+      assertThat(table.getColumns()).allSatisfy(column -> assertThat(column.getWidth()).isNull());
+      assertThat(table.getRows()).hasSize(6);
+
+      Map<String, Object> first = table.getRows().getFirst();
+      assertThat(first).containsEntry("Status", "Waiting for capacity").containsEntry("Progress", "45%").containsEntry("Rows", "450 / 1,000")
+         .containsEntry("Process", "Import orders").containsEntry("Started", "Started 10m ago");
+      assertThat(first.get("Path").toString()).endsWith("/" + queued.getValue(ID));
+      CompositeWidgetData jobCell = (CompositeWidgetData) first.get(JobsWidgetRenderer.JOB_CELL);
+      assertThat(texts(jobCell)).containsExactly(USER_A + " " + QUEUED, "Import orders");
+      assertThat(jobCell.getBlocks().getFirst().getLink().getHref()).isEqualTo(first.get("Path"));
+      CompositeWidgetData progressCell = (CompositeWidgetData) first.get(JobsWidgetRenderer.PROGRESS_CELL);
+      ProgressBarBlockData bar = (ProgressBarBlockData) progressCell.getBlocks().getFirst();
+      assertThat(bar.getValues().getPercent()).isEqualByComparingTo("45");
+      assertThat(bar.getValues().getValue()).isEqualTo("45%");
+      assertThat(texts((CompositeWidgetData) first.get(JobsWidgetRenderer.STATUS_CELL))).containsExactly("Waiting for capacity", "Started 10m ago");
+
+      Map<String, Object> running = table.getRows().stream().filter(row -> "Running".equals(row.get("Status"))).findFirst().orElseThrow();
+      assertThat(running).containsEntry("Rate", "45 rows/min").containsEntry("ETA", "in 12m");
+      assertThat(texts((CompositeWidgetData) running.get(JobsWidgetRenderer.STATUS_CELL))).containsExactly("Running", "45 rows/min, ETA in 12m");
+      assertThat(table.getRows()).extracting(row -> row.get("Status")).contains("Awaiting Confirmation", "Planning", "Pausing", "Cancelling");
+
+      login(USER_B, false);
+      assertThat(((TableData) render("jobsHomeActive", null)).getRows()).hasSize(1);
+      login("admin", true);
+      assertThat(((TableData) render("jobsHomeActive", null)).getRows()).hasSize(7);
+   }
+
+
+
+   /*******************************************************************************
+    ** Missing totals fall back to chunks; missing times do not invent a rate;
+    ** with neither total the cell says so instead of drawing a bar.
+    *******************************************************************************/
+   @Test
+   void unknownProgressAndLimit() throws Exception
+   {
+      QRecord job = job(USER_A, RUNNING, null);
+      update(JOBS_JOB, job.getValue(ID), ROWS_TOTAL, null);
+      update(JOBS_JOB, job.getValue(ID), STARTED_AT, null);
+      login(USER_A, false);
+      TableData table = (TableData) render("jobsHomeActive", null);
+      assertThat(table.getRows().getFirst()).containsEntry("Progress", "40%").containsEntry("Rows", "450 / Unknown")
+         .containsEntry("Rate", "Not started").containsEntry("ETA", "Unknown").containsEntry("Process", "importOrders");
+      QContext.setQSession(new QSystemUserSession());
+      update(JOBS_JOB, job.getValue(ID), CHUNKS_TOTAL, null);
+      for(int i = 0; i < 30; i++)
+      {
+         job(USER_A, QUEUED, null);
+      }
+      login(USER_A, false);
+      table = (TableData) render("jobsHomeActive", null);
+      assertThat(table.getRows()).hasSize(25);
+      Map<String, Object> first = table.getRows().getFirst();
+      assertThat(first).containsEntry("Progress", "Unknown");
+      assertThat(texts((CompositeWidgetData) first.get(JobsWidgetRenderer.PROGRESS_CELL))).containsExactly("Unknown total", "450 / Unknown rows");
+   }
+
+
+
+   /*******************************************************************************
+    ** Attention: failed and needs-review first, then completed with errors;
+    ** old errors excluded; the error clipped to 120 characters on its own line.
+    *******************************************************************************/
+   @Test
+   void attentionTable() throws Exception
+   {
+      seed();
+      job(USER_A, FAILED, now.minus(8, ChronoUnit.DAYS));
+      job(USER_A, COMPLETED_WITH_ERRORS, now.minus(1, ChronoUnit.MINUTES));
+      job(USER_B, FAILED, now);
+      login(USER_A, false);
+      TableData data = (TableData) render("jobsHomeAttention", null);
+      assertThat(data.getLabel()).isNull();
+      assertThat(data.getColumns()).extracting(TableData.Column::getHeader).containsExactly("Job", "Status");
+      assertThat(data.getRows()).hasSize(4);
+      assertThat(data.getRows()).extracting(row -> row.get("Status"))
+         .containsExactly("Failed", "Needs Review", "Completed With Errors", "Completed With Errors");
+      assertThat(data.getRows().get(2)).containsEntry("Finished", "Finished 1m ago");
+      assertThat(data.getRows()).allSatisfy(row ->
+      {
+         assertThat(row.get("Rows failed")).isEqualTo(10L);
+         assertThat(row.get("Error").toString()).hasSize(120).endsWith(JobsFormat.ELLIPSIS);
+         assertThat(texts((CompositeWidgetData) row.get(JobsWidgetRenderer.JOB_CELL))).hasSize(3).last().isEqualTo(row.get("Error"));
+         assertThat(texts((CompositeWidgetData) row.get(JobsWidgetRenderer.STATUS_CELL))).contains("10 rows failed");
+      });
+      QContext.setQSession(new QSystemUserSession());
+      update(JOBS_JOB, data.getRows().getFirst().get("Path").toString().replaceAll(".*/", ""), ROWS_FAILED, 0);
+      login(USER_A, false);
+      Map<String, Object> failedFirst = ((TableData) render("jobsHomeAttention", null)).getRows().getFirst();
+      assertThat(texts((CompositeWidgetData) failedFirst.get(JobsWidgetRenderer.STATUS_CELL))).containsExactly("Failed", "Finished 30m ago");
+      login(USER_B, false);
+      assertThat(((TableData) render("jobsHomeAttention", null)).getRows()).hasSize(1);
+      login("admin", true);
+      assertThat(((TableData) render("jobsHomeAttention", null)).getRows()).hasSize(5);
+   }
+
+
+
+   /*******************************************************************************
+    ** Throughput reads secured chunks, names series by process label and groups
+    ** overflow processes as Other.
+    *******************************************************************************/
+   @Test
+   void throughputGroupingAndIsolation() throws Exception
+   {
+      seed();
+      for(int i = 0; i < 6; i++)
+      {
+         instance.addProcess(new QProcessMetaData().withName("process" + i).withLabel("Process " + i));
+         QRecord job = job(USER_A, COMPLETED, now);
+         update(JOBS_JOB, job.getValue(ID), PROCESS_NAME, "process" + i);
+         insert(JOBS_CHUNK, new QRecord().withValue(JOB_ID, job.getValue(ID)).withValue(SEQ, 1)
+            .withValue(STATUS, DONE).withValue(ROWS_IN, (i + 1) * 100).withValue(FINISHED_AT, now.minusSeconds(10)));
+      }
+      login(USER_A, false);
+      ChartData chart = (ChartData) render("jobsHomeThroughput", null);
+      assertThat(chart.getType()).isEqualTo(WidgetType.STACKED_BAR_CHART.getType());
+      assertThat(chart.getTitle()).isNull();
+      assertThat(chart.getChartData().getLabels()).hasSize(24).allSatisfy(label -> assertThat(label).matches("\\d\\d:00"));
+      assertThat(chart.getChartData().getDatasets()).hasSize(6).extracting(ChartData.Data.Dataset::getLabel).contains("Process 5", "Other");
+      assertThat(chartTotal(chart)).isEqualTo(3300);
+      login(USER_B, false);
+      assertThat(chartTotal((ChartData) render("jobsHomeThroughput", null))).isEqualTo(100);
+      login("admin", true);
+      assertThat(chartTotal((ChartData) render("jobsHomeThroughput", null))).isEqualTo(3400);
+   }
+
+
+
+   /*******************************************************************************
+    ** Thirty daily buckets contain only recent terminal jobs, labelled by status.
+    *******************************************************************************/
+   @Test
+   void history() throws Exception
+   {
+      seed();
+      job(USER_A, COMPLETED, now.minus(40, ChronoUnit.DAYS));
+      job(USER_B, FAILED, now);
+      login(USER_A, false);
+      ChartData chart = (ChartData) render("jobsHomeHistory", null);
+      assertThat(chart.getTitle()).isNull();
+      assertThat(chart.getChartData().getLabels()).hasSize(30).allSatisfy(label -> assertThat(label).matches("[A-Z][a-z]{2} \\d{1,2}"));
+      assertThat(chart.getChartData().getDatasets()).extracting(ChartData.Data.Dataset::getLabel)
+         .containsExactly("Completed", "Completed With Errors", "Failed", "Cancelled");
+      assertThat(chartTotal(chart)).isEqualTo(4);
+      login(USER_B, false);
+      assertThat(chartTotal((ChartData) render("jobsHomeHistory", null))).isEqualTo(1);
+      login("admin", true);
+      assertThat(chartTotal((ChartData) render("jobsHomeHistory", null))).isEqualTo(5);
+   }
+
+
+
+   /*******************************************************************************
+    ** Workers are permission-gated even when a caller bypasses widget dispatch.
+    *******************************************************************************/
+   @Test
+   void workers() throws Exception
+   {
+      seed();
+      login(USER_A, false);
+      RenderWidgetInput input  = new RenderWidgetInput().withWidgetMetaData(instance.getWidget("jobsHomeWorkers"));
+      TableData         denied = (TableData) new JobsWidgetRenderer().render(input).getWidgetData();
+      assertThat(denied.getRows()).isEmpty();
+      assertThat(denied.getNoRowsFoundHTML()).contains("jobs_admin");
+      login("admin", true);
+      TableData table = (TableData) render("jobsHomeWorkers", null);
+      assertThat(table.getColumns()).extracting(TableData.Column::getHeader).containsExactly("Pod", "Capacity", "Last heartbeat");
+      assertThat(table.getRows()).singleElement().satisfies(row ->
+      {
+         assertThat(row).containsEntry("Pod", "pod-1").containsEntry("Threads", "4 threads").containsEntry("Draining", "No");
+         assertThat(row.get("Last heartbeat").toString()).matches("1[2-9]s ago");
+         assertThat(row.get("Started").toString()).matches("Started [A-Z][a-z]{2} \\d{1,2}, \\d\\d:\\d\\d UTC");
+         assertThat(texts((CompositeWidgetData) row.get(JobsWidgetRenderer.CAPACITY_CELL))).containsExactly("4 threads", "Draining: No");
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Record progress and audit links never disclose another owner's job.
+    *******************************************************************************/
+   @Test
+   void progressAndAudit() throws Exception
+   {
+      List<QRecord> jobs = seed();
+      String id = jobs.get(3).getValueString(ID);
+      login(USER_A, false);
+      MultiStatisticsData progress = (MultiStatisticsData) render("jobsJobProgress", id);
+      assertThat(progress.getStatisticsGroupData()).extracting(MultiStatisticsData.StatisticsGroupData::getHeader).containsExactly("Rows", "Chunks", "Speed");
+      assertThat(groupStat(progress, "Rows", "Succeeded")).isEqualTo(440);
+      assertThat(groupStat(progress, "Rows", "Failed")).isEqualTo(10);
+      assertThat(groupStat(progress, "Rows", "Total")).isEqualTo(1000);
+      assertThat(group(progress, "Rows").getSubheader()).isEqualTo("45% done");
+      assertThat(groupStat(progress, "Chunks", "Done")).isEqualTo(4);
+      assertThat(groupStat(progress, "Speed", "Rows per minute")).isEqualTo(45);
+      assertThat(group(progress, "Speed").getSubheader()).isEqualTo("ETA in 12m");
+      String completed = jobs.get(8).getValueString(ID);
+      assertThat(group((MultiStatisticsData) render("jobsJobProgress", completed), "Speed").getSubheader()).isEqualTo("Completed 30m ago");
+      String paused = jobs.get(5).getValueString(ID);
+      assertThat(group((MultiStatisticsData) render("jobsJobProgress", paused), "Speed").getSubheader()).isEqualTo("Paused");
+      QContext.setQSession(new QSystemUserSession());
+      update(JOBS_JOB, jobs.get(5).getValue(ID), ROWS_TOTAL, null);
+      update(JOBS_JOB, jobs.get(5).getValue(ID), CHUNKS_TOTAL, null);
+      login(USER_A, false);
+      MultiStatisticsData unknown = (MultiStatisticsData) render("jobsJobProgress", paused);
+      assertThat(group(unknown, "Rows").getSubheader()).isEqualTo("Total not known yet");
+      assertThat(group(unknown, "Rows").getStatisticList()).hasSize(2);
+      assertThat(group(unknown, "Chunks").getStatisticList()).hasSize(2);
+      StatisticsData audit = (StatisticsData) render("jobsJobAudit", id);
+      assertThat(audit.getCount()).isEqualTo(1);
+      assertThat(URLDecoder.decode(audit.getCountURL(), StandardCharsets.UTF_8)).contains("CONTAINS", "job:" + id + "/", "message");
+      login(USER_B, false);
+      assertThat(((MultiStatisticsData) render("jobsJobProgress", id)).getStatisticsGroupData()).isEmpty();
+      assertThat(((StatisticsData) render("jobsJobAudit", id)).getCount()).isEqualTo(0);
+      assertThat(((StatisticsData) render("jobsJobAudit", id)).getCountURL()).isNull();
+      login("admin", true);
+      assertThat(((StatisticsData) render("jobsJobAudit", id)).getCount()).isEqualTo(1);
+   }
+
+   /*******************************************************************************
+    ** Child renderer ordering, owner locks and parent-record selection.
+    *******************************************************************************/
+   @Test
+   void childrenUseCoreRenderer() throws Exception
+   {
+      QRecord job = job(USER_A, RUNNING, null);
+      children(job);
+      insert(JOBS_CHUNK, new QRecord().withValue(JOB_ID, job.getValue(ID)).withValue(SEQ, 0).withValue(STATUS, DONE));
+      insert(JOBS_EVENT, new QRecord().withValue(JOB_ID, job.getValue(ID)).withValue(TYPE, RUNNING));
+      login(USER_A, false);
+      String id = job.getValueString(ID);
+      ChildRecordListData chunks = (ChildRecordListData) render("jobsJobChunks", id);
+      assertThat(chunks.getQueryOutput().getRecords()).extracting(record -> record.getValueInteger(SEQ)).containsExactly(0, 1);
+      ChildRecordListData errors = (ChildRecordListData) render("jobsJobRowErrors", id);
+      assertThat(errors.getTotalRows()).isEqualTo(1);
+      ChildRecordListData events = (ChildRecordListData) render("jobsJobEvents", id);
+      assertThat(events.getQueryOutput().getRecords()).extracting(record -> record.getValueString(TYPE)).containsExactly(RUNNING, COMPLETED);
+      login(USER_B, false);
+      assertThatThrownBy(() -> render("jobsJobChunks", id)).isInstanceOf(QException.class);
+      login("admin", true);
+      assertThat(((ChildRecordListData) render("jobsJobChunks", id)).getTotalRows()).isEqualTo(2);
+   }
+
+   /*******************************************************************************
+    ** Empty widgets still return typed, renderable payloads.
+    *******************************************************************************/
+   @Test
+   void emptyStates() throws Exception
+   {
+      login(USER_A, false);
+      assertThat(((TableData) render("jobsHomeActive", null)).getNoRowsFoundHTML()).isEqualTo("No jobs are running");
+      assertThat(((TableData) render("jobsHomeAttention", null)).getRows()).isEmpty();
+      assertThat(count((MultiStatisticsData) render("jobsHomeSummary", null), "Running")).isZero();
+      assertThat(chartTotal((ChartData) render("jobsHomeThroughput", null))).isZero();
+      assertThat(chartTotal((ChartData) render("jobsHomeHistory", null))).isZero();
+      assertThat(((MultiStatisticsData) render("jobsJobProgress", null)).getStatisticsGroupData()).isEmpty();
+      assertThat(((StatisticsData) render("jobsJobAudit", null)).getCount()).isEqualTo(0);
+      login("admin", true);
+      assertThat(((TableData) render("jobsHomeWorkers", null)).getNoRowsFoundHTML()).isEqualTo("No workers have registered");
+      assertThatThrownBy(() -> new JobsWidgetRenderer().render(new RenderWidgetInput().withWidgetMetaData(new QWidgetMetaData().withName("unknown"))))
+         .hasMessageContaining("Unknown Jobs widget");
+   }
+
+   /*******************************************************************************
+    ** Review counts belong only to listed jobs; core controls attention eligibility.
+    *******************************************************************************/
+   @Test
+   void attentionEffectsAndOwnerIsolation() throws Exception
+   {
+      QRecord review = job(USER_A, NEEDS_REVIEW, now);
+      effect(review, UNKNOWN, true);
+      effect(review, UNKNOWN, true);
+      effect(review, PENDING, false);
+      effect(job(USER_A, RUNNING, now), UNKNOWN, true);
+      QRecord other = job(USER_B, FAILED, now.minusSeconds(60));
+      effect(other, UNKNOWN, true);
+      QSession viewer = login(USER_A, false);
+      TableData attention = (TableData) render("jobsHomeAttention", null);
+      assertThat(attention.getRows()).hasSize(1);
+      assertThat(texts((CompositeWidgetData) attention.getRows().getFirst().get(JobsWidgetRenderer.STATUS_CELL))).contains("2 effects need review");
+      assertThat(QContext.getQSession()).isSameAs(viewer);
+      login(USER_B, false);
+      attention = (TableData) render("jobsHomeAttention", null);
+      assertThat(attention.getRows()).hasSize(1);
+      assertThat(texts((CompositeWidgetData) attention.getRows().getFirst().get(JobsWidgetRenderer.STATUS_CELL)))
+         .contains("1 effects need review").doesNotContain("2 effects need review");
+      assertThat(records(JOBS_EFFECT)).hasSize(1).allSatisfy(record -> assertThat(record.getValue(JOB_ID)).isEqualTo(other.getValue(ID)));
+   }
+
+   /*******************************************************************************
+    ** Effects use disjoint statistics and the secured core child renderer.
+    *******************************************************************************/
+   @Test
+   void effectsProgressAndChildList() throws Exception
+   {
+      QRecord job = job(USER_A, NEEDS_REVIEW, now);
+      for(String status : List.of(SENT, SENT, PENDING, SENDING, UNKNOWN, FAILED))
+      {
+         effect(job, status, false);
+      }
+      QRecord firstReview = effect(job, UNKNOWN, true);
+      effect(job, PENDING, true);
+      effect(job(USER_B, RUNNING, null), SENT, false);
+      String id = job.getValueString(ID);
+      QSession viewer = login(USER_A, false);
+      MultiStatisticsData progress = (MultiStatisticsData) render("jobsJobProgress", id);
+      assertThat(progress.getStatisticsGroupData()).extracting(MultiStatisticsData.StatisticsGroupData::getHeader)
+         .containsExactly("Rows", "Chunks", "Speed", "Effects");
+      assertThat(groupStat(progress, "Effects", "Sent")).isEqualTo(2);
+      assertThat(groupStat(progress, "Effects", "Pending")).isEqualTo(3);
+      assertThat(groupStat(progress, "Effects", "Failed")).isEqualTo(1);
+      assertThat(groupStat(progress, "Effects", "Needs review")).isEqualTo(2);
+      ChildRecordListData effects = (ChildRecordListData) render("jobsJobEffects", id);
+      assertThat(effects.getTotalRows()).isEqualTo(8);
+      assertThat(effects.getQueryOutput().getRecords()).extracting(record -> record.getValueInteger(ID)).isSorted();
+      assertThat(effects.getQueryOutput().getRecords()).extracting(record -> record.getValue(ID)).contains(firstReview.getValue(ID));
+      assertThat(QContext.getQSession()).isSameAs(viewer);
+      login(USER_B, false);
+      assertThat(((MultiStatisticsData) render("jobsJobProgress", id)).getStatisticsGroupData()).isEmpty();
+      assertThatThrownBy(() -> render("jobsJobEffects", id)).isInstanceOf(QException.class);
+      login("admin", true);
+      assertThat(((ChildRecordListData) render("jobsJobEffects", id)).getTotalRows()).isEqualTo(8);
+   }
+
+   /*******************************************************************************
+    ** Attention and progress tolerate a host omitting the effects table.
+    *******************************************************************************/
+   @Test
+   void widgetsWithoutEffectsTable() throws Exception
+   {
+      QRecord job = job(USER_A, FAILED, now);
+      instance.getTables().remove(JOBS_EFFECT);
+      login(USER_A, false);
+      assertThat(((TableData) render("jobsHomeAttention", null)).getRows()).hasSize(1);
+      assertThat(((MultiStatisticsData) render("jobsJobProgress", job.getValueString(ID))).getStatisticsGroupData())
+         .extracting(MultiStatisticsData.StatisticsGroupData::getHeader).containsExactly("Rows", "Chunks", "Speed");
+   }
+
+   /*******************************************************************************
+    ** Test fixture update.
+    *******************************************************************************/
+   private void update(String table, Serializable id, String field, Serializable value) throws QException
+   {
+      new UpdateAction().execute(new UpdateInput().withTableName(table).withRecord(new QRecord().withValue(ID, id).withValue(field, value)));
+   }
+
+   /*******************************************************************************
+    ** Find a group by its header.
+    *******************************************************************************/
+   private MultiStatisticsData.StatisticsGroupData group(MultiStatisticsData data, String header)
+   {
+      return (data.getStatisticsGroupData().stream().filter(group -> header.equals(group.getHeader())).findFirst().orElseThrow());
+   }
+
+
+
+   /*******************************************************************************
+    ** Find a statistic by its label in any group (labels are unique on the home).
+    *******************************************************************************/
+   private MultiStatisticsData.StatisticsGroupData.Statistic stat(MultiStatisticsData data, String label)
+   {
+      return (data.getStatisticsGroupData().stream().flatMap(group -> group.getStatisticList().stream())
+         .filter(statistic -> label.equals(statistic.getLabel())).findFirst().orElseThrow());
+   }
+
+
+
+   /*******************************************************************************
+    ** Numeric statistic value by label.
+    *******************************************************************************/
+   private int count(MultiStatisticsData data, String label)
+   {
+      return (stat(data, label).getValue());
+   }
+
+
+
+   /*******************************************************************************
+    ** Numeric statistic value by group header and label.
+    *******************************************************************************/
+   private int groupStat(MultiStatisticsData data, String header, String label)
+   {
+      return (group(data, header).getStatisticList().stream().filter(statistic -> label.equals(statistic.getLabel())).findFirst().orElseThrow().getValue());
+   }
+
+
+
+   /*******************************************************************************
+    ** The text of each text block in a composite cell, in order.
+    *******************************************************************************/
+   private List<String> texts(CompositeWidgetData cell)
+   {
+      return (cell.getBlocks().stream().filter(block -> block instanceof TextBlockData)
+         .map(block -> ((TextBlockData) block).getValues().getText()).toList());
+   }
+
+
+
+   /*******************************************************************************
+    ** Total over all chart buckets and series.
+    *******************************************************************************/
+   private long chartTotal(ChartData chart)
+   {
+      return (chart.getChartData().getDatasets().stream().flatMap(dataset -> dataset.getData().stream()).mapToLong(Number::longValue).sum());
+   }
+}