Merge pull request 'build(deps): align with Gitea registry (parent 2.2.1, QQQ 4.1.0-RC.26)' (#34) from deps/align-gitea into develop

pom.xml: parent -> 2.2.1
pom.xml: qqq.snapshot.version: 4.1.0-RC.14 -> 4.1.0-RC.26
pom.xml: removed Central snapshot repository
pom.xml: added qqq.version=4.1.0-RC.26 (parent BOM import)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.circleci/config.yml

@@ -0,0 +1,14 @@
+version: 2.1
+setup: true
+orbs:
+  qqq-orb: kingsrook/qqq-orb@0.8
+workflows:
+  setup:
+    jobs:
+    - qqq-orb/generate_pipeline:
+        orb_version: '0.8'
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.gitea/workflows/build.yaml

@@ -0,0 +1,7 @@
+name: build
+on:
+  push:
+jobs:
+  maven:
+    uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
+    secrets: inherit

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -0,0 +1,74 @@
+name: Release candidate
+'on':
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: Plan a cut without changing release state
+        type: boolean
+        default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
+  pull_request:
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
+permissions:
+  contents: write
+  actions: read
+  checks: read
+jobs:
+  release-rc:
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
+    with:
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,3 @@
+wrapperVersion=3.3.4
+distributionType=only-script
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: '0.8'
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

AGENTS.md

@@ -0,0 +1,68 @@
+# AGENTS.md
+
+This file provides guidance to Codex and other agents when working with this repository.
+
+## Project overview
+
+QBit Cluster runs a QQQ application as one unit on N nodes: node registry and heartbeats
+(`cluster_node`), lost-node detection with the `cluster.node.lost` alert, compare-and-set leases
+for singletons (`cluster_lease`, `ClusterLeases`), seeded settings (`cluster_setting`), Drain node
+/ Cancel drain record actions, the `cluster.node.heartbeat` health check and the Cluster app with
+the Nodes widget. Read `docs/DESIGN.md` and `README.md` first.
+
+## Build commands
+
+Java 21; always use the wrapper (offline in this workspace):
+
+```bash
+./mvnw -o verify                       # checkstyle, tests, JaCoCo (>=70% instructions, >=90% classes)
+./mvnw -o test -Dtest=ClusterLeasesTest
+```
+
+Tests use JUnit 5, AssertJ, MOCK auth and the memory backend; `ClusterChangelogTest` runs the
+shipped changelogs through real Liquibase on H2 and the runtime through the RDBMS module. No
+Docker. If core cluster classes are missing, the shared `~/.m2` 4.1.0-SNAPSHOT was overwritten:
+reinstall QQQ from the `feature/cluster-ha-p1` worktree.
+
+## Architecture by package
+
+- `com.kingsrook.qbits.cluster`: `ClusterQBitProducer` (adds everything directly),
+  `ClusterQBitConfig`, `ClusterLeases`.
+- `model`: `ClusterTables` (table metadata, field and table names), `NodeStatus`,
+  `ClusterSettings` (defaults + load), `ClusterSettingValidator`.
+- `runtime`: `ClusterRuntime` (thread, tick, stop), `ClusterRuntimeService`,
+  `ClusterNodeRegistry` (own row), `ClusterNodeInfo` (env, roles, JVM), `ClusterSweeper`,
+  `ClusterAlerts`, `ClusterSystem` (system-session scope).
+- `processes`: `ClusterNodeActions` (metadata), `ClusterNodeActionSteps` (Review / Result).
+- `widgets`: `ClusterNodesWidgetRenderer`. `health`: `ClusterHeartbeatHealthCheck`.
+
+## Invariants
+
+- Every write that two nodes could race on is a `ConditionalUpdateAction` compare-and-set; times
+  come from the backend clock (`databaseNow`), never the pod clock.
+- The qbit never runs DDL. Schema and defaults ship as Liquibase SQL in the jar; defaults are
+  insert-only with empty rollbacks. Column names = snake_case of field names; keep the changelog,
+  `ClusterTables` and `ClusterSettings.DEFAULTS` in step (tests enforce it).
+- Processes are record actions only (never in navigation), standard streamed ETL, plain labels.
+- Renderers read in the viewer's session; only the runtime's own bookkeeping runs as system.
+- Every table, field, process, widget and possible-value source has help in
+  `help/qbit-cluster.yaml` (`ClusterQBitProducerTest.helpIsComplete`). Summaries are plain text,
+  at most 120 characters, no underscores.
+
+## Naming and code style
+
+Tables `cluster_*`, permissions `cluster.read`, `cluster.write`, `cluster.drain`, processes
+`clusterDrainNode` / `clusterCancelDrain`, widget `clusterNodes`, lease `cluster-sweeper`.
+Kingsrook style: three-space indent, braces on new lines, `return (x);`, flower-box Javadoc on
+every class and method, the Apache license header. Do not weaken checkstyle or coverage gates.
+
+## Dependencies
+
+Parent `com.kingsrook:qbit-build-parent:2.0.0`; the default `qqq-snapshot` profile imports
+`qqq-bom-pom` `${qqq.snapshot.version}` (4.1.0-SNAPSHOT until a QQQ RC ships the cluster APIs).
+
+## Release flow
+
+CircleCI (`kingsrook/qqq-orb`): feature branches test; `develop` publishes SNAPSHOTs; `release/*`
+publishes RCs (cut by `.github/workflows/release-rc.yml`); `vX.Y.Z` tags publish releases.
+Publishing requires the owner's authorization. Update CHANGELOG before a release.

CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+All notable changes follow [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
+and [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Changed
+
+- Reconciled legacy release-branch history with the current Gitea CI while retaining the develop
+  SNAPSHOT revision, so normal RC cuts can merge cleanly.
+
+### Fixed
+
+- SQL changelog headers: explanatory "-- Include it from the host..." comments were parsed as a
+  licensed Liquibase `include` directive. Replaced with neutral wording; changeset bodies and
+  schema are unchanged.
+
+## [0.1.0] - Unreleased
+
+### Added
+
+- Tables `cluster_node`, `cluster_lease`, `cluster_setting` (standard QQQ tables, `cluster.read` /
+  `cluster.write`), node status possible values, and the Cluster app with the Nodes widget
+  (nodes grouped by role, status, heartbeat age).
+- Liquibase changelogs in the jar: `db/liquibase/qbit-cluster/cluster.changelog.sql` (schema) and
+  `cluster-defaults.changelog.sql` (insert-only seeded settings).
+- Runtime: node registration and heartbeat, drain requests flowing to `QNodeLifecycle`, DRAINING /
+  STOPPED tracking, lost-node sweep with the `cluster.node.lost` alert, purge of old rows, all on
+  the database clock; `ClusterRuntimeService` for `QApplicationLauncher`.
+- `ClusterLeases`: compare-and-set leases (`tryAcquire`, `renew`, `release`, `isLeader`).
+- Record actions Drain node and Cancel drain (standard ETL Review / Result, `cluster.drain`).
+- Health check `cluster.node.heartbeat`; help bundle `help/qbit-cluster.yaml`.
+- Built against QQQ `4.1.0-SNAPSHOT` (feature/cluster-ha-p1); the release switches
+  `qqq.snapshot.version` to the RC that ships the cluster APIs.

CLAUDE.md

@@ -0,0 +1,68 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code when working with this repository.
+
+## Project overview
+
+QBit Cluster runs a QQQ application as one unit on N nodes: node registry and heartbeats
+(`cluster_node`), lost-node detection with the `cluster.node.lost` alert, compare-and-set leases
+for singletons (`cluster_lease`, `ClusterLeases`), seeded settings (`cluster_setting`), Drain node
+/ Cancel drain record actions, the `cluster.node.heartbeat` health check and the Cluster app with
+the Nodes widget. Read `docs/DESIGN.md` and `README.md` first.
+
+## Build commands
+
+Java 21; always use the wrapper (offline in this workspace):
+
+```bash
+./mvnw -o verify                       # checkstyle, tests, JaCoCo (>=70% instructions, >=90% classes)
+./mvnw -o test -Dtest=ClusterLeasesTest
+```
+
+Tests use JUnit 5, AssertJ, MOCK auth and the memory backend; `ClusterChangelogTest` runs the
+shipped changelogs through real Liquibase on H2 and the runtime through the RDBMS module. No
+Docker. If core cluster classes are missing, the shared `~/.m2` 4.1.0-SNAPSHOT was overwritten:
+reinstall QQQ from the `feature/cluster-ha-p1` worktree.
+
+## Architecture by package
+
+- `com.kingsrook.qbits.cluster`: `ClusterQBitProducer` (adds everything directly),
+  `ClusterQBitConfig`, `ClusterLeases`.
+- `model`: `ClusterTables` (table metadata, field and table names), `NodeStatus`,
+  `ClusterSettings` (defaults + load), `ClusterSettingValidator`.
+- `runtime`: `ClusterRuntime` (thread, tick, stop), `ClusterRuntimeService`,
+  `ClusterNodeRegistry` (own row), `ClusterNodeInfo` (env, roles, JVM), `ClusterSweeper`,
+  `ClusterAlerts`, `ClusterSystem` (system-session scope).
+- `processes`: `ClusterNodeActions` (metadata), `ClusterNodeActionSteps` (Review / Result).
+- `widgets`: `ClusterNodesWidgetRenderer`. `health`: `ClusterHeartbeatHealthCheck`.
+
+## Invariants
+
+- Every write that two nodes could race on is a `ConditionalUpdateAction` compare-and-set; times
+  come from the backend clock (`databaseNow`), never the pod clock.
+- The qbit never runs DDL. Schema and defaults ship as Liquibase SQL in the jar; defaults are
+  insert-only with empty rollbacks. Column names = snake_case of field names; keep the changelog,
+  `ClusterTables` and `ClusterSettings.DEFAULTS` in step (tests enforce it).
+- Processes are record actions only (never in navigation), standard streamed ETL, plain labels.
+- Renderers read in the viewer's session; only the runtime's own bookkeeping runs as system.
+- Every table, field, process, widget and possible-value source has help in
+  `help/qbit-cluster.yaml` (`ClusterQBitProducerTest.helpIsComplete`). Summaries are plain text,
+  at most 120 characters, no underscores.
+
+## Naming and code style
+
+Tables `cluster_*`, permissions `cluster.read`, `cluster.write`, `cluster.drain`, processes
+`clusterDrainNode` / `clusterCancelDrain`, widget `clusterNodes`, lease `cluster-sweeper`.
+Kingsrook style: three-space indent, braces on new lines, `return (x);`, flower-box Javadoc on
+every class and method, the Apache license header. Do not weaken checkstyle or coverage gates.
+
+## Dependencies
+
+Parent `com.kingsrook:qbit-build-parent:2.0.0`; the default `qqq-snapshot` profile imports
+`qqq-bom-pom` `${qqq.snapshot.version}` (4.1.0-SNAPSHOT until a QQQ RC ships the cluster APIs).
+
+## Release flow
+
+CircleCI (`kingsrook/qqq-orb`): feature branches test; `develop` publishes SNAPSHOTs; `release/*`
+publishes RCs (cut by `.github/workflows/release-rc.yml`); `vX.Y.Z` tags publish releases.
+Publishing requires the owner's authorization. Update CHANGELOG before a release.

README.md

@@ -0,0 +1,142 @@
+# QBit Cluster
+
+Run a QQQ application as one unit on N nodes. Each node registers itself in `cluster_node` and
+sends a heartbeat; a node that goes quiet is marked Lost and raises an alert; one node at a time
+holds named leases for singleton work (`ClusterLeases`); an administrator can drain a node from
+the UI; and an Admin home widget shows the nodes by role. Core QQQ stays orchestrator-agnostic:
+this qbit builds on core's `QNodeRoles`, `QNodeLifecycle` and `AsyncJobRegistry`.
+
+Design: [QRun-IO/qqq docs/specs/2026-10-15-cluster-ha-design.md](https://github.com/QRun-IO/qqq/blob/develop/docs/specs/2026-10-15-cluster-ha-design.md),
+section E (summary in [docs/DESIGN.md](docs/DESIGN.md)).
+
+## Requirements
+
+- Java 21.
+- QQQ 4.1 with the cluster APIs (`com.kingsrook.qqq.backend.core.cluster.QNodeRoles`,
+  `QNodeLifecycle`, `AsyncJobRegistry.runningCount()`, `ConditionalUpdateAction`). Until a QQQ
+  RC ships them, the build uses a locally installed `4.1.0-SNAPSHOT` (the `qqq.snapshot.version`
+  property in `pom.xml`).
+- A backend with the `CONDITIONAL_UPDATE` capability for the `cluster_` tables (the RDBMS module;
+  the memory backend in tests). PostgreSQL in production.
+
+## Install
+
+```xml
+<dependency>
+   <groupId>com.kingsrook.qbits</groupId>
+   <artifactId>qbit-cluster</artifactId>
+   <version>0.1.0-RC.1</version><!-- first release; see GitHub releases for the latest -->
+</dependency>
+```
+
+```java
+new ClusterQBitProducer()
+   .withQBitConfig(new ClusterQBitConfig()
+      .withBackendName("rdbms")           // required
+      .withParentAppName("admin"))         // optional: Admin > Cluster
+   .produce(qInstance);
+```
+
+Database (Liquibase, from the jar; the host runs it, the qbit never runs DDL):
+
+```yaml
+- include: { file: db/liquibase/qbit-cluster/cluster.changelog.sql }
+- include: { file: db/liquibase/qbit-cluster/cluster-defaults.changelog.sql }
+```
+
+Both are idempotent, PostgreSQL-first (also H2), have no foreign keys and need no superuser.
+
+Start the runtime with the application:
+
+- `QApplicationLauncher` starts it automatically (the producer registers `ClusterRuntimeService`).
+- A host that starts services itself (e.g. with `QApplicationJavalinServer`) calls
+  `new ClusterRuntimeService().start(server.getQInstance())` after the server starts, and
+  `stop()` in a shutdown hook.
+
+## Tables
+
+| Table | What | Who writes it |
+|---|---|---|
+| `cluster_node` | One row per node: node id, pod, Kubernetes node, zone, deployment, roles, version, started/heartbeat times, status (UP, DRAINING, LOST, STOPPED), worker threads, scheduler/ESB flags, running steps, heap, drain request | Each node (no insert or edit in the UI; Drain node / Cancel drain; rows may be deleted) |
+| `cluster_lease` | Named leases: holder, expiry, version (compare-and-set) | `ClusterLeases`; an admin may edit to force a hand-over |
+| `cluster_setting` | `heartbeat_seconds`=15, `lost_after_seconds`=60, `retain_stopped_days`=7, `alert_on_lost`=true (seeded; validated on save) | Administrators |
+
+Node placement comes from optional environment variables (the Kubernetes Downward API):
+`QQQ_POD_NAME`, `QQQ_K8S_NODE_NAME`, `QQQ_ZONE`, `QQQ_DEPLOYMENT`; the version from
+`ClusterQBitConfig.withAppVersion`, else `QQQ_APP_VERSION`, else the QQQ version. Node id and
+roles come from core (`QQQ_NODE_ID` / `HOSTNAME`, `QQQ_NODE_ROLES`).
+
+## Runtime
+
+- On start: upsert this node's row (status UP, drain request cleared).
+- Every `heartbeat_seconds`: write the heartbeat and live fields; if `drain_requested` is set,
+  call `QNodeLifecycle.startDraining("requested by <user>: <reason>")`.
+- A `QNodeLifecycle.onDrain` listener marks the node DRAINING (from a request or SIGTERM) and
+  gives up leases; stop (service stop or JVM shutdown hook) marks it STOPPED.
+- The node holding the `cluster-sweeper` lease marks nodes with a heartbeat older than
+  `lost_after_seconds` LOST, raises `cluster.node.lost` through core `QAlerts` (qbit-alerts turns
+  it into an Alerts record), and deletes STOPPED/LOST rows older than `retain_stopped_days`. The
+  sweep is idempotent, so a crash mid-sweep is simply redone by the next leader.
+- Health check `cluster.node.heartbeat`: UP when this node's heartbeat is within two intervals,
+  DEGRADED when late, DOWN past `lost_after_seconds`.
+
+All times compare on the database clock (`ConditionalUpdateAction.databaseNow`), not pod clocks.
+
+## Leases
+
+```java
+if(ClusterLeases.tryAcquire("my-singleton", Duration.ofSeconds(60)))
+{
+   // leader for up to 60 s; call again (or renew) each cycle to keep it
+}
+ClusterLeases.renew(name, ttl);  ClusterLeases.release(name);  ClusterLeases.isLeader(name);
+ClusterLeases.as("some-node-id").tryAcquire(...);   // explicit node id
+```
+
+Every change is one atomic `UPDATE cluster_lease SET ... WHERE name = ? AND version = ? AND
+(expires_at < now OR holder_node_id = me OR holder_node_id IS NULL)` through core's
+`ConditionalUpdateAction`.
+
+## Permissions
+
+| Permission | Grants |
+|---|---|
+| `cluster.read` | The Cluster app, the Nodes widget, reading the three tables |
+| `cluster.write` | Editing settings and leases, deleting old node rows |
+| `cluster.drain` | The Drain node and Cancel drain record actions |
+
+Base names are configurable: `withPermissionBaseName("cluster")`, `withDrainPermissionName("cluster.drain")`.
+
+## Configuration
+
+| Option | Default | |
+|---|---|---|
+| `backendName` | (required) | Backend of the `cluster_` tables |
+| `appName` / `appLabel` | `cluster` / `Cluster` | The app (its home shows the Nodes widget) |
+| `parentAppName` | none | Attach the app under an existing app (e.g. `admin`) |
+| `permissionBaseName` | `cluster` | `cluster.read` / `cluster.write` |
+| `drainPermissionName` | `cluster.drain` | |
+| `appVersion` | none | Version each node reports |
+| `widgetRefreshSeconds` | 15 | |
+| `registerRuntimeService` | true | Add `ClusterRuntimeService` to the instance's runtime services |
+
+Table names are fixed (`cluster_` prefix, per the QRun qbit standard).
+
+## Adopting in another app
+
+1. Add the dependency and produce the qbit (above), after the backend exists and before the
+   host attaches its admin apps.
+2. Include both changelogs in the host's master changelog.
+3. Start `ClusterRuntimeService` (automatic with `QApplicationLauncher`).
+4. Grant `cluster.read`, `cluster.write`, `cluster.drain` to administrators.
+5. Optionally set the Downward API variables in the Deployment.
+
+## Build
+
+```bash
+./mvnw verify      # checkstyle, tests (memory backend + H2/Liquibase), JaCoCo gates
+```
+
+## License
+
+Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).

checkstyle/config.xml

@@ -0,0 +1,246 @@
+<?xml version="1.0"?>
+<!--
+  ~ QQQ - Low-code Application Framework for Engineers.
+  ~ Copyright (C) 2021-2022.  Kingsrook, LLC
+  ~ 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+  ~ contact@kingsrook.com
+  ~ https://github.com/Kingsrook/
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     https://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
+<!--
+   See reference at https://checkstyle.sourceforge.io/checks.html
+-->
+<module name = "Checker">
+   <property name="charset" value="UTF-8"/>
+
+   <property name="severity" value="warning"/>
+
+   <property name="fileExtensions" value="java, properties, xml"/>
+   <!-- Checks for whitespace                        -->
+   <!-- See http://checkstyle.sf.net/config_whitespace.html -->
+   <module name="FileTabCharacter">
+      <property name="eachLine" value="true"/>
+   </module>
+
+   <!--
+   <module name="LineLength">
+      <property name="max" value="100"/>
+      <property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
+   </module>
+   -->
+
+   <module name="TreeWalker">
+      <module name="SuppressWarningsHolder"/>
+      <module name="OuterTypeFilename"/>
+      <module name="IllegalTokenText">
+         <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
+         <property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
+         <property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
+      </module>
+      <module name="AvoidEscapedUnicodeCharacters">
+         <property name="allowEscapesForControlCharacters" value="true"/>
+         <property name="allowByTailComment" value="true"/>
+         <property name="allowNonPrintableEscapes" value="true"/>
+      </module>
+      <module name="AvoidStarImport"/>
+      <module name="OneTopLevelClass"/>
+      <module name="NoLineWrap"/>
+      <module name="EmptyBlock">
+         <property name="option" value="TEXT"/>
+         <property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
+      </module>
+      <module name="NeedBraces"/>
+      <module name="LeftCurly">
+         <property name="option" value="nl"/>
+      </module>
+      <module name="RightCurly">
+         <property name="option" value="alone"/>
+      </module>
+      <!--
+      <module name="WhitespaceAround">
+         <property name="allowEmptyConstructors" value="true"/>
+         <property name="allowEmptyMethods" value="true"/>
+         <property name="allowEmptyTypes" value="true"/>
+         <property name="allowEmptyLoops" value="true"/>
+         <message key="ws.notFollowed"
+          value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
+          <message key="ws.notPreceded"
+          value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
+      </module>
+      -->
+      <module name="OneStatementPerLine"/>
+      <module name="MultipleVariableDeclarations"/>
+      <module name="ArrayTypeStyle"/>
+      <module name="MissingSwitchDefault"/>
+      <module name="FallThrough"/>
+      <module name="UpperEll"/>
+      <module name="ModifierOrder"/>
+      <module name="EmptyLineSeparator">
+         <property name="allowNoEmptyLineBetweenFields" value="true"/>
+      </module>
+      <module name="SeparatorWrap">
+         <property name="id" value="SeparatorWrapDot"/>
+         <property name="tokens" value="DOT"/>
+         <property name="option" value="nl"/>
+      </module>
+      <module name="SeparatorWrap">
+         <property name="id" value="SeparatorWrapComma"/>
+         <property name="tokens" value="COMMA"/>
+         <property name="option" value="EOL"/>
+      </module>
+      <module name="SeparatorWrap">
+         <!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
+         <property name="id" value="SeparatorWrapEllipsis"/>
+         <property name="tokens" value="ELLIPSIS"/>
+         <property name="option" value="EOL"/>
+      </module>
+      <module name="SeparatorWrap">
+         <!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
+         <property name="id" value="SeparatorWrapArrayDeclarator"/>
+         <property name="tokens" value="ARRAY_DECLARATOR"/>
+         <property name="option" value="EOL"/>
+      </module>
+      <module name="SeparatorWrap">
+         <property name="id" value="SeparatorWrapMethodRef"/>
+         <property name="tokens" value="METHOD_REF"/>
+         <property name="option" value="nl"/>
+      </module>
+      <module name="PackageName">
+         <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
+         <message key="name.invalidPattern"
+          value="Package name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="TypeName">
+         <message key="name.invalidPattern"
+          value="Type name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="MemberName">
+         <property name="format" value="^[_a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
+         <message key="name.invalidPattern" value="Member name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="ParameterName">
+         <property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
+         <message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="CatchParameterName">
+         <property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
+         <message key="name.invalidPattern" value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="LocalVariableName">
+         <property name="tokens" value="VARIABLE_DEF"/>
+         <property name="format" value="^[a-zA-Z]([a-z0-9A-Z][a-zA-Z0-9]*)?$"/>
+         <message key="name.invalidPattern" value="Local variable name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="ClassTypeParameterName">
+         <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
+         <message key="name.invalidPattern" value="Class type name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="MethodTypeParameterName">
+         <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
+         <message key="name.invalidPattern" value="Method type name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="InterfaceTypeParameterName">
+         <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][_A-Z0-9]*$)"/>
+         <message key="name.invalidPattern" value="Interface type name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="NoFinalizer"/>
+      <module name="GenericWhitespace">
+         <message key="ws.followed" value="GenericWhitespace ''{0}'' is followed by whitespace."/>
+         <message key="ws.preceded" value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
+         <message key="ws.illegalFollow" value="GenericWhitespace ''{0}'' should followed by whitespace."/>
+         <message key="ws.notPreceded" value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
+      </module>
+      <module name="Indentation">
+         <property name="basicOffset" value="3"/>
+         <property name="braceAdjustment" value="0"/>
+         <property name="caseIndent" value="3"/>
+         <property name="throwsIndent" value="6"/>
+         <property name="lineWrappingIndentation" value="3"/>
+         <property name="arrayInitIndent" value="6"/>
+      </module>
+      <!--
+      <module name="AbbreviationAsWordInName">
+         <property name="ignoreFinal" value="false"/>
+         <property name="allowedAbbreviationLength" value="1"/>
+      </module>
+      <module name="OverloadMethodsDeclarationOrder"/>
+      <module name="VariableDeclarationUsageDistance"/>
+      <module name="CustomImportOrder">
+         <property name="sortImportsInGroupAlphabetically" value="true"/>
+         <property name="separateLineBetweenGroups" value="true"/>
+         <property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
+      </module>
+      -->
+      <module name="CustomImportOrder">
+         <property name="customImportOrderRules" value="SPECIAL_IMPORTS###STANDARD_JAVA_PACKAGE###THIRD_PARTY_PACKAGE###STATIC"/>
+         <property name="specialImportsRegExp" value="^javax\."/>
+         <property name="standardPackageRegExp" value="^java\."/>
+         <property name="sortImportsInGroupAlphabetically" value="true"/>
+         <property name="separateLineBetweenGroups" value="false"/>
+      </module>
+      <module name="MethodParamPad"/>
+      <module name="NoWhitespaceBefore">
+         <property name="tokens" value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
+         <property name="allowLineBreaks" value="true"/>
+      </module>
+      <module name="ParenPad"/>
+      <module name="OperatorWrap">
+         <property name="option" value="NL"/>
+         <property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
+      </module>
+      <module name="AnnotationLocation">
+         <property name="id" value="AnnotationLocationMostCases"/>
+         <property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
+      </module>
+      <module name="AnnotationLocation">
+         <property name="id" value="AnnotationLocationVariables"/>
+         <property name="tokens" value="VARIABLE_DEF"/>
+         <property name="allowSamelineMultipleAnnotations" value="true"/>
+      </module>
+      <module name="JavadocMethod">
+         <property name="allowMissingParamTags" value="true"/>
+         <property name="allowMissingReturnTag" value="true"/>
+         <property name="allowedAnnotations" value="Override, Test"/>
+      </module>
+      <module name="MissingJavadocMethod">
+         <property name="scope" value="private"/>
+      </module>
+      <module name="MissingJavadocType">
+         <property name="scope" value="private"/>
+      </module>
+      <module name="MethodName">
+         <property name="format" value="^[a-z][a-zA-Z0-9_]*$"/>
+         <message key="name.invalidPattern"
+          value="Method name ''{0}'' must match pattern ''{1}''."/>
+      </module>
+      <module name="EmptyCatchBlock">
+         <property name="exceptionVariableName" value="expected"/>
+      </module>
+      <module name="CommentsIndentation"/>
+
+      <!-- this would be nice, but requires the the @inheritDoc javadoc tag... -->
+      <module name="MissingOverride"/>
+
+   </module>
+   <module name="Header">
+     <property name="headerFile" value="checkstyle/license.txt"/>
+     <property name="fileExtensions" value="java"/>
+     <property name="ignoreLines" value="3"/>
+   </module>
+   <module name="SuppressWarningsFilter"/>
+</module>

checkstyle/license.txt

@@ -0,0 +1,19 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

docs/DESIGN.md

@@ -0,0 +1,33 @@
+# QBit Cluster design
+
+The approved design is section E of
+[QRun-IO/qqq docs/specs/2026-10-15-cluster-ha-design.md](https://github.com/QRun-IO/qqq/blob/develop/docs/specs/2026-10-15-cluster-ha-design.md)
+("QQQ on N nodes: cluster, HA, DR and scaling"). Sections A (node roles, `QNodeRoles`) and D
+(graceful drain, `QNodeLifecycle`) are the core APIs this qbit builds on.
+
+## What this qbit owns (P1)
+
+- **Registry** (`cluster_node`): one row per node, written by the node itself with
+  compare-and-set updates keyed by node id (no audits or record events; bookkeeping every 15 s).
+- **Leases** (`cluster_lease`, `ClusterLeases`): leadership for non-Quartz singletons. One atomic
+  conditional UPDATE per change, guarded by `version` and by "expired, free, or already mine".
+  Times on the database clock.
+- **Settings** (`cluster_setting`): seeded by an insert-only changelog; the runtime falls back to
+  the same defaults when a row is missing or invalid, and the table rejects invalid values.
+- **Drain**: a record action sets `drain_requested`; the node polls its own row each heartbeat and
+  calls `QNodeLifecycle.startDraining`, whose listener marks the row DRAINING. Core flips `/ready`
+  to 503 and stops claiming work. Cancel works only while the node is still UP (compare-and-set).
+- **Lost nodes**: the `cluster-sweeper` lease holder marks stale UP/DRAINING rows LOST
+  (compare-and-set on status and heartbeat) and raises `cluster.node.lost` via core `QAlerts`. A
+  LOST node that beats again goes back to UP and its alert resolves.
+- **Retention**: the same sweep deletes STOPPED/LOST rows older than `retain_stopped_days`. It is
+  idempotent, so a sweep interrupted by a crash is simply redone by the next leader; no separate
+  durable job is needed.
+- **Health**: `cluster.node.heartbeat` (this node's own heartbeat is current).
+- **UI**: the Cluster app (its home is the Nodes TABLE widget, full width) and the three tables,
+  through the standard `/qqq/v1` metadata; no custom frontend.
+
+## Not in P1
+
+Scaling Deployments through a Kubernetes client, metrics for HPA/KEDA, and requests-in-flight on
+the node row are P3 (spec section E). Cross-node cache invalidation is core P2 (`ClusterBus`).

mvnw

@@ -0,0 +1,295 @@
+#!/bin/sh
+# ----------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ----------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------
+# Apache Maven Wrapper startup batch script, version 3.3.4
+#
+# Optional ENV vars
+# -----------------
+#   JAVA_HOME - location of a JDK home dir, required when download maven via java source
+#   MVNW_REPOURL - repo url base for downloading maven distribution
+#   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+#   MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
+# ----------------------------------------------------------------------------
+
+set -euf
+[ "${MVNW_VERBOSE-}" != debug ] || set -x
+
+# OS specific support.
+native_path() { printf %s\\n "$1"; }
+case "$(uname)" in
+CYGWIN* | MINGW*)
+  [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
+  native_path() { cygpath --path --windows "$1"; }
+  ;;
+esac
+
+# set JAVACMD and JAVACCMD
+set_java_home() {
+  # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
+  if [ -n "${JAVA_HOME-}" ]; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ]; then
+      # IBM's JDK on AIX uses strange locations for the executables
+      JAVACMD="$JAVA_HOME/jre/sh/java"
+      JAVACCMD="$JAVA_HOME/jre/sh/javac"
+    else
+      JAVACMD="$JAVA_HOME/bin/java"
+      JAVACCMD="$JAVA_HOME/bin/javac"
+
+      if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
+        echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
+        echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
+        return 1
+      fi
+    fi
+  else
+    JAVACMD="$(
+      'set' +e
+      'unset' -f command 2>/dev/null
+      'command' -v java
+    )" || :
+    JAVACCMD="$(
+      'set' +e
+      'unset' -f command 2>/dev/null
+      'command' -v javac
+    )" || :
+
+    if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
+      echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
+      return 1
+    fi
+  fi
+}
+
+# hash string like Java String::hashCode
+hash_string() {
+  str="${1:-}" h=0
+  while [ -n "$str" ]; do
+    char="${str%"${str#?}"}"
+    h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
+    str="${str#?}"
+  done
+  printf %x\\n $h
+}
+
+verbose() { :; }
+[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
+
+die() {
+  printf %s\\n "$1" >&2
+  exit 1
+}
+
+trim() {
+  # MWRAPPER-139:
+  #   Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
+  #   Needed for removing poorly interpreted newline sequences when running in more
+  #   exotic environments such as mingw bash on Windows.
+  printf "%s" "${1}" | tr -d '[:space:]'
+}
+
+scriptDir="$(dirname "$0")"
+scriptName="$(basename "$0")"
+
+# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
+while IFS="=" read -r key value; do
+  case "${key-}" in
+  distributionUrl) distributionUrl=$(trim "${value-}") ;;
+  distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
+  esac
+done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
+[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
+
+case "${distributionUrl##*/}" in
+maven-mvnd-*bin.*)
+  MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
+  case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
+  *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
+  :Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
+  :Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
+  :Linux*x86_64*) distributionPlatform=linux-amd64 ;;
+  *)
+    echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
+    distributionPlatform=linux-amd64
+    ;;
+  esac
+  distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
+  ;;
+maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
+*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
+esac
+
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
+[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
+distributionUrlName="${distributionUrl##*/}"
+distributionUrlNameMain="${distributionUrlName%.*}"
+distributionUrlNameMain="${distributionUrlNameMain%-bin}"
+MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
+MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
+
+exec_maven() {
+  unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
+  exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
+}
+
+if [ -d "$MAVEN_HOME" ]; then
+  verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+  exec_maven "$@"
+fi
+
+case "${distributionUrl-}" in
+*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
+*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
+esac
+
+# prepare tmp dir
+if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
+  clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
+  trap clean HUP INT TERM EXIT
+else
+  die "cannot create temp dir"
+fi
+
+mkdir -p -- "${MAVEN_HOME%/*}"
+
+# Download and Install Apache Maven
+verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+verbose "Downloading from: $distributionUrl"
+verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+# select .zip or .tar.gz
+if ! command -v unzip >/dev/null; then
+  distributionUrl="${distributionUrl%.zip}.tar.gz"
+  distributionUrlName="${distributionUrl##*/}"
+fi
+
+# verbose opt
+__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
+[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
+
+# normalize http auth
+case "${MVNW_PASSWORD:+has-password}" in
+'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
+has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
+esac
+
+if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
+  verbose "Found wget ... using wget"
+  wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
+elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
+  verbose "Found curl ... using curl"
+  curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
+elif set_java_home; then
+  verbose "Falling back to use Java to download"
+  javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
+  targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
+  cat >"$javaSource" <<-END
+	public class Downloader extends java.net.Authenticator
+	{
+	  protected java.net.PasswordAuthentication getPasswordAuthentication()
+	  {
+	    return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
+	  }
+	  public static void main( String[] args ) throws Exception
+	  {
+	    setDefault( new Downloader() );
+	    java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
+	  }
+	}
+	END
+  # For Cygwin/MinGW, switch paths to Windows format before running javac and java
+  verbose " - Compiling Downloader.java ..."
+  "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
+  verbose " - Running Downloader.java ..."
+  "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
+fi
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+if [ -n "${distributionSha256Sum-}" ]; then
+  distributionSha256Result=false
+  if [ "$MVN_CMD" = mvnd.sh ]; then
+    echo "Checksum validation is not supported for maven-mvnd." >&2
+    echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
+    exit 1
+  elif command -v sha256sum >/dev/null; then
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
+      distributionSha256Result=true
+    fi
+  elif command -v shasum >/dev/null; then
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
+      distributionSha256Result=true
+    fi
+  else
+    echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
+    echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
+    exit 1
+  fi
+  if [ $distributionSha256Result = false ]; then
+    echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
+    echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
+    exit 1
+  fi
+fi
+
+# unzip and move
+if command -v unzip >/dev/null; then
+  unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
+else
+  tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
+fi
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+actualDistributionDir=""
+
+# First try the expected directory name (for regular distributions)
+if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
+  if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
+    actualDistributionDir="$distributionUrlNameMain"
+  fi
+fi
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if [ -z "$actualDistributionDir" ]; then
+  # enable globbing to iterate over items
+  set +f
+  for dir in "$TMP_DOWNLOAD_DIR"/*; do
+    if [ -d "$dir" ]; then
+      if [ -f "$dir/bin/$MVN_CMD" ]; then
+        actualDistributionDir="$(basename "$dir")"
+        break
+      fi
+    fi
+  done
+  set -f
+fi
+
+if [ -z "$actualDistributionDir" ]; then
+  verbose "Contents of $TMP_DOWNLOAD_DIR:"
+  verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
+  die "Could not find Maven distribution directory in extracted archive"
+fi
+
+verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
+mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
+
+clean || :
+exec_maven "$@"

mvnw.cmd

@@ -0,0 +1,189 @@
+<# : batch portion
+@REM ----------------------------------------------------------------------------
+@REM Licensed to the Apache Software Foundation (ASF) under one
+@REM or more contributor license agreements.  See the NOTICE file
+@REM distributed with this work for additional information
+@REM regarding copyright ownership.  The ASF licenses this file
+@REM to you under the Apache License, Version 2.0 (the
+@REM "License"); you may not use this file except in compliance
+@REM with the License.  You may obtain a copy of the License at
+@REM
+@REM    http://www.apache.org/licenses/LICENSE-2.0
+@REM
+@REM Unless required by applicable law or agreed to in writing,
+@REM software distributed under the License is distributed on an
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@REM KIND, either express or implied.  See the License for the
+@REM specific language governing permissions and limitations
+@REM under the License.
+@REM ----------------------------------------------------------------------------
+
+@REM ----------------------------------------------------------------------------
+@REM Apache Maven Wrapper startup batch script, version 3.3.4
+@REM
+@REM Optional ENV vars
+@REM   MVNW_REPOURL - repo url base for downloading maven distribution
+@REM   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
+@REM   MVNW_VERBOSE - true: enable verbose log; others: silence the output
+@REM ----------------------------------------------------------------------------
+
+@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
+@SET __MVNW_CMD__=
+@SET __MVNW_ERROR__=
+@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
+@SET PSModulePath=
+@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
+  IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
+)
+@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
+@SET __MVNW_PSMODULEP_SAVE=
+@SET __MVNW_ARG0_NAME__=
+@SET MVNW_USERNAME=
+@SET MVNW_PASSWORD=
+@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*)
+@echo Cannot start maven from wrapper >&2 && exit /b 1
+@GOTO :EOF
+: end batch / begin powershell #>
+
+$ErrorActionPreference = "Stop"
+if ($env:MVNW_VERBOSE -eq "true") {
+  $VerbosePreference = "Continue"
+}
+
+# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
+$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
+if (!$distributionUrl) {
+  Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
+}
+
+switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
+  "maven-mvnd-*" {
+    $USE_MVND = $true
+    $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
+    $MVN_CMD = "mvnd.cmd"
+    break
+  }
+  default {
+    $USE_MVND = $false
+    $MVN_CMD = $script -replace '^mvnw','mvn'
+    break
+  }
+}
+
+# apply MVNW_REPOURL and calculate MAVEN_HOME
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
+if ($env:MVNW_REPOURL) {
+  $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" }
+  $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')"
+}
+$distributionUrlName = $distributionUrl -replace '^.*/',''
+$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
+
+$MAVEN_M2_PATH = "$HOME/.m2"
+if ($env:MAVEN_USER_HOME) {
+  $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME"
+}
+
+if (-not (Test-Path -Path $MAVEN_M2_PATH)) {
+    New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null
+}
+
+$MAVEN_WRAPPER_DISTS = $null
+if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) {
+  $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists"
+} else {
+  $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists"
+}
+
+$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain"
+$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
+$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
+
+if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
+  Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
+  Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
+  exit $?
+}
+
+if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
+  Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
+}
+
+# prepare tmp dir
+$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
+$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
+$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
+trap {
+  if ($TMP_DOWNLOAD_DIR.Exists) {
+    try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+    catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+  }
+}
+
+New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
+
+# Download and Install Apache Maven
+Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
+Write-Verbose "Downloading from: $distributionUrl"
+Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
+
+$webclient = New-Object System.Net.WebClient
+if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
+  $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
+}
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
+
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
+$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
+if ($distributionSha256Sum) {
+  if ($USE_MVND) {
+    Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
+  }
+  Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
+  if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
+    Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
+  }
+}
+
+# unzip and move
+Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
+
+# Find the actual extracted directory name (handles snapshots where filename != directory name)
+$actualDistributionDir = ""
+
+# First try the expected directory name (for regular distributions)
+$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain"
+$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD"
+if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) {
+  $actualDistributionDir = $distributionUrlNameMain
+}
+
+# If not found, search for any directory with the Maven executable (for snapshots)
+if (!$actualDistributionDir) {
+  Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object {
+    $testPath = Join-Path $_.FullName "bin/$MVN_CMD"
+    if (Test-Path -Path $testPath -PathType Leaf) {
+      $actualDistributionDir = $_.Name
+    }
+  }
+}
+
+if (!$actualDistributionDir) {
+  Write-Error "Could not find Maven distribution directory in extracted archive"
+}
+
+Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir"
+Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null
+try {
+  Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
+} catch {
+  if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
+    Write-Error "fail to move MAVEN_HOME"
+  }
+} finally {
+  try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
+  catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
+}
+
+Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"

pom.xml

@@ -0,0 +1,230 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2026 Kingsrook, LLC
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+   <properties>
+      <qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
+      <revision>0.3.0-SNAPSHOT</revision>
+      <maven.compiler.source>21</maven.compiler.source>
+      <maven.compiler.target>21</maven.compiler.target>
+      <jacoco.version>0.8.15</jacoco.version>
+      <liquibase.version>4.33.0</liquibase.version>
+   </properties>
+
+   <parent>
+      <groupId>com.kingsrook</groupId>
+      <artifactId>qbit-build-parent</artifactId>
+      <version>2.2.1</version>
+   </parent>
+
+   <groupId>com.kingsrook.qbits</groupId>
+   <artifactId>qbit-cluster</artifactId>
+   <version>${revision}</version>
+   <packaging>jar</packaging>
+
+   <name>QBit Cluster</name>
+   <description>QBit for running a QQQ application on N nodes: node registry with heartbeats, lost-node detection and alerts, lease-based leadership for singletons, drain requests and an Admin view of the nodes</description>
+   <url>https://github.com/QRun-IO/qbit-cluster</url>
+
+   <modelVersion>4.0.0</modelVersion>
+
+   <licenses>
+      <license>
+         <name>Apache License, Version 2.0</name>
+         <url>https://www.apache.org/licenses/LICENSE-2.0</url>
+         <distribution>repo</distribution>
+      </license>
+   </licenses>
+
+   <developers>
+      <developer>
+         <id>kingsrook</id>
+         <name>Kingsrook Team</name>
+         <email>contact@kingsrook.com</email>
+         <organization>Kingsrook, LLC</organization>
+         <organizationUrl>https://kingsrook.com</organizationUrl>
+      </developer>
+   </developers>
+
+   <scm>
+      <connection>scm:git:git://github.com/QRun-IO/qbit-cluster.git</connection>
+      <developerConnection>scm:git:ssh://github.com:QRun-IO/qbit-cluster.git</developerConnection>
+      <url>https://github.com/QRun-IO/qbit-cluster</url>
+      <tag>HEAD</tag>
+   </scm>
+
+   <dependencies>
+      <dependency>
+         <groupId>com.kingsrook.qqq</groupId>
+         <artifactId>qqq-backend-core</artifactId>
+      </dependency>
+
+      <!-- Tests: the memory backend for the runtime, plus H2 through the RDBMS module and real -->
+      <!-- Liquibase, so the shipped changelogs and the lease compare-and-set run as real SQL.   -->
+      <dependency>
+         <groupId>com.kingsrook.qqq</groupId>
+         <artifactId>qqq-backend-module-rdbms</artifactId>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>com.h2database</groupId>
+         <artifactId>h2</artifactId>
+         <version>2.2.220</version>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>org.liquibase</groupId>
+         <artifactId>liquibase-core</artifactId>
+         <version>${liquibase.version}</version>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>org.junit.jupiter</groupId>
+         <artifactId>junit-jupiter</artifactId>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>org.assertj</groupId>
+         <artifactId>assertj-core</artifactId>
+         <version>3.27.7</version>
+         <scope>test</scope>
+      </dependency>
+   </dependencies>
+
+   <build>
+      <resources>
+         <!-- Only the version resource is filtered; changelogs and help are copied as-is. -->
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>true</filtering>
+            <includes>
+               <include>**/*.properties</include>
+            </includes>
+         </resource>
+         <resource>
+            <directory>src/main/resources</directory>
+            <filtering>false</filtering>
+            <excludes>
+               <exclude>**/*.properties</exclude>
+            </excludes>
+         </resource>
+      </resources>
+      <plugins>
+         <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-checkstyle-plugin</artifactId>
+            <version>3.3.1</version>
+            <dependencies>
+               <dependency>
+                  <groupId>com.puppycrawl.tools</groupId>
+                  <artifactId>checkstyle</artifactId>
+                  <version>10.12.6</version>
+               </dependency>
+            </dependencies>
+            <configuration>
+               <configLocation>checkstyle/config.xml</configLocation>
+               <headerLocation>checkstyle/license.txt</headerLocation>
+               <consoleOutput>true</consoleOutput>
+               <failOnViolation>true</failOnViolation>
+               <violationSeverity>warning</violationSeverity>
+               <includeTestSourceDirectory>true</includeTestSourceDirectory>
+            </configuration>
+            <executions>
+               <execution>
+                  <id>validate</id>
+                  <phase>validate</phase>
+                  <goals><goal>check</goal></goals>
+               </execution>
+            </executions>
+         </plugin>
+         <plugin>
+            <groupId>org.jacoco</groupId>
+            <artifactId>jacoco-maven-plugin</artifactId>
+            <version>${jacoco.version}</version>
+            <executions>
+               <execution>
+                  <id>prepare-agent</id>
+                  <goals>
+                     <goal>prepare-agent</goal>
+                  </goals>
+               </execution>
+               <execution>
+                  <id>report</id>
+                  <phase>verify</phase>
+                  <goals>
+                     <goal>report</goal>
+                  </goals>
+               </execution>
+               <execution>
+                  <id>check</id>
+                  <goals>
+                     <goal>check</goal>
+                  </goals>
+                  <configuration>
+                     <rules>
+                        <rule>
+                           <element>BUNDLE</element>
+                           <limits>
+                              <limit>
+                                 <counter>INSTRUCTION</counter>
+                                 <value>COVEREDRATIO</value>
+                                 <minimum>0.70</minimum>
+                              </limit>
+                              <limit>
+                                 <counter>CLASS</counter>
+                                 <value>COVEREDRATIO</value>
+                                 <minimum>0.90</minimum>
+                              </limit>
+                           </limits>
+                        </rule>
+                     </rules>
+                  </configuration>
+               </execution>
+            </executions>
+         </plugin>
+      </plugins>
+   </build>
+
+   <profiles>
+      <!-- Built against qqq 4.1. The cluster APIs (QNodeRoles, QNodeLifecycle,              -->
+      <!-- AsyncJobRegistry.runningCount) are not in a released QQQ RC yet, so this profile   -->
+      <!-- imports the locally installed 4.1.0-SNAPSHOT BOM (QRun-IO/qqq feature/cluster-ha-p1). -->
+      <!-- The release switches qqq.snapshot.version to the first RC that ships them: a      -->
+      <!-- one-line change (ADR-0007, as in the other qbits).                                -->
+      <profile>
+         <id>qqq-snapshot</id>
+         <activation>
+            <activeByDefault>true</activeByDefault>
+         </activation>
+         <properties>
+            <qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
+         </properties>
+         <dependencyManagement>
+            <dependencies>
+               <dependency>
+                  <groupId>com.kingsrook.qqq</groupId>
+                  <artifactId>qqq-bom-pom</artifactId>
+                  <version>${qqq.snapshot.version}</version>
+                  <type>pom</type>
+                  <scope>import</scope>
+               </dependency>
+            </dependencies>
+         </dependencyManagement>
+      </profile>
+   </profiles>
+</project>

src/main/java/com/kingsrook/qbits/cluster/ClusterLeases.java

@@ -0,0 +1,348 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.io.Serializable;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.runtime.ClusterSystem;
+import com.kingsrook.qqq.backend.core.actions.tables.ConditionalUpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.ConditionalUpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.ACQUIRED_AT;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.EXPIRES_AT;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.HOLDER_NODE_ID;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.LEASE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.LEASE_VERSION;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.MODIFY_DATE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.NAME;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** Lease-based leadership for singletons that are not Quartz jobs: at most one
+ ** node holds a named lease at a time, until it expires.
+ **
+ ** <pre>
+ ** if(ClusterLeases.tryAcquire("my-singleton", Duration.ofSeconds(60)))
+ ** {
+ **    // this node is the leader for (at most) the next 60 seconds;
+ **    // call tryAcquire (or renew) again well before then to keep it
+ ** }
+ ** </pre>
+ **
+ ** Every change is ONE atomic compare-and-set (core ConditionalUpdateAction;
+ ** for an RDBMS a single UPDATE cluster_lease SET ... WHERE name = ? AND
+ ** version = ? AND (expires_at &lt; now OR holder_node_id = me OR holder_node_id
+ ** IS NULL)), so two nodes can never both win. Times come from the backend's
+ ** clock (the database clock on an RDBMS), not each pod's own. The version
+ ** counter goes up on every change.
+ **
+ ** The static methods act for this node (QNodeRoles.nodeId()); as(nodeId) acts
+ ** for a given node id (tests, tools). They run as the system user, in the
+ ** current QContext's instance.
+ *******************************************************************************/
+public final class ClusterLeases
+{
+   private static final QLogger LOG = QLogger.getLogger(ClusterLeases.class);
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private ClusterLeases()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Acquire (or extend) the named lease for this node. True when this node
+    ** holds it for the next ttl.
+    *******************************************************************************/
+   public static boolean tryAcquire(String name, Duration ttl) throws QException
+   {
+      return (as(QNodeRoles.nodeId()).tryAcquire(name, ttl));
+   }
+
+
+
+   /*******************************************************************************
+    ** Extend a lease this node already holds. False when another node has it.
+    *******************************************************************************/
+   public static boolean renew(String name, Duration ttl) throws QException
+   {
+      return (as(QNodeRoles.nodeId()).renew(name, ttl));
+   }
+
+
+
+   /*******************************************************************************
+    ** Give up a lease this node holds (a no-op otherwise), so another node can
+    ** take it at once.
+    *******************************************************************************/
+   public static boolean release(String name) throws QException
+   {
+      return (as(QNodeRoles.nodeId()).release(name));
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether this node holds the named lease and it has not expired.
+    *******************************************************************************/
+   public static boolean isLeader(String name) throws QException
+   {
+      return (as(QNodeRoles.nodeId()).isLeader(name));
+   }
+
+
+
+   /*******************************************************************************
+    ** Leases for a given node id.
+    *******************************************************************************/
+   public static Holder as(String nodeId)
+   {
+      return (new Holder(nodeId));
+   }
+
+
+
+   /*******************************************************************************
+    ** The lease operations for one node id.
+    *******************************************************************************/
+   public static final class Holder
+   {
+      private final String nodeId;
+
+
+
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      private Holder(String nodeId)
+      {
+         if(nodeId == null || nodeId.isBlank())
+         {
+            throw (new IllegalArgumentException("A lease holder needs a node id"));
+         }
+         this.nodeId = nodeId;
+      }
+
+
+
+      /*******************************************************************************
+       ** Acquire, or extend, the lease: wins when it is free, expired or already
+       ** ours.
+       *******************************************************************************/
+      public boolean tryAcquire(String name, Duration ttl) throws QException
+      {
+         requireTtl(ttl);
+         return (ClusterSystem.call(() ->
+         {
+            Instant now   = now();
+            QRecord lease = read(name);
+            if(lease == null)
+            {
+               return (insert(name, now, ttl));
+            }
+
+            boolean ours = nodeId.equals(lease.getValueString(HOLDER_NODE_ID));
+            QQueryFilter freeOrOurs = new QQueryFilter()
+               .withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+               .withCriteria(new QFilterCriteria(EXPIRES_AT, QCriteriaOperator.LESS_THAN, now))
+               .withCriteria(new QFilterCriteria(HOLDER_NODE_ID, QCriteriaOperator.EQUALS, nodeId))
+               .withCriteria(new QFilterCriteria(HOLDER_NODE_ID, QCriteriaOperator.IS_BLANK));
+            boolean won = compareAndSet(lease, freeOrOurs, nodeId, now.plus(ttl), ours && lease.getValueInstant(ACQUIRED_AT) != null ? lease.getValueInstant(ACQUIRED_AT) : now, now);
+            if(won && !ours)
+            {
+               LOG.info("Acquired cluster lease", logPair("lease", name), logPair("nodeId", nodeId), logPair("previousHolder", lease.getValueString(HOLDER_NODE_ID)));
+            }
+            return (won);
+         }));
+      }
+
+
+
+      /*******************************************************************************
+       ** Extend the lease only if this node holds it (expired or not, as long as
+       ** no other node took it meanwhile).
+       *******************************************************************************/
+      public boolean renew(String name, Duration ttl) throws QException
+      {
+         requireTtl(ttl);
+         return (ClusterSystem.call(() ->
+         {
+            Instant now   = now();
+            QRecord lease = read(name);
+            if(lease == null || !nodeId.equals(lease.getValueString(HOLDER_NODE_ID)))
+            {
+               return (false);
+            }
+            Instant acquiredAt = lease.getValueInstant(ACQUIRED_AT) == null ? now : lease.getValueInstant(ACQUIRED_AT);
+            return (compareAndSet(lease, new QQueryFilter(new QFilterCriteria(HOLDER_NODE_ID, QCriteriaOperator.EQUALS, nodeId)), nodeId, now.plus(ttl), acquiredAt, now));
+         }));
+      }
+
+
+
+      /*******************************************************************************
+       ** Release the lease if this node holds it.
+       *******************************************************************************/
+      public boolean release(String name) throws QException
+      {
+         return (ClusterSystem.call(() ->
+         {
+            QRecord lease = read(name);
+            if(lease == null || !nodeId.equals(lease.getValueString(HOLDER_NODE_ID)))
+            {
+               return (false);
+            }
+            Instant now = now();
+            return (compareAndSet(lease, new QQueryFilter(new QFilterCriteria(HOLDER_NODE_ID, QCriteriaOperator.EQUALS, nodeId)), null, now, null, now));
+         }));
+      }
+
+
+
+      /*******************************************************************************
+       ** Whether this node holds the lease and it has not expired.
+       *******************************************************************************/
+      public boolean isLeader(String name) throws QException
+      {
+         return (ClusterSystem.call(() ->
+         {
+            QRecord lease = read(name);
+            if(lease == null || !nodeId.equals(lease.getValueString(HOLDER_NODE_ID)))
+            {
+               return (false);
+            }
+            Instant expiresAt = lease.getValueInstant(EXPIRES_AT);
+            return (expiresAt != null && expiresAt.isAfter(now()));
+         }));
+      }
+
+
+
+      /*******************************************************************************
+       ** The CAS: the row must still have the version we read, and pass the guard.
+       *******************************************************************************/
+      private boolean compareAndSet(QRecord lease, QQueryFilter guard, String holder, Instant expiresAt, Instant acquiredAt, Instant now) throws QException
+      {
+         Integer version = lease.getValueInteger(LEASE_VERSION) == null ? 0 : lease.getValueInteger(LEASE_VERSION);
+         QQueryFilter filter = new QQueryFilter()
+            .withCriteria(new QFilterCriteria(NAME, QCriteriaOperator.EQUALS, lease.getValueString(NAME)))
+            .withCriteria(lease.getValueInteger(LEASE_VERSION) == null
+               ? new QFilterCriteria(LEASE_VERSION, QCriteriaOperator.IS_BLANK)
+               : new QFilterCriteria(LEASE_VERSION, QCriteriaOperator.EQUALS, version))
+            .withSubFilter(guard);
+         int count = new ConditionalUpdateAction().execute(new ConditionalUpdateInput(LEASE)
+            .withFilter(filter)
+            .withValue(HOLDER_NODE_ID, holder)
+            .withValue(EXPIRES_AT, expiresAt)
+            .withValue(ACQUIRED_AT, acquiredAt)
+            .withValue(LEASE_VERSION, version + 1)
+            .withValue(MODIFY_DATE, now));
+         return (count == 1);
+      }
+
+
+
+      /*******************************************************************************
+       ** First holder: insert the row. A concurrent insert by another node fails
+       ** on the unique name; then whoever's row is there decides.
+       *******************************************************************************/
+      private boolean insert(String name, Instant now, Duration ttl) throws QException
+      {
+         try
+         {
+            QRecord inserted = new InsertAction().execute(new InsertInput(LEASE).withRecord(new QRecord()
+               .withValue(NAME, name)
+               .withValue(HOLDER_NODE_ID, nodeId)
+               .withValue(EXPIRES_AT, now.plus(ttl))
+               .withValue(ACQUIRED_AT, now)
+               .withValue(LEASE_VERSION, 1))).getRecords().get(0);
+            if(CollectionUtils.nullSafeIsEmpty(inserted.getErrors()))
+            {
+               LOG.info("Acquired cluster lease", logPair("lease", name), logPair("nodeId", nodeId));
+               return (true);
+            }
+         }
+         catch(QException e)
+         {
+            LOG.debug("Lease insert lost a race; reading the winner", logPair("lease", name));
+         }
+         QRecord lease = read(name);
+         return (lease != null && nodeId.equals(lease.getValueString(HOLDER_NODE_ID)));
+      }
+
+
+
+      /*******************************************************************************
+       ** The lease row, or null.
+       *******************************************************************************/
+      private QRecord read(String name) throws QException
+      {
+         List<QRecord> records = new QueryAction().execute(new QueryInput(LEASE)
+            .withFilter(new QQueryFilter(new QFilterCriteria(NAME, QCriteriaOperator.EQUALS, (Serializable) name)))).getRecords();
+         return (records.isEmpty() ? null : records.get(0));
+      }
+
+
+
+      /*******************************************************************************
+       ** The backend clock.
+       *******************************************************************************/
+      private Instant now() throws QException
+      {
+         return (new ConditionalUpdateAction().databaseNow(ClusterTables.LEASE));
+      }
+
+
+
+      /*******************************************************************************
+       ** A lease must last a positive time.
+       *******************************************************************************/
+      private static void requireTtl(Duration ttl)
+      {
+         if(ttl == null || ttl.isNegative() || ttl.isZero())
+         {
+            throw (new IllegalArgumentException("A lease ttl must be positive"));
+         }
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/ClusterQBitConfig.java

@@ -0,0 +1,386 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.util.List;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** Host configuration of the Cluster qbit.
+ **
+ ** Table names are fixed (cluster_node, cluster_lease, cluster_setting): the
+ ** QRun qbit standard names a qbit's tables with the qbit's own prefix, and the
+ ** shipped changelog creates exactly those. The app's place in the navigation
+ ** is the host's choice: set parentAppName (for example "admin"), or attach the
+ ** app yourself.
+ *******************************************************************************/
+public class ClusterQBitConfig implements QBitConfig
+{
+   private String  backendName;
+   private String  appName                = "cluster";
+   private String  appLabel               = "Cluster";
+   private String  parentAppName;
+   private String  permissionBaseName     = "cluster";
+   private String  drainPermissionName    = "cluster.drain";
+   private String  appVersion;
+   private Integer widgetRefreshSeconds   = 15;
+   private Boolean registerRuntimeService = true;
+
+
+
+   /*******************************************************************************
+    ** Report invalid settings before adding any metadata.
+    *******************************************************************************/
+   @Override
+   public void validate(QInstance qInstance, List<String> errors)
+   {
+      assertCondition(StringUtils.hasContent(backendName), "backendName is required for the Cluster QBit", errors);
+      assertCondition(backendName == null || qInstance.getBackend(backendName) != null, "backendName must name a backend in the instance", errors);
+      assertCondition(appName != null && appName.matches("[A-Za-z][A-Za-z0-9_]*"), "appName must be a valid name", errors);
+      assertCondition(parentAppName == null || (!parentAppName.equals(appName) && qInstance.getApp(parentAppName) != null), "parentAppName must name an existing, different app", errors);
+      assertCondition(StringUtils.hasContent(permissionBaseName), "permissionBaseName is required", errors);
+      assertCondition(StringUtils.hasContent(drainPermissionName), "drainPermissionName is required", errors);
+      assertCondition(widgetRefreshSeconds != null && widgetRefreshSeconds > 0, "widgetRefreshSeconds must be positive", errors);
+   }
+
+
+
+   /*******************************************************************************
+    ** Tables without their own backend use the configured backend.
+    *******************************************************************************/
+   @Override
+   public String getDefaultBackendNameForTables()
+   {
+      return (backendName);
+   }
+
+
+
+   /*******************************************************************************
+    ** The permission that reads the cluster tables, app and widget.
+    *******************************************************************************/
+   public String getReadPermissionName()
+   {
+      return (permissionBaseName + ".read");
+   }
+
+
+
+   /*******************************************************************************
+    ** The permission that edits the cluster tables.
+    *******************************************************************************/
+   public String getWritePermissionName()
+   {
+      return (permissionBaseName + ".write");
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for backendName
+    *******************************************************************************/
+   public String getBackendName()
+   {
+      return (this.backendName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for backendName
+    *******************************************************************************/
+   public void setBackendName(String backendName)
+   {
+      this.backendName = backendName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for backendName: the backend holding the cluster_ tables
+    ** (required). Lease compare-and-set needs its CONDITIONAL_UPDATE capability
+    ** (the RDBMS and memory backends have it).
+    *******************************************************************************/
+   public ClusterQBitConfig withBackendName(String backendName)
+   {
+      this.backendName = backendName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for appName
+    *******************************************************************************/
+   public String getAppName()
+   {
+      return (this.appName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for appName
+    *******************************************************************************/
+   public void setAppName(String appName)
+   {
+      this.appName = appName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for appName (default "cluster").
+    *******************************************************************************/
+   public ClusterQBitConfig withAppName(String appName)
+   {
+      this.appName = appName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for appLabel
+    *******************************************************************************/
+   public String getAppLabel()
+   {
+      return (this.appLabel);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for appLabel
+    *******************************************************************************/
+   public void setAppLabel(String appLabel)
+   {
+      this.appLabel = appLabel;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for appLabel (default "Cluster").
+    *******************************************************************************/
+   public ClusterQBitConfig withAppLabel(String appLabel)
+   {
+      this.appLabel = appLabel;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for parentAppName
+    *******************************************************************************/
+   public String getParentAppName()
+   {
+      return (this.parentAppName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for parentAppName
+    *******************************************************************************/
+   public void setParentAppName(String parentAppName)
+   {
+      this.parentAppName = parentAppName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for parentAppName: an existing app to attach the Cluster app
+    ** under (for example "admin"). Null (the default) leaves placement to the host.
+    *******************************************************************************/
+   public ClusterQBitConfig withParentAppName(String parentAppName)
+   {
+      this.parentAppName = parentAppName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for permissionBaseName
+    *******************************************************************************/
+   public String getPermissionBaseName()
+   {
+      return (this.permissionBaseName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for permissionBaseName
+    *******************************************************************************/
+   public void setPermissionBaseName(String permissionBaseName)
+   {
+      this.permissionBaseName = permissionBaseName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for permissionBaseName (default "cluster": the tables, app
+    ** and widget use cluster.read and cluster.write).
+    *******************************************************************************/
+   public ClusterQBitConfig withPermissionBaseName(String permissionBaseName)
+   {
+      this.permissionBaseName = permissionBaseName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for drainPermissionName
+    *******************************************************************************/
+   public String getDrainPermissionName()
+   {
+      return (this.drainPermissionName);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for drainPermissionName
+    *******************************************************************************/
+   public void setDrainPermissionName(String drainPermissionName)
+   {
+      this.drainPermissionName = drainPermissionName;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for drainPermissionName: the exact permission that runs Drain
+    ** node and Cancel drain (default "cluster.drain").
+    *******************************************************************************/
+   public ClusterQBitConfig withDrainPermissionName(String drainPermissionName)
+   {
+      this.drainPermissionName = drainPermissionName;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for appVersion
+    *******************************************************************************/
+   public String getAppVersion()
+   {
+      return (this.appVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for appVersion
+    *******************************************************************************/
+   public void setAppVersion(String appVersion)
+   {
+      this.appVersion = appVersion;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for appVersion: the application version each node reports.
+    ** Without it, the environment variable QQQ_APP_VERSION, else the QQQ version.
+    *******************************************************************************/
+   public ClusterQBitConfig withAppVersion(String appVersion)
+   {
+      this.appVersion = appVersion;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for widgetRefreshSeconds
+    *******************************************************************************/
+   public Integer getWidgetRefreshSeconds()
+   {
+      return (this.widgetRefreshSeconds);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for widgetRefreshSeconds
+    *******************************************************************************/
+   public void setWidgetRefreshSeconds(Integer widgetRefreshSeconds)
+   {
+      this.widgetRefreshSeconds = widgetRefreshSeconds;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for widgetRefreshSeconds (the Nodes widget, default 15).
+    *******************************************************************************/
+   public ClusterQBitConfig withWidgetRefreshSeconds(Integer widgetRefreshSeconds)
+   {
+      this.widgetRefreshSeconds = widgetRefreshSeconds;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for registerRuntimeService
+    *******************************************************************************/
+   public Boolean getRegisterRuntimeService()
+   {
+      return (this.registerRuntimeService);
+   }
+
+
+
+   /*******************************************************************************
+    ** Setter for registerRuntimeService
+    *******************************************************************************/
+   public void setRegisterRuntimeService(Boolean registerRuntimeService)
+   {
+      this.registerRuntimeService = registerRuntimeService;
+   }
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for registerRuntimeService: add ClusterRuntimeService to the
+    ** instance's runtime services (default true), so QApplicationLauncher starts
+    ** the heartbeat. A host that starts services itself starts it directly.
+    *******************************************************************************/
+   public ClusterQBitConfig withRegisterRuntimeService(Boolean registerRuntimeService)
+   {
+      this.registerRuntimeService = registerRuntimeService;
+      return (this);
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/ClusterQBitProducer.java

@@ -0,0 +1,249 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import com.kingsrook.qbits.cluster.health.ClusterHeartbeatHealthCheck;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.processes.ClusterNodeActions;
+import com.kingsrook.qbits.cluster.runtime.ClusterAlerts;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntimeService;
+import com.kingsrook.qbits.cluster.widgets.ClusterNodesWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.health.HealthCategory;
+import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
+import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
+import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+
+
+/*******************************************************************************
+ ** Produces the Cluster QBit into an instance:
+ ** - tables cluster_node, cluster_lease, cluster_setting and the node status
+ **   possible values;
+ ** - record actions Drain node and Cancel drain on cluster_node;
+ ** - the Nodes widget and the Cluster app (its home), attached under
+ **   parentAppName when the config names one;
+ ** - the cluster.node.lost alert type, the cluster.node.heartbeat health check
+ **   and the help bundle (help/qbit-cluster.yaml);
+ ** - ClusterRuntimeService among the instance's runtime services.
+ **
+ ** <pre>
+ ** new ClusterQBitProducer()
+ **    .withQBitConfig(new ClusterQBitConfig().withBackendName("rdbms").withParentAppName("admin"))
+ **    .produce(qInstance);
+ ** </pre>
+ *******************************************************************************/
+public class ClusterQBitProducer implements QBitMetaDataProducer<ClusterQBitConfig>
+{
+   public static final String GROUP_ID    = "com.kingsrook.qbits";
+   public static final String ARTIFACT_ID = "qbit-cluster";
+   public static final String NAMESPACE   = "cluster";
+   public static final String HELP_BUNDLE = "help/qbit-cluster.yaml";
+
+   private ClusterQBitConfig qBitConfig;
+
+
+
+   /*******************************************************************************
+    ** Fluent setter for the config.
+    *******************************************************************************/
+   public ClusterQBitProducer withQBitConfig(ClusterQBitConfig config)
+   {
+      this.qBitConfig = config;
+      return (this);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for the config.
+    *******************************************************************************/
+   @Override
+   public ClusterQBitConfig getQBitConfig()
+   {
+      return (qBitConfig);
+   }
+
+
+
+   /*******************************************************************************
+    ** Stable namespace, independent of the host's navigation label.
+    *******************************************************************************/
+   @Override
+   public String getNamespace()
+   {
+      return (NAMESPACE);
+   }
+
+
+
+   /*******************************************************************************
+    ** The qbit's identity, with the Maven-filtered version.
+    *******************************************************************************/
+   @Override
+   public QBitMetaData getQBitMetaData()
+   {
+      return (new QBitMetaData().withGroupId(GROUP_ID).withArtifactId(ARTIFACT_ID).withVersion(version()).withNamespace(NAMESPACE).withConfig(qBitConfig));
+   }
+
+
+
+   /*******************************************************************************
+    ** The packaged version.
+    *******************************************************************************/
+   public static String version()
+   {
+      try(InputStream stream = ClusterQBitProducer.class.getResourceAsStream("/qbit-cluster.properties"))
+      {
+         Properties properties = new Properties();
+         if(stream != null)
+         {
+            properties.load(stream);
+         }
+         return (properties.getProperty("version", "unknown"));
+      }
+      catch(IOException e)
+      {
+         throw (new IllegalStateException("Cannot read the qbit-cluster version resource", e));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Validate, then add everything to the instance. Returns empty output, as
+    ** everything is already added.
+    *******************************************************************************/
+   @Override
+   public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException
+   {
+      if(qBitConfig == null)
+      {
+         throw (new QException("ClusterQBitConfig is required"));
+      }
+      List<String> errors = new ArrayList<>();
+      qBitConfig.validate(qInstance, errors);
+      if(!errors.isEmpty())
+      {
+         throw (new QBitConfigValidationException(qBitConfig, errors));
+      }
+
+      QBitMetaData qBit = getQBitMetaData();
+      qInstance.addQBit(qBit);
+
+      for(QPossibleValueSource possibleValueSource : ClusterTables.possibleValueSources())
+      {
+         qInstance.addPossibleValueSource(possibleValueSource);
+      }
+      List<QTableMetaData> tables = ClusterTables.tables(qBitConfig);
+      for(QTableMetaData table : tables)
+      {
+         table.setSourceQBitName(qBit.getName());
+         qInstance.addTable(table);
+      }
+      for(QProcessMetaData process : ClusterNodeActions.processes(qBitConfig))
+      {
+         process.setSourceQBitName(qBit.getName());
+         qInstance.addProcess(process);
+      }
+
+      qInstance.addWidget(new QWidgetMetaData()
+         .withName(ClusterNodesWidgetRenderer.NAME)
+         .withLabel("Nodes")
+         .withType(WidgetType.TABLE.getType())
+         .withGridColumns(12)
+         .withPermissionRules(readRules())
+         .withCodeReference(new QCodeReference(ClusterNodesWidgetRenderer.class))
+         .withDefaultValue("refreshSeconds", qBitConfig.getWidgetRefreshSeconds()));
+
+      QAppMetaData app = new QAppMetaData()
+         .withName(qBitConfig.getAppName())
+         .withLabel(qBitConfig.getAppLabel())
+         .withIcon(new QIcon("hub"))
+         .withPermissionRules(readRules())
+         .withWidgets(new ArrayList<>(List.of(ClusterNodesWidgetRenderer.NAME)));
+      tables.forEach(app::withChild);
+      qInstance.addApp(app);
+      if(qBitConfig.getParentAppName() != null)
+      {
+         QAppMetaData parent = qInstance.getApp(qBitConfig.getParentAppName());
+         if(CollectionUtils.nonNullList(parent.getChildren()).stream().noneMatch(child -> app.getName().equals(child.getName())))
+         {
+            parent.withChild(app);
+         }
+      }
+
+      ClusterAlerts.register(qInstance, qBit.getName());
+
+      HealthChecksMetaData.ofOrWithNew(qInstance).withCheck(new HealthCheckMetaData()
+         .withName(ClusterHeartbeatHealthCheck.NAME)
+         .withLabel("Cluster heartbeat")
+         .withComponent(NAMESPACE)
+         .withKind(HealthCheckKind.DEPENDENCY)
+         .withCategory(HealthCategory.CORE)
+         .withIntervalSeconds(60)
+         .withCheck(new QCodeReference(ClusterHeartbeatHealthCheck.class)));
+
+      if(Boolean.TRUE.equals(qBitConfig.getRegisterRuntimeService())
+         && CollectionUtils.nonNullList(qInstance.getRuntimeServices()).stream().noneMatch(reference -> ClusterRuntimeService.class.getName().equals(reference.getName())))
+      {
+         qInstance.withRuntimeService(new QCodeReference(ClusterRuntimeService.class));
+      }
+
+      HelpBundles.register(qInstance, HELP_BUNDLE);
+      return (new MetaDataProducerMultiOutput());
+   }
+
+
+
+   /*******************************************************************************
+    ** The app and widget need the exact read permission (cluster.read).
+    *******************************************************************************/
+   private QPermissionRules readRules()
+   {
+      return (new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
+         .withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(qBitConfig.getReadPermissionName())));
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/health/ClusterHeartbeatHealthCheck.java

@@ -0,0 +1,112 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.health;
+
+
+import java.time.Duration;
+import java.time.Instant;
+import com.kingsrook.qbits.cluster.model.ClusterSettings;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntime;
+import com.kingsrook.qbits.cluster.runtime.ClusterSystem;
+import com.kingsrook.qqq.backend.core.actions.tables.ConditionalUpdateAction;
+import com.kingsrook.qqq.backend.core.health.HealthCheck;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.health.HealthDetails;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** cluster.node.heartbeat: this node's own heartbeat is current.
+ ** - UP: the last heartbeat is within two heartbeat intervals;
+ ** - DEGRADED: older than that, but not yet lost_after_seconds (writes are slow
+ **   or failing);
+ ** - DOWN: older than lost_after_seconds (other nodes will mark this one LOST),
+ **   or the row is missing or cannot be read;
+ ** - UNKNOWN: the cluster runtime is not running in this JVM.
+ *******************************************************************************/
+public class ClusterHeartbeatHealthCheck implements HealthCheck
+{
+   public static final String NAME = "cluster.node.heartbeat";
+
+
+
+   /*******************************************************************************
+    ** Check this node's row.
+    *******************************************************************************/
+   @Override
+   public HealthCheckOutcome run(HealthCheckInput input)
+   {
+      ClusterRuntime runtime = ClusterRuntime.current();
+      if(runtime == null)
+      {
+         return (HealthCheckOutcome.unknown("The cluster runtime is not running on this node"));
+      }
+
+      QInstance qInstance = input.getQInstance() == null ? runtime.getQInstance() : input.getQInstance();
+      try
+      {
+         return (ClusterSystem.call(qInstance, () ->
+         {
+            QRecord node = runtime.getRegistry().read();
+            if(node == null)
+            {
+               return (HealthCheckOutcome.down("This node is not registered in the cluster"));
+            }
+            Instant heartbeatAt = node.getValueInstant(ClusterTables.HEARTBEAT_AT);
+            if(heartbeatAt == null)
+            {
+               return (HealthCheckOutcome.down("This node has no heartbeat"));
+            }
+
+            ClusterSettings settings = runtime.getSettings();
+            long            age      = Math.max(0, Duration.between(heartbeatAt, new ConditionalUpdateAction().databaseNow(ClusterTables.NODE)).toSeconds());
+            HealthDetails details = new HealthDetails()
+               .withValue("nodeId", runtime.getRegistry().getNodeId())
+               .withValue("status", node.getValueString(ClusterTables.STATUS))
+               .withValue("heartbeatAgeSeconds", age)
+               .withValue("heartbeatSeconds", settings.heartbeatSeconds());
+
+            HealthCheckOutcome outcome;
+            if(age <= 2L * settings.heartbeatSeconds())
+            {
+               outcome = HealthCheckOutcome.up("Heartbeat is current");
+            }
+            else if(age < settings.lostAfterSeconds())
+            {
+               outcome = HealthCheckOutcome.degraded("Heartbeat is late (" + age + " seconds old)");
+            }
+            else
+            {
+               outcome = HealthCheckOutcome.down("Heartbeat is " + age + " seconds old; other nodes treat this node as lost");
+            }
+            return (outcome.withDetails(details));
+         }));
+      }
+      catch(Exception e)
+      {
+         return (HealthCheckOutcome.down("This node's heartbeat could not be read"));
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/model/ClusterSettingValidator.java

@@ -0,0 +1,67 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.model;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
+import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
+
+
+/*******************************************************************************
+ ** Rejects a cluster_setting value that the runtime could not use (for example
+ ** heartbeat_seconds = "soon"), so a typo is caught on save rather than being
+ ** silently replaced by the default. Unknown keys are allowed.
+ *******************************************************************************/
+public class ClusterSettingValidator implements TableCustomizerInterface
+{
+
+   /*******************************************************************************
+    ** Validate inserts and updates (an update may carry only the value; the key
+    ** then comes from the stored row).
+    *******************************************************************************/
+   @Override
+   public List<QRecord> preInsertOrUpdate(AbstractActionInput input, List<QRecord> records, boolean isPreview, Optional<List<QRecord>> oldRecordList)
+   {
+      Map<Serializable, QRecord> oldById = new HashMap<>();
+      oldRecordList.ifPresent(list -> list.forEach(old -> oldById.put(old.getValue(ClusterTables.ID), old)));
+
+      for(QRecord record : records)
+      {
+         QRecord old   = oldById.get(record.getValue(ClusterTables.ID));
+         String  key   = record.getValues().containsKey(ClusterTables.SETTING_KEY) ? record.getValueString(ClusterTables.SETTING_KEY) : old == null ? null : old.getValueString(ClusterTables.SETTING_KEY);
+         String  value = record.getValues().containsKey(ClusterTables.SETTING_VALUE) ? record.getValueString(ClusterTables.SETTING_VALUE) : old == null ? null : old.getValueString(ClusterTables.SETTING_VALUE);
+         String  error = ClusterSettings.validate(key, value);
+         if(error != null)
+         {
+            record.addError(new BadInputStatusMessage(error));
+         }
+      }
+      return (records);
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/model/ClusterSettings.java

@@ -0,0 +1,195 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.model;
+
+
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The cluster's settings (cluster_setting rows), with the defaults used when a
+ ** row is missing or its value is not valid. The defaults are the same values
+ ** the shipped defaults changelog seeds.
+ **
+ ** @param heartbeatSeconds how often each node writes its heartbeat
+ ** @param lostAfterSeconds a node whose heartbeat is older than this is LOST
+ ** @param retainStoppedDays STOPPED and LOST rows older than this are deleted
+ ** @param alertOnLost whether a lost node raises the cluster.node.lost alert
+ *******************************************************************************/
+public record ClusterSettings(int heartbeatSeconds, int lostAfterSeconds, int retainStoppedDays, boolean alertOnLost)
+{
+   public static final String HEARTBEAT_SECONDS   = "heartbeat_seconds";
+   public static final String LOST_AFTER_SECONDS  = "lost_after_seconds";
+   public static final String RETAIN_STOPPED_DAYS = "retain_stopped_days";
+   public static final String ALERT_ON_LOST       = "alert_on_lost";
+
+   /** each known key, with its default value and its description (as seeded). */
+   public static final Map<String, List<String>> DEFAULTS = defaults();
+
+   private static final QLogger LOG = QLogger.getLogger(ClusterSettings.class);
+
+
+
+   /*******************************************************************************
+    ** The built-in defaults.
+    *******************************************************************************/
+   public static ClusterSettings defaultSettings()
+   {
+      return (new ClusterSettings(15, 60, 7, true));
+   }
+
+
+
+   /*******************************************************************************
+    ** Read the settings (the caller supplies a session that may read the table).
+    ** A missing or invalid row keeps its default.
+    *******************************************************************************/
+   public static ClusterSettings load() throws QException
+   {
+      Map<String, String> values = new LinkedHashMap<>();
+      for(QRecord record : new QueryAction().execute(new QueryInput(ClusterTables.SETTING)).getRecords())
+      {
+         values.put(record.getValueString(ClusterTables.SETTING_KEY), record.getValueString(ClusterTables.SETTING_VALUE));
+      }
+      return (from(values));
+   }
+
+
+
+   /*******************************************************************************
+    ** Settings from key/value pairs.
+    *******************************************************************************/
+   public static ClusterSettings from(Map<String, String> values)
+   {
+      ClusterSettings defaults  = defaultSettings();
+      int             heartbeat = positive(values, HEARTBEAT_SECONDS, defaults.heartbeatSeconds());
+      int             lostAfter = positive(values, LOST_AFTER_SECONDS, defaults.lostAfterSeconds());
+      if(lostAfter <= heartbeat)
+      {
+         LOG.warn("Cluster setting lost_after_seconds must be greater than heartbeat_seconds; using three heartbeats", logPair("heartbeatSeconds", heartbeat), logPair("lostAfterSeconds", lostAfter));
+         lostAfter = heartbeat * 3;
+      }
+      int     retain = positive(values, RETAIN_STOPPED_DAYS, defaults.retainStoppedDays());
+      Boolean alert  = parseBoolean(values.get(ALERT_ON_LOST));
+      return (new ClusterSettings(heartbeat, lostAfter, retain, alert == null ? defaults.alertOnLost() : alert));
+   }
+
+
+
+   /*******************************************************************************
+    ** Why a value is not valid for a key (null when it is, or the key is not a
+    ** known one).
+    *******************************************************************************/
+   public static String validate(String key, String value)
+   {
+      if(key == null || !DEFAULTS.containsKey(key))
+      {
+         return (null);
+      }
+      if(ALERT_ON_LOST.equals(key))
+      {
+         return (parseBoolean(value) == null ? "The value of " + key + " must be true or false." : null);
+      }
+      Integer number = parseInteger(value);
+      return (number == null || number < 1 ? "The value of " + key + " must be a whole number of at least 1." : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** A positive integer setting, or the default.
+    *******************************************************************************/
+   private static int positive(Map<String, String> values, String key, int defaultValue)
+   {
+      String value = values.get(key);
+      if(value == null)
+      {
+         return (defaultValue);
+      }
+      Integer number = parseInteger(value);
+      if(number == null || number < 1)
+      {
+         LOG.warn("Ignoring a cluster setting that is not a positive whole number", logPair("key", key), logPair("value", value));
+         return (defaultValue);
+      }
+      return (number);
+   }
+
+
+
+   /*******************************************************************************
+    ** Parse an integer, or null.
+    *******************************************************************************/
+   private static Integer parseInteger(String value)
+   {
+      try
+      {
+         return (value == null ? null : Integer.valueOf(value.trim()));
+      }
+      catch(NumberFormatException e)
+      {
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Parse true / false (any case), or null.
+    *******************************************************************************/
+   private static Boolean parseBoolean(String value)
+   {
+      if(value == null)
+      {
+         return (null);
+      }
+      String trimmed = value.trim();
+      if("true".equalsIgnoreCase(trimmed))
+      {
+         return (true);
+      }
+      return ("false".equalsIgnoreCase(trimmed) ? false : null);
+   }
+
+
+
+   /*******************************************************************************
+    ** The seeded keys, values and descriptions (in seed order).
+    *******************************************************************************/
+   private static Map<String, List<String>> defaults()
+   {
+      Map<String, List<String>> map = new LinkedHashMap<>();
+      map.put(HEARTBEAT_SECONDS, List.of("15", "How often, in seconds, each node records that it is alive."));
+      map.put(LOST_AFTER_SECONDS, List.of("60", "A node with no heartbeat for this many seconds is marked Lost."));
+      map.put(RETAIN_STOPPED_DAYS, List.of("7", "Days to keep the rows of stopped and lost nodes before they are deleted."));
+      map.put(ALERT_ON_LOST, List.of("true", "Whether a lost node raises an alert (true or false)."));
+      return (java.util.Collections.unmodifiableMap(map));
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/model/ClusterTables.java

@@ -0,0 +1,273 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.model;
+
+
+import java.util.List;
+import com.kingsrook.qbits.cluster.ClusterQBitConfig;
+import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QFieldSection;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Tier;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.UniqueKey;
+
+
+/*******************************************************************************
+ ** The three cluster_ tables. Each is an ordinary QQQ table (query, view,
+ ** export, API), secured by the read / write permissions of the configured
+ ** permission base name rather than by hiding it:
+ ** - cluster_node: one row per node, written by the nodes themselves (no insert
+ **   or edit in the UI; Drain node / Cancel drain are record actions; old rows
+ **   may be deleted);
+ ** - cluster_lease: leadership leases (ClusterLeases); an administrator may
+ **   edit a lease (for example clear its holder to force a hand-over);
+ ** - cluster_setting: the cluster's settings, seeded with defaults.
+ **
+ ** Column names are the snake_case of the field names (the shipped Liquibase
+ ** changelog db/liquibase/qbit-cluster/cluster.changelog.sql).
+ *******************************************************************************/
+public final class ClusterTables
+{
+   public static final String NODE    = "cluster_node";
+   public static final String LEASE   = "cluster_lease";
+   public static final String SETTING = "cluster_setting";
+
+   public static final List<String> ALL = List.of(NODE, LEASE, SETTING);
+
+   ///////////////////////
+   // cluster_node      //
+   ///////////////////////
+   public static final String ID                  = "id";
+   public static final String NODE_ID             = "nodeId";
+   public static final String POD_NAME            = "podName";
+   public static final String K8S_NODE_NAME       = "k8sNodeName";
+   public static final String ZONE                = "zone";
+   public static final String DEPLOYMENT          = "deployment";
+   public static final String ROLES               = "roles";
+   public static final String VERSION             = "version";
+   public static final String STARTED_AT          = "startedAt";
+   public static final String HEARTBEAT_AT        = "heartbeatAt";
+   public static final String STATUS              = "status";
+   public static final String WORKER_THREADS      = "workerThreads";
+   public static final String SCHEDULER_ACTIVE    = "schedulerActive";
+   public static final String ESB_TRIGGERS_ACTIVE = "esbTriggersActive";
+   public static final String ASYNC_JOBS_RUNNING  = "asyncJobsRunning";
+   public static final String HEAP_USED_MB        = "heapUsedMb";
+   public static final String HEAP_MAX_MB         = "heapMaxMb";
+   public static final String DRAIN_REQUESTED     = "drainRequested";
+   public static final String DRAIN_REQUESTED_BY  = "drainRequestedBy";
+   public static final String DRAIN_REASON        = "drainReason";
+   public static final String CREATE_DATE         = "createDate";
+   public static final String MODIFY_DATE         = "modifyDate";
+
+   ///////////////////////
+   // cluster_lease     //
+   ///////////////////////
+   public static final String NAME           = "name";
+   public static final String HOLDER_NODE_ID = "holderNodeId";
+   public static final String EXPIRES_AT     = "expiresAt";
+   public static final String LEASE_VERSION  = "version";
+   public static final String ACQUIRED_AT    = "acquiredAt";
+
+   ///////////////////////
+   // cluster_setting   //
+   ///////////////////////
+   public static final String SETTING_KEY   = "settingKey";
+   public static final String SETTING_VALUE = "settingValue";
+   public static final String DESCRIPTION   = "description";
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private ClusterTables()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The possible-value sources: node statuses, and nodes (for lease holders).
+    *******************************************************************************/
+   public static List<QPossibleValueSource> possibleValueSources()
+   {
+      return (List.of(QPossibleValueSource.newForEnum(NodeStatus.NAME, NodeStatus.values())));
+   }
+
+
+
+   /*******************************************************************************
+    ** Every table.
+    *******************************************************************************/
+   public static List<QTableMetaData> tables(ClusterQBitConfig config)
+   {
+      return (List.of(nodeTable(config), leaseTable(config), settingTable(config)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Read / write permissions on the configured base name (cluster.read and
+    ** cluster.write by default).
+    *******************************************************************************/
+   public static QPermissionRules tablePermissionRules(ClusterQBitConfig config)
+   {
+      return (new QPermissionRules().withLevel(PermissionLevel.READ_WRITE_PERMISSIONS).withPermissionBaseName(config.getPermissionBaseName()));
+   }
+
+
+
+   /*******************************************************************************
+    ** cluster_node
+    *******************************************************************************/
+   public static QTableMetaData nodeTable(ClusterQBitConfig config)
+   {
+      QTableMetaData table = base(config, NODE, "Nodes", "dns")
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields(NODE_ID)
+         .withUniqueKey(new UniqueKey(NODE_ID))
+         .withField(field(NODE_ID, "node_id", QFieldType.STRING).withLabel("Node").withMaxLength(250).withIsRequired(true))
+         .withField(field(POD_NAME, "pod_name", QFieldType.STRING).withLabel("Pod").withMaxLength(250))
+         .withField(field(K8S_NODE_NAME, "k8s_node_name", QFieldType.STRING).withLabel("Kubernetes Node").withMaxLength(250))
+         .withField(field(ZONE, "zone", QFieldType.STRING).withLabel("Zone").withMaxLength(100))
+         .withField(field(DEPLOYMENT, "deployment", QFieldType.STRING).withLabel("Deployment").withMaxLength(250))
+         .withField(field(ROLES, "roles", QFieldType.STRING).withLabel("Roles").withMaxLength(100))
+         .withField(field(VERSION, "version", QFieldType.STRING).withLabel("Version").withMaxLength(100))
+         .withField(field(STARTED_AT, "started_at", QFieldType.DATE_TIME).withLabel("Started"))
+         .withField(field(HEARTBEAT_AT, "heartbeat_at", QFieldType.DATE_TIME).withLabel("Last Heartbeat"))
+         .withField(field(STATUS, "status", QFieldType.STRING).withLabel("Status").withMaxLength(20).withPossibleValueSourceName(NodeStatus.NAME))
+         .withField(field(WORKER_THREADS, "worker_threads", QFieldType.INTEGER).withLabel("Job Worker Threads"))
+         .withField(field(SCHEDULER_ACTIVE, "scheduler_active", QFieldType.BOOLEAN).withLabel("Runs Schedules"))
+         .withField(field(ESB_TRIGGERS_ACTIVE, "esb_triggers_active", QFieldType.BOOLEAN).withLabel("Runs ESB Triggers"))
+         .withField(field(ASYNC_JOBS_RUNNING, "async_jobs_running", QFieldType.INTEGER).withLabel("Steps Running"))
+         .withField(field(HEAP_USED_MB, "heap_used_mb", QFieldType.INTEGER).withLabel("Memory Used (MB)").withDisplayFormat(DisplayFormat.COMMAS))
+         .withField(field(HEAP_MAX_MB, "heap_max_mb", QFieldType.INTEGER).withLabel("Memory Limit (MB)").withDisplayFormat(DisplayFormat.COMMAS))
+         .withField(field(DRAIN_REQUESTED, "drain_requested", QFieldType.BOOLEAN).withLabel("Drain Requested"))
+         .withField(field(DRAIN_REQUESTED_BY, "drain_requested_by", QFieldType.STRING).withLabel("Drain Requested By").withMaxLength(250))
+         .withField(field(DRAIN_REASON, "drain_reason", QFieldType.STRING).withLabel("Drain Reason").withMaxLength(500))
+         .withSection(new QFieldSection("identity", "Node", new QIcon("dns"), Tier.T1, List.of(ID, NODE_ID, STATUS, ROLES, HEARTBEAT_AT)))
+         .withSection(new QFieldSection("placement", "Where It Runs", new QIcon("place"), Tier.T2, List.of(POD_NAME, K8S_NODE_NAME, ZONE, DEPLOYMENT, VERSION)))
+         .withSection(new QFieldSection("work", "Work and Memory", new QIcon("memory"), Tier.T2,
+            List.of(WORKER_THREADS, SCHEDULER_ACTIVE, ESB_TRIGGERS_ACTIVE, ASYNC_JOBS_RUNNING, HEAP_USED_MB, HEAP_MAX_MB)))
+         .withSection(new QFieldSection("drain", "Drain", new QIcon("logout"), Tier.T2, List.of(DRAIN_REQUESTED, DRAIN_REQUESTED_BY, DRAIN_REASON)))
+         .withSection(dates(STARTED_AT));
+      table.getFields().values().forEach(field -> field.setIsEditable(false));
+      table.withoutCapabilities(Capability.TABLE_INSERT, Capability.TABLE_UPDATE);
+      return (table);
+   }
+
+
+
+   /*******************************************************************************
+    ** cluster_lease
+    *******************************************************************************/
+   public static QTableMetaData leaseTable(ClusterQBitConfig config)
+   {
+      QTableMetaData table = base(config, LEASE, "Leases", "key")
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields(NAME)
+         .withUniqueKey(new UniqueKey(NAME))
+         .withField(field(NAME, "name", QFieldType.STRING).withLabel("Name").withMaxLength(100).withIsRequired(true).withIsEditable(false))
+         .withField(field(HOLDER_NODE_ID, "holder_node_id", QFieldType.STRING).withLabel("Held By").withMaxLength(250))
+         .withField(field(EXPIRES_AT, "expires_at", QFieldType.DATE_TIME).withLabel("Expires"))
+         .withField(field(LEASE_VERSION, "version", QFieldType.INTEGER).withLabel("Version").withIsEditable(false))
+         .withField(field(ACQUIRED_AT, "acquired_at", QFieldType.DATE_TIME).withLabel("Held Since").withIsEditable(false))
+         .withSection(new QFieldSection("identity", "Lease", new QIcon("key"), Tier.T1, List.of(ID, NAME, HOLDER_NODE_ID, EXPIRES_AT)))
+         .withSection(dates(ACQUIRED_AT, LEASE_VERSION));
+      table.withoutCapabilities(Capability.TABLE_INSERT);
+      return (table);
+   }
+
+
+
+   /*******************************************************************************
+    ** cluster_setting
+    *******************************************************************************/
+   public static QTableMetaData settingTable(ClusterQBitConfig config)
+   {
+      QTableMetaData table = base(config, SETTING, "Settings", "tune")
+         .withRecordLabelFormat("%s")
+         .withRecordLabelFields(SETTING_KEY)
+         .withUniqueKey(new UniqueKey(SETTING_KEY))
+         .withField(field(SETTING_KEY, "setting_key", QFieldType.STRING).withLabel("Setting").withMaxLength(100).withIsRequired(true))
+         .withField(field(SETTING_VALUE, "setting_value", QFieldType.STRING).withLabel("Value").withMaxLength(250).withIsRequired(true))
+         .withField(field(DESCRIPTION, "description", QFieldType.STRING).withLabel("Description").withMaxLength(500))
+         .withSection(new QFieldSection("identity", "Setting", new QIcon("tune"), Tier.T1, List.of(ID, SETTING_KEY, SETTING_VALUE, DESCRIPTION)))
+         .withSection(dates());
+      QCodeReference validator = new QCodeReference(ClusterSettingValidator.class);
+      table.withCustomizer(TableCustomizers.PRE_INSERT_RECORD, validator);
+      table.withCustomizer(TableCustomizers.PRE_UPDATE_RECORD, validator);
+      return (table);
+   }
+
+
+
+   /*******************************************************************************
+    ** The id, create and modify date fields and the table basics.
+    *******************************************************************************/
+   private static QTableMetaData base(ClusterQBitConfig config, String name, String label, String icon)
+   {
+      return (new QTableMetaData()
+         .withName(name)
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withBackendName(config.getBackendName())
+         .withPrimaryKeyField(ID)
+         .withPermissionRules(tablePermissionRules(config))
+         .withField(field(ID, "id", QFieldType.INTEGER).withLabel("Id").withIsEditable(false))
+         .withField(field(CREATE_DATE, "create_date", QFieldType.DATE_TIME).withLabel("Create Date").withIsEditable(false))
+         .withField(field(MODIFY_DATE, "modify_date", QFieldType.DATE_TIME).withLabel("Modify Date").withIsEditable(false)));
+   }
+
+
+
+   /*******************************************************************************
+    ** A field with its column name.
+    *******************************************************************************/
+   private static QFieldMetaData field(String name, String column, QFieldType type)
+   {
+      return (new QFieldMetaData(name, type).withBackendName(column));
+   }
+
+
+
+   /*******************************************************************************
+    ** The dates section (with any extra fields first).
+    *******************************************************************************/
+   private static QFieldSection dates(String... extraFields)
+   {
+      List<String> fields = new java.util.ArrayList<>(List.of(extraFields));
+      fields.add(CREATE_DATE);
+      fields.add(MODIFY_DATE);
+      return (new QFieldSection("dates", "Dates", new QIcon("calendar_month"), Tier.T3, fields));
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/model/NodeStatus.java

@@ -0,0 +1,78 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.model;
+
+
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
+
+
+/*******************************************************************************
+ ** Where a node is in its life (cluster_node.status):
+ ** - UP: running and sending heartbeats;
+ ** - DRAINING: finishing in-flight work, not taking new requests or work;
+ ** - LOST: stopped sending heartbeats without saying goodbye (crash, kill,
+ **   network partition) - set by the sweeper;
+ ** - STOPPED: shut down cleanly.
+ *******************************************************************************/
+public enum NodeStatus implements PossibleValueEnum<String>
+{
+   UP("Up"),
+   DRAINING("Draining"),
+   LOST("Lost"),
+   STOPPED("Stopped");
+
+   public static final String NAME = "cluster_node_status";
+
+   private final String label;
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   NodeStatus(String label)
+   {
+      this.label = label;
+   }
+
+
+
+   /*******************************************************************************
+    ** Possible value id.
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueId()
+   {
+      return (name());
+   }
+
+
+
+   /*******************************************************************************
+    ** Possible value label.
+    *******************************************************************************/
+   @Override
+   public String getPossibleValueLabel()
+   {
+      return (label);
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/processes/ClusterNodeActionSteps.java

@@ -0,0 +1,522 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.processes;
+
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qbits.cluster.runtime.ClusterSystem;
+import com.kingsrook.qqq.backend.core.actions.tables.ConditionalUpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
+import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.ConditionalUpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.DRAIN_REASON;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.DRAIN_REQUESTED;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.DRAIN_REQUESTED_BY;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.ID;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.MODIFY_DATE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.NODE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.NODE_ID;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.STATUS;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The steps of the two cluster_node record actions, each a standard streamed
+ ** ETL (Review: what will happen and what is skipped and why; Result: what
+ ** happened):
+ ** - Drain node: sets the drain request (who, why) on nodes that are UP. The
+ **   node picks it up on its next heartbeat and starts draining.
+ ** - Cancel drain: clears a request the node has not acted on yet. A node that
+ **   is already draining cannot be undrained.
+ ** Both writes are compare-and-set on the node's status, so a node that starts
+ ** draining between Review and Result is reported, never overwritten.
+ *******************************************************************************/
+public final class ClusterNodeActionSteps
+{
+   public static final String FIELD_REASON = "clusterDrainReason";
+
+   private static final QLogger LOG = QLogger.getLogger(ClusterNodeActionSteps.class);
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private ClusterNodeActionSteps()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Review for Drain node.
+    *******************************************************************************/
+   public static class DrainTransform extends BaseTransform
+   {
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      public DrainTransform()
+      {
+         super("node will be asked to drain", "nodes will be asked to drain");
+      }
+
+
+
+      /*******************************************************************************
+       ** Only an UP node without a pending request can be drained.
+       *******************************************************************************/
+      @Override
+      String skipReason(QRecord node)
+      {
+         if(!NodeStatus.UP.name().equals(node.getValueString(STATUS)))
+         {
+            return ("Only a node that is up can be drained; these are draining, lost or stopped.");
+         }
+         if(Boolean.TRUE.equals(node.getValueBoolean(DRAIN_REQUESTED)))
+         {
+            return ("A drain was already requested for these nodes.");
+         }
+         return (null);
+      }
+
+
+
+      /*******************************************************************************
+       ** Warn when the drain would leave no node up.
+       *******************************************************************************/
+      @Override
+      void addWarnings(List<ProcessSummaryLineInterface> lines, int willCount)
+      {
+         try
+         {
+            Integer up = ClusterSystem.call(() -> new CountAction().execute(new CountInput(NODE).withFilter(new QQueryFilter()
+               .withCriteria(new QFilterCriteria(STATUS, QCriteriaOperator.EQUALS, NodeStatus.UP.name()))
+               .withSubFilter(notRequested()))).getCount());
+            if(up != null && willCount > 0 && up - willCount <= 0)
+            {
+               lines.add(new ProcessSummaryLine(Status.WARNING, null, "No other node is up: after this drain, no node will take new requests or work until one starts."));
+            }
+         }
+         catch(QException e)
+         {
+            LOG.debug("Could not count the nodes that are up", e);
+         }
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Result for Drain node.
+    *******************************************************************************/
+   public static class DrainLoad extends BaseLoad
+   {
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      public DrainLoad()
+      {
+         super("node will be asked to drain", "nodes will be asked to drain", "node was asked to drain", "nodes were asked to drain");
+      }
+
+
+
+      /*******************************************************************************
+       ** Set the request where the node is still UP with no request.
+       *******************************************************************************/
+      @Override
+      boolean apply(RunBackendStepInput input, QRecord node, Instant now) throws QException
+      {
+         String reason = input.getValueString(FIELD_REASON);
+         String by     = userLabel(QContext.getQSession());
+         int count = ClusterSystem.call(() -> new ConditionalUpdateAction().execute(new ConditionalUpdateInput(NODE)
+            .withFilter(new QQueryFilter()
+               .withCriteria(new QFilterCriteria(ID, QCriteriaOperator.EQUALS, node.getValue(ID)))
+               .withCriteria(new QFilterCriteria(STATUS, QCriteriaOperator.EQUALS, NodeStatus.UP.name()))
+               .withSubFilter(notRequested()))
+            .withValue(DRAIN_REQUESTED, true)
+            .withValue(DRAIN_REQUESTED_BY, by)
+            .withValue(DRAIN_REASON, StringUtils.hasContent(reason) ? truncate(reason.trim(), 500) : null)
+            .withValue(MODIFY_DATE, now)));
+         if(count == 1)
+         {
+            LOG.info("Cluster node drain requested", logPair("nodeId", node.getValueString(NODE_ID)), logPair("requestedBy", by), logPair("reason", reason));
+         }
+         return (count == 1);
+      }
+
+
+
+      /*******************************************************************************
+       ** Why a node could not be asked.
+       *******************************************************************************/
+      @Override
+      String failedMessage()
+      {
+         return ("These nodes stopped being up, or were already asked, before the drain was saved.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Review for Cancel drain.
+    *******************************************************************************/
+   public static class CancelTransform extends BaseTransform
+   {
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      public CancelTransform()
+      {
+         super("node's drain request will be cancelled", "nodes' drain requests will be cancelled");
+      }
+
+
+
+      /*******************************************************************************
+       ** Only a pending request on an UP node can be cancelled.
+       *******************************************************************************/
+      @Override
+      String skipReason(QRecord node)
+      {
+         if(!Boolean.TRUE.equals(node.getValueBoolean(DRAIN_REQUESTED)))
+         {
+            return ("No drain was requested for these nodes.");
+         }
+         if(!NodeStatus.UP.name().equals(node.getValueString(STATUS)))
+         {
+            return ("These nodes have already started draining (or stopped). Draining cannot be undone; restart the node instead.");
+         }
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Result for Cancel drain.
+    *******************************************************************************/
+   public static class CancelLoad extends BaseLoad
+   {
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      public CancelLoad()
+      {
+         super("node's drain request will be cancelled", "nodes' drain requests will be cancelled", "node's drain request was cancelled", "nodes' drain requests were cancelled");
+      }
+
+
+
+      /*******************************************************************************
+       ** Clear the request while the node is still UP.
+       *******************************************************************************/
+      @Override
+      boolean apply(RunBackendStepInput input, QRecord node, Instant now) throws QException
+      {
+         int count = ClusterSystem.call(() -> new ConditionalUpdateAction().execute(new ConditionalUpdateInput(NODE)
+            .withFilter(new QQueryFilter()
+               .withCriteria(new QFilterCriteria(ID, QCriteriaOperator.EQUALS, node.getValue(ID)))
+               .withCriteria(new QFilterCriteria(STATUS, QCriteriaOperator.EQUALS, NodeStatus.UP.name()))
+               .withCriteria(new QFilterCriteria(DRAIN_REQUESTED, QCriteriaOperator.EQUALS, true)))
+            .withValue(DRAIN_REQUESTED, false)
+            .withValue(DRAIN_REQUESTED_BY, null)
+            .withValue(DRAIN_REASON, null)
+            .withValue(MODIFY_DATE, now)));
+         if(count == 1)
+         {
+            LOG.info("Cluster node drain request cancelled", logPair("nodeId", node.getValueString(NODE_ID)), logPair("by", userLabel(QContext.getQSession())));
+         }
+         return (count == 1);
+      }
+
+
+
+      /*******************************************************************************
+       ** Why a request could not be cancelled.
+       *******************************************************************************/
+      @Override
+      String failedMessage()
+      {
+         return ("These nodes started draining before the cancel was saved; draining cannot be undone.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Review: sort the selected nodes into "will" and "skipped (why)".
+    *******************************************************************************/
+   abstract static class BaseTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
+   {
+      private final ProcessSummaryLine       will;
+      private final List<ProcessSummaryLine> skipped = new ArrayList<>();
+
+
+
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      BaseTransform(String singular, String plural)
+      {
+         will = line(Status.OK, singular, plural, singular, plural);
+      }
+
+
+
+      /*******************************************************************************
+       ** Why a node is skipped, or null.
+       *******************************************************************************/
+      abstract String skipReason(QRecord node);
+
+
+
+      /*******************************************************************************
+       ** Extra Review lines.
+       *******************************************************************************/
+      void addWarnings(List<ProcessSummaryLineInterface> lines, int willCount)
+      {
+      }
+
+
+
+      /*******************************************************************************
+       ** Sort one page.
+       *******************************************************************************/
+      @Override
+      public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output)
+      {
+         for(QRecord node : input.getRecords())
+         {
+            String reason = skipReason(node);
+            if(reason == null)
+            {
+               will.incrementCountAndAddPrimaryKey(node.getValue(ID));
+               output.addRecord(node);
+            }
+            else
+            {
+               skippedLine(reason).incrementCountAndAddPrimaryKey(node.getValue(ID));
+            }
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** The Review lines (the skipped lines repeat on the Result screen).
+       *******************************************************************************/
+      @Override
+      public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
+      {
+         ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
+         if(!isForResultScreen)
+         {
+            will.addSelfToListIfAnyCount(lines);
+            addWarnings(lines, will.getCount() == null ? 0 : will.getCount());
+         }
+         for(ProcessSummaryLine line : skipped)
+         {
+            line.addSelfToListIfAnyCount(lines);
+         }
+         return (lines);
+      }
+
+
+
+      /*******************************************************************************
+       ** The skipped line for a reason.
+       *******************************************************************************/
+      private ProcessSummaryLine skippedLine(String reason)
+      {
+         for(ProcessSummaryLine line : skipped)
+         {
+            if(reason.equals(line.getMessage()))
+            {
+               return (line);
+            }
+         }
+         ProcessSummaryLine line = new ProcessSummaryLine(Status.INFO, 0, reason);
+         line.setSingularFutureMessage("node will be skipped. " + reason);
+         line.setPluralFutureMessage("nodes will be skipped. " + reason);
+         line.setSingularPastMessage("node was skipped. " + reason);
+         line.setPluralPastMessage("nodes were skipped. " + reason);
+         line.setMessage(reason);
+         skipped.add(line);
+         return (line);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Result: apply the change to each node.
+    *******************************************************************************/
+   abstract static class BaseLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
+   {
+      private final ProcessSummaryLine done;
+      private final ProcessSummaryLine failed;
+
+
+
+      /*******************************************************************************
+       ** Constructor.
+       *******************************************************************************/
+      BaseLoad(String singularFuture, String pluralFuture, String singularPast, String pluralPast)
+      {
+         done = line(Status.OK, singularFuture, pluralFuture, singularPast, pluralPast);
+         failed = new ProcessSummaryLine(Status.ERROR, 0, null);
+      }
+
+
+
+      /*******************************************************************************
+       ** Change one node; false when its state changed first.
+       *******************************************************************************/
+      abstract boolean apply(RunBackendStepInput input, QRecord node, Instant now) throws QException;
+
+
+
+      /*******************************************************************************
+       ** The message for nodes that could not be changed.
+       *******************************************************************************/
+      abstract String failedMessage();
+
+
+
+      /*******************************************************************************
+       ** Apply one page.
+       *******************************************************************************/
+      @Override
+      public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
+      {
+         Instant now = new ConditionalUpdateAction().databaseNow(NODE);
+         for(QRecord node : input.getRecords())
+         {
+            if(apply(input, node, now))
+            {
+               done.incrementCountAndAddPrimaryKey(node.getValue(ID));
+            }
+            else
+            {
+               failed.incrementCountAndAddPrimaryKey(node.getValue(ID));
+            }
+         }
+      }
+
+
+
+      /*******************************************************************************
+       ** The Result lines.
+       *******************************************************************************/
+      @Override
+      public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
+      {
+         ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
+         done.addSelfToListIfAnyCount(lines);
+         failed.setSingularPastMessage("node was not changed. " + failedMessage());
+         failed.setPluralPastMessage("nodes were not changed. " + failedMessage());
+         failed.addSelfToListIfAnyCount(lines);
+         if(getTransformStep() instanceof ProcessSummaryProviderInterface transform)
+         {
+            lines.addAll(transform.getProcessSummary(output, true));
+         }
+         return (lines);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** drain_requested is false or empty.
+    *******************************************************************************/
+   static QQueryFilter notRequested()
+   {
+      return (new QQueryFilter()
+         .withBooleanOperator(QQueryFilter.BooleanOperator.OR)
+         .withCriteria(new QFilterCriteria(DRAIN_REQUESTED, QCriteriaOperator.EQUALS, false))
+         .withCriteria(new QFilterCriteria(DRAIN_REQUESTED, QCriteriaOperator.IS_BLANK)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Who the session is, for drain_requested_by.
+    *******************************************************************************/
+   static String userLabel(QSession session)
+   {
+      if(session == null || session.getUser() == null)
+      {
+         return ("an administrator");
+      }
+      String label = StringUtils.hasContent(session.getUser().getFullName()) ? session.getUser().getFullName() : session.getUser().getIdReference();
+      return (StringUtils.hasContent(label) ? truncate(label, 250) : "an administrator");
+   }
+
+
+
+   /*******************************************************************************
+    ** A counted summary line.
+    *******************************************************************************/
+   static ProcessSummaryLine line(Status status, String singularFuture, String pluralFuture, String singularPast, String pluralPast)
+   {
+      ProcessSummaryLine line = new ProcessSummaryLine(status, 0, null);
+      line.setSingularFutureMessage(singularFuture);
+      line.setPluralFutureMessage(pluralFuture);
+      line.setSingularPastMessage(singularPast);
+      line.setPluralPastMessage(pluralPast);
+      return (line);
+   }
+
+
+
+   /*******************************************************************************
+    ** Cut text to a column's length.
+    *******************************************************************************/
+   static String truncate(String value, int max)
+   {
+      return (value.length() <= max ? value : value.substring(0, max));
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/processes/ClusterNodeActions.java

@@ -0,0 +1,111 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.processes;
+
+
+import java.util.List;
+import com.kingsrook.qbits.cluster.ClusterQBitConfig;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qqq.backend.core.actions.permissions.UseOtherPermissionNameCustomPermissionChecker;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QComponentType;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendComponentMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QFrontendStepMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+
+
+/*******************************************************************************
+ ** The cluster_node record actions (never in the navigation): Drain node and
+ ** Cancel drain. Each is a standard streamed ETL process on cluster_node with
+ ** selected records in, a Review screen (the confirmation) and a Result
+ ** screen. Both need the configured drain permission (cluster.drain).
+ *******************************************************************************/
+public final class ClusterNodeActions
+{
+   public static final String DRAIN_NODE   = "clusterDrainNode";
+   public static final String CANCEL_DRAIN = "clusterCancelDrain";
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private ClusterNodeActions()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Both processes.
+    *******************************************************************************/
+   public static List<QProcessMetaData> processes(ClusterQBitConfig config)
+   {
+      QProcessMetaData drain = action(config, DRAIN_NODE, "Drain node", "logout", ClusterNodeActionSteps.DrainTransform.class, ClusterNodeActionSteps.DrainLoad.class,
+         "This node will stop taking new requests and work, finish what it is doing, and keep running until it is stopped or restarted.");
+      drain.withStep(0, new QFrontendStepMetaData()
+         .withName("input")
+         .withLabel("Drain node")
+         .withComponent(new QFrontendComponentMetaData().withType(QComponentType.EDIT_FORM))
+         .withFormFields(List.of(new QFieldMetaData(ClusterNodeActionSteps.FIELD_REASON, QFieldType.STRING).withLabel("Reason").withMaxLength(500))));
+
+      QProcessMetaData cancel = action(config, CANCEL_DRAIN, "Cancel drain", "login", ClusterNodeActionSteps.CancelTransform.class, ClusterNodeActionSteps.CancelLoad.class,
+         "The drain request will be withdrawn before the node acts on it.");
+      return (List.of(drain, cancel));
+   }
+
+
+
+   /*******************************************************************************
+    ** One record action.
+    *******************************************************************************/
+   private static QProcessMetaData action(ClusterQBitConfig config, String name, String label, String icon, Class<? extends AbstractTransformStep> transform,
+      Class<? extends AbstractLoadStep> load, String previewMessage)
+   {
+      QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
+         .withName(name)
+         .withLabel(label)
+         .withIcon(new QIcon(icon))
+         .withTableName(ClusterTables.NODE)
+         .withSourceTable(ClusterTables.NODE)
+         .withDestinationTable(ClusterTables.NODE)
+         .withExtractStepClass(ExtractViaQueryStep.class)
+         .withTransformStepClass(transform)
+         .withLoadStepClass(load)
+         .withSupportsFullValidation(true)
+         .withDoFullValidation(true)
+         .withMinInputRecords(1)
+         .withPreviewMessage(previewMessage)
+         .getProcessMetaData();
+      process.setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION)
+         .withCustomPermissionChecker(UseOtherPermissionNameCustomPermissionChecker.build(config.getDrainPermissionName())));
+      return (process);
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/runtime/ClusterAlerts.java

@@ -0,0 +1,101 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.runtime;
+
+
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef;
+import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
+import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
+import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
+import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
+import com.kingsrook.qqq.backend.core.alerts.QAlerts;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+
+
+/*******************************************************************************
+ ** The cluster.node.lost alert, through core's alerts SPI (QAlerts). With
+ ** qbit-alerts installed it becomes an Alerts record and notifications;
+ ** without it, core logs it. Raising never fails the sweep (QAlerts catches).
+ *******************************************************************************/
+public final class ClusterAlerts
+{
+   public static final String NODE_LOST = "cluster.node.lost";
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private ClusterAlerts()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Declare the alert type (once per instance).
+    *******************************************************************************/
+   public static void register(QInstance qInstance, String qBitName)
+   {
+      if(QAlertTypeMetaData.of(qInstance, NODE_LOST) != null)
+      {
+         return;
+      }
+      AlertsInstanceMetaData.ofOrWithNew(qInstance).withAlertType(new QAlertTypeMetaData()
+         .withName(NODE_LOST)
+         .withLabel("Cluster node lost")
+         .withDefaultSeverity(AlertSeverity.WARNING)
+         .withSubjectTableName(ClusterTables.NODE)
+         .withSourceQBitName(qBitName)
+         .withDescription("A node of the application stopped sending heartbeats without shutting down cleanly.")
+         .withHowToFix("Check why the pod or server stopped (crash, out of memory, eviction, network). The other nodes keep serving; "
+            + "a replacement usually starts on its own. The alert resolves when the node's heartbeat returns or its row is purged."));
+   }
+
+
+
+   /*******************************************************************************
+    ** Raise the alert for a node marked LOST (deduplicated by node id).
+    *******************************************************************************/
+   public static void raiseLost(QRecord node, long secondsSinceHeartbeat)
+   {
+      String nodeId = node.getValueString(ClusterTables.NODE_ID);
+      QAlerts.raise(new AlertSignal(NODE_LOST)
+         .withDedupKey(nodeId)
+         .withTitle("Node " + nodeId + " stopped sending heartbeats")
+         .withSummary("No heartbeat for " + secondsSinceHeartbeat + " seconds. Roles: " + node.getValueString(ClusterTables.ROLES) + ".")
+         .withRecordRef(new AlertRecordRef(ClusterTables.NODE, node.getValue(ClusterTables.ID)))
+         .withAttribute("nodeId", nodeId));
+   }
+
+
+
+   /*******************************************************************************
+    ** Resolve a node's alert (a no-op when none is open).
+    *******************************************************************************/
+   public static void resolveLost(String nodeId, String note)
+   {
+      QAlerts.resolve(NODE_LOST, nodeId, note);
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/runtime/ClusterNodeInfo.java

@@ -0,0 +1,201 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.runtime;
+
+
+import java.io.InputStream;
+import java.util.Map;
+import java.util.Properties;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import com.kingsrook.qbits.cluster.ClusterQBitConfig;
+import com.kingsrook.qqq.backend.core.actions.async.AsyncJobRegistry;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.ASYNC_JOBS_RUNNING;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.DEPLOYMENT;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.ESB_TRIGGERS_ACTIVE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.HEAP_MAX_MB;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.HEAP_USED_MB;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.K8S_NODE_NAME;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.NODE_ID;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.POD_NAME;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.ROLES;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.SCHEDULER_ACTIVE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.VERSION;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.WORKER_THREADS;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.ZONE;
+
+
+/*******************************************************************************
+ ** What this node reports about itself. Placement comes from the Kubernetes
+ ** Downward API environment (each optional): QQQ_POD_NAME, QQQ_K8S_NODE_NAME,
+ ** QQQ_ZONE, QQQ_DEPLOYMENT. Roles and role flags come from core QNodeRoles,
+ ** running steps from AsyncJobRegistry, memory from the JVM.
+ *******************************************************************************/
+public class ClusterNodeInfo
+{
+   public static final String ENV_POD_NAME    = "QQQ_POD_NAME";
+   public static final String ENV_K8S_NODE    = "QQQ_K8S_NODE_NAME";
+   public static final String ENV_ZONE        = "QQQ_ZONE";
+   public static final String ENV_DEPLOYMENT  = "QQQ_DEPLOYMENT";
+   public static final String ENV_APP_VERSION = "QQQ_APP_VERSION";
+
+   private static final long MB = 1024L * 1024L;
+
+   private final ClusterQBitConfig        config;
+   private final Function<String, String> environment;
+
+
+
+   /*******************************************************************************
+    ** Constructor (the process environment).
+    *******************************************************************************/
+   public ClusterNodeInfo(ClusterQBitConfig config)
+   {
+      this(config, System::getenv);
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor with an environment lookup (tests).
+    *******************************************************************************/
+   public ClusterNodeInfo(ClusterQBitConfig config, Function<String, String> environment)
+   {
+      this.config = config;
+      this.environment = environment;
+   }
+
+
+
+   /*******************************************************************************
+    ** This node's id.
+    *******************************************************************************/
+   public String nodeId()
+   {
+      return (QNodeRoles.nodeId());
+   }
+
+
+
+   /*******************************************************************************
+    ** The fixed fields, written when the node starts.
+    *******************************************************************************/
+   public QRecord identity()
+   {
+      return (new QRecord()
+         .withValue(NODE_ID, nodeId())
+         .withValue(POD_NAME, env(ENV_POD_NAME))
+         .withValue(K8S_NODE_NAME, env(ENV_K8S_NODE))
+         .withValue(ZONE, env(ENV_ZONE))
+         .withValue(DEPLOYMENT, env(ENV_DEPLOYMENT))
+         .withValue(ROLES, roles())
+         .withValue(VERSION, version()));
+   }
+
+
+
+   /*******************************************************************************
+    ** The live fields, written on every heartbeat.
+    *******************************************************************************/
+   public Map<String, java.io.Serializable> live()
+   {
+      QNodeRoles.Settings settings = QNodeRoles.get();
+      Runtime             runtime  = Runtime.getRuntime();
+      Map<String, java.io.Serializable> values = new java.util.LinkedHashMap<>();
+      values.put(WORKER_THREADS, settings.workerThreads());
+      values.put(SCHEDULER_ACTIVE, settings.scheduler());
+      values.put(ESB_TRIGGERS_ACTIVE, settings.esbTriggers());
+      values.put(ASYNC_JOBS_RUNNING, AsyncJobRegistry.runningCount());
+      values.put(HEAP_USED_MB, (int) ((runtime.totalMemory() - runtime.freeMemory()) / MB));
+      values.put(HEAP_MAX_MB, runtime.maxMemory() == Long.MAX_VALUE ? null : (int) (runtime.maxMemory() / MB));
+      return (values);
+   }
+
+
+
+   /*******************************************************************************
+    ** This node's roles as a comma list, in role order (WEB,WORKER,...).
+    *******************************************************************************/
+   public static String roles()
+   {
+      return (QNodeRoles.get().roles().stream().sorted().map(Enum::name).collect(Collectors.joining(",")));
+   }
+
+
+
+   /*******************************************************************************
+    ** The application version: the config, else QQQ_APP_VERSION, else "qqq " and
+    ** the QQQ version.
+    *******************************************************************************/
+   public String version()
+   {
+      if(StringUtils.hasContent(config.getAppVersion()))
+      {
+         return (config.getAppVersion());
+      }
+      String fromEnvironment = env(ENV_APP_VERSION);
+      if(fromEnvironment != null)
+      {
+         return (fromEnvironment);
+      }
+      String qqqVersion = qqqVersion();
+      return (qqqVersion == null ? null : "qqq " + qqqVersion);
+   }
+
+
+
+   /*******************************************************************************
+    ** The QQQ (core) version, from its Maven metadata in the jar.
+    *******************************************************************************/
+   static String qqqVersion()
+   {
+      try(InputStream stream = QInstance.class.getResourceAsStream("/META-INF/maven/com.kingsrook.qqq/qqq-backend-core/pom.properties"))
+      {
+         if(stream == null)
+         {
+            return (QInstance.class.getPackage().getImplementationVersion());
+         }
+         Properties properties = new Properties();
+         properties.load(stream);
+         return (properties.getProperty("version"));
+      }
+      catch(Exception e)
+      {
+         return (null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A trimmed environment value, or null.
+    *******************************************************************************/
+   private String env(String name)
+   {
+      String value = environment.apply(name);
+      return (StringUtils.hasContent(value) ? value.trim() : null);
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/runtime/ClusterNodeRegistry.java

@@ -0,0 +1,285 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.runtime;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qqq.backend.core.actions.tables.ConditionalUpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.cluster.QNodeLifecycle;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.ConditionalUpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.DRAIN_REASON;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.DRAIN_REQUESTED;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.DRAIN_REQUESTED_BY;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.HEARTBEAT_AT;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.MODIFY_DATE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.NODE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.NODE_ID;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.STARTED_AT;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.STATUS;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** This node's row in cluster_node: registered (upserted) at start, refreshed by
+ ** each heartbeat, and the place a drain request arrives. Writes are
+ ** compare-and-set updates keyed by node id (no audits or record events: this
+ ** is bookkeeping, every 15 seconds per node); the first registration inserts.
+ ** Every method runs as the system user in the given instance.
+ *******************************************************************************/
+public class ClusterNodeRegistry
+{
+   private static final QLogger LOG = QLogger.getLogger(ClusterNodeRegistry.class);
+
+   private final QInstance       qInstance;
+   private final ClusterNodeInfo info;
+   private final String          nodeId;
+
+   private volatile boolean stopped = false;
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   public ClusterNodeRegistry(QInstance qInstance, ClusterNodeInfo info)
+   {
+      this.qInstance = qInstance;
+      this.info = info;
+      this.nodeId = info.nodeId();
+   }
+
+
+
+   /*******************************************************************************
+    ** Register this node: insert or refresh its row with status UP (or DRAINING
+    ** if it already is), a new start time, and any old drain request cleared (a
+    ** new process does not inherit its predecessor's request).
+    *******************************************************************************/
+   public void register() throws QException
+   {
+      ClusterSystem.call(qInstance, () ->
+      {
+         Instant now      = now();
+         QRecord existing = read();
+
+         Map<String, Serializable> values = new LinkedHashMap<>(info.identity().getValues());
+         values.putAll(info.live());
+         values.put(STATUS, localStatus().name());
+         values.put(STARTED_AT, now);
+         values.put(HEARTBEAT_AT, now);
+         values.put(DRAIN_REQUESTED, false);
+         values.put(DRAIN_REQUESTED_BY, null);
+         values.put(DRAIN_REASON, null);
+
+         if(existing == null)
+         {
+            QRecord record = new QRecord();
+            record.setValues(values);
+            QRecord inserted = new InsertAction().execute(new InsertInput(NODE).withRecord(record)).getRecords().get(0);
+            if(CollectionUtils.nullSafeHasContents(inserted.getErrors()))
+            {
+               throw (new QException("Could not register cluster node " + nodeId + ": " + inserted.getErrors().get(0).getMessage()));
+            }
+         }
+         else
+         {
+            values.put(MODIFY_DATE, now);
+            update(values);
+            resolveIfWasLost(existing);
+         }
+         LOG.info("Registered cluster node", logPair("nodeId", nodeId), logPair("roles", values.get(ClusterTables.ROLES)), logPair("version", values.get(ClusterTables.VERSION)));
+         return (null);
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** One heartbeat: pick up a drain request (start draining), then write the
+    ** live fields and the heartbeat time. A missing row (deleted, or purged
+    ** while the node was paused) is registered again.
+    *******************************************************************************/
+   public void heartbeat() throws QException
+   {
+      if(stopped)
+      {
+         return;
+      }
+
+      QRecord existing = ClusterSystem.call(qInstance, this::read);
+      if(existing == null)
+      {
+         LOG.info("This node's cluster row is missing; registering again", logPair("nodeId", nodeId));
+         register();
+         return;
+      }
+
+      if(Boolean.TRUE.equals(existing.getValueBoolean(DRAIN_REQUESTED)) && !QNodeLifecycle.isDraining())
+      {
+         String by     = StringUtils.hasContent(existing.getValueString(DRAIN_REQUESTED_BY)) ? existing.getValueString(DRAIN_REQUESTED_BY) : "an administrator";
+         String reason = existing.getValueString(DRAIN_REASON);
+         LOG.info("Drain requested for this node", logPair("nodeId", nodeId), logPair("requestedBy", by), logPair("reason", reason));
+         QNodeLifecycle.startDraining("requested by " + by + (StringUtils.hasContent(reason) ? ": " + reason : ""));
+      }
+
+      ClusterSystem.call(qInstance, () ->
+      {
+         Map<String, Serializable> values = new LinkedHashMap<>(info.live());
+         values.put(HEARTBEAT_AT, now());
+         values.put(STATUS, localStatus().name());
+         update(values);
+         resolveIfWasLost(existing);
+         return (null);
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a status change of this node (DRAINING, STOPPED). STOPPED also
+    ** records the time, which retention counts from, and stops heartbeats.
+    *******************************************************************************/
+   public void markStatus(NodeStatus status) throws QException
+   {
+      if(stopped)
+      {
+         return;
+      }
+      if(status == NodeStatus.STOPPED)
+      {
+         stopped = true;
+      }
+
+      ClusterSystem.call(qInstance, () ->
+      {
+         Instant                   now    = now();
+         Map<String, Serializable> values = new LinkedHashMap<>();
+         values.put(STATUS, status.name());
+         values.put(MODIFY_DATE, now);
+         if(status == NodeStatus.STOPPED)
+         {
+            values.put(HEARTBEAT_AT, now);
+         }
+         update(values);
+         LOG.info("Cluster node status", logPair("nodeId", nodeId), logPair("status", status));
+         return (null);
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** This node's row, or null (the caller runs as the system user).
+    *******************************************************************************/
+   public QRecord read() throws QException
+   {
+      List<QRecord> records = new QueryAction().execute(new QueryInput(NODE)
+         .withFilter(new QQueryFilter(new QFilterCriteria(NODE_ID, QCriteriaOperator.EQUALS, nodeId)))).getRecords();
+      return (records.isEmpty() ? null : records.get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for nodeId
+    *******************************************************************************/
+   public String getNodeId()
+   {
+      return (nodeId);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether this node was marked STOPPED.
+    *******************************************************************************/
+   public boolean isStopped()
+   {
+      return (stopped);
+   }
+
+
+
+   /*******************************************************************************
+    ** UP, or DRAINING once core's lifecycle says so.
+    *******************************************************************************/
+   static NodeStatus localStatus()
+   {
+      return (QNodeLifecycle.isDraining() ? NodeStatus.DRAINING : NodeStatus.UP);
+   }
+
+
+
+   /*******************************************************************************
+    ** The update of this node's row.
+    *******************************************************************************/
+   private void update(Map<String, Serializable> values) throws QException
+   {
+      new ConditionalUpdateAction().execute(new ConditionalUpdateInput(NODE)
+         .withFilter(new QQueryFilter(new QFilterCriteria(NODE_ID, QCriteriaOperator.EQUALS, nodeId)))
+         .withValues(values));
+   }
+
+
+
+   /*******************************************************************************
+    ** A node marked LOST that beats again (a long pause, a partition) resolves
+    ** its alert.
+    *******************************************************************************/
+   private void resolveIfWasLost(QRecord existing)
+   {
+      if(NodeStatus.LOST.name().equals(existing.getValueString(STATUS)))
+      {
+         LOG.info("A node marked lost is sending heartbeats again", logPair("nodeId", nodeId));
+         ClusterAlerts.resolveLost(nodeId, "The node is sending heartbeats again.");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The backend clock.
+    *******************************************************************************/
+   private static Instant now() throws QException
+   {
+      return (new ConditionalUpdateAction().databaseNow(NODE));
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/runtime/ClusterRuntime.java

@@ -0,0 +1,316 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.runtime;
+
+
+import java.time.Duration;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import com.kingsrook.qbits.cluster.ClusterLeases;
+import com.kingsrook.qbits.cluster.ClusterQBitConfig;
+import com.kingsrook.qbits.cluster.model.ClusterSettings;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qqq.backend.core.cluster.QNodeLifecycle;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** This node's cluster runtime: one daemon thread that
+ ** 1. registers the node (status UP) at start;
+ ** 2. every heartbeat_seconds writes the heartbeat and live fields, and picks
+ **    up a drain request (QNodeLifecycle.startDraining);
+ ** 3. on the node holding the "cluster-sweeper" lease, runs the lost-node
+ **    sweep and purge (ClusterSweeper).
+ ** A QNodeLifecycle drain listener marks the node DRAINING (whatever started
+ ** the drain: a request, or SIGTERM); stop (the runtime service, or the JVM
+ ** shutdown hook) marks it STOPPED and releases the lease. A failed tick is
+ ** logged and retried on the next one; it never stops the thread.
+ *******************************************************************************/
+public class ClusterRuntime
+{
+   private static final QLogger LOG = QLogger.getLogger(ClusterRuntime.class);
+
+   private static volatile ClusterRuntime current;
+
+   private final QInstance           qInstance;
+   private final ClusterNodeRegistry registry;
+   private final ClusterSweeper      sweeper;
+
+   private volatile ClusterSettings          settings = ClusterSettings.defaultSettings();
+   private volatile ScheduledExecutorService executor;
+   private volatile Thread                   shutdownHook;
+   private volatile boolean                  stopped  = false;
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   public ClusterRuntime(QInstance qInstance, ClusterQBitConfig config)
+   {
+      this(qInstance, new ClusterNodeInfo(config));
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor with a node-info source (tests).
+    *******************************************************************************/
+   public ClusterRuntime(QInstance qInstance, ClusterNodeInfo info)
+   {
+      this.qInstance = qInstance;
+      this.registry = new ClusterNodeRegistry(qInstance, info);
+      this.sweeper = new ClusterSweeper(qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    ** The runtime running in this JVM, or null.
+    *******************************************************************************/
+   public static ClusterRuntime current()
+   {
+      return (current);
+   }
+
+
+
+   /*******************************************************************************
+    ** Register the node and the drain listener, without the background thread
+    ** (start() does both; tests drive tick() themselves).
+    *******************************************************************************/
+   public void initialize() throws QException
+   {
+      settings = loadSettings();
+      registry.register();
+      QNodeLifecycle.onDrain(reason ->
+      {
+         try
+         {
+            registry.markStatus(NodeStatus.DRAINING);
+            releaseLease();
+         }
+         catch(Exception e)
+         {
+            LOG.warn("Could not record that this node is draining", e, logPair("nodeId", registry.getNodeId()));
+         }
+      });
+      current = this;
+   }
+
+
+
+   /*******************************************************************************
+    ** Initialize, then heartbeat on a daemon thread, and mark the node STOPPED
+    ** if the JVM shuts down first.
+    *******************************************************************************/
+   public void start() throws QException
+   {
+      initialize();
+      executor = Executors.newSingleThreadScheduledExecutor(runnable ->
+      {
+         Thread thread = new Thread(runnable, "qqq-cluster-heartbeat");
+         thread.setDaemon(true);
+         return (thread);
+      });
+      scheduleNext();
+
+      shutdownHook = new Thread(this::stop, "qqq-cluster-shutdown");
+      Runtime.getRuntime().addShutdownHook(shutdownHook);
+      LOG.info("Cluster runtime started", logPair("nodeId", registry.getNodeId()), logPair("heartbeatSeconds", settings.heartbeatSeconds()));
+   }
+
+
+
+   /*******************************************************************************
+    ** One heartbeat, then (on the leader) one sweep.
+    *******************************************************************************/
+   public void tick()
+   {
+      if(stopped)
+      {
+         return;
+      }
+      try
+      {
+         settings = loadSettings();
+         registry.heartbeat();
+
+         if(QNodeLifecycle.isDraining())
+         {
+            releaseLease();
+            return;
+         }
+
+         ClusterLeases.Holder lease = ClusterLeases.as(registry.getNodeId());
+         boolean leader = ClusterSystem.call(qInstance, () -> lease.tryAcquire(ClusterSweeper.LEASE_NAME, Duration.ofSeconds(settings.lostAfterSeconds())));
+         if(leader)
+         {
+            sweeper.sweep(settings);
+         }
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Cluster heartbeat failed; retrying on the next one", e, logPair("nodeId", registry.getNodeId()));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop (idempotent): stop heartbeats, release the sweeper lease, mark the
+    ** node STOPPED. Never throws.
+    *******************************************************************************/
+   public void stop()
+   {
+      synchronized(this)
+      {
+         if(stopped)
+         {
+            return;
+         }
+         stopped = true;
+      }
+
+      if(executor != null)
+      {
+         executor.shutdownNow();
+      }
+      try
+      {
+         releaseLease();
+         registry.markStatus(NodeStatus.STOPPED);
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not mark this cluster node stopped", e, logPair("nodeId", registry.getNodeId()));
+      }
+      removeShutdownHook();
+      if(current == this)
+      {
+         current = null;
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for registry
+    *******************************************************************************/
+   public ClusterNodeRegistry getRegistry()
+   {
+      return (registry);
+   }
+
+
+
+   /*******************************************************************************
+    ** The settings read on the last tick.
+    *******************************************************************************/
+   public ClusterSettings getSettings()
+   {
+      return (settings);
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for qInstance
+    *******************************************************************************/
+   public QInstance getQInstance()
+   {
+      return (qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run a tick, then schedule the next at the current heartbeat interval.
+    *******************************************************************************/
+   private void scheduleNext()
+   {
+      ScheduledExecutorService service = executor;
+      if(stopped || service == null || service.isShutdown())
+      {
+         return;
+      }
+      service.schedule(() ->
+      {
+         tick();
+         scheduleNext();
+      }, settings.heartbeatSeconds(), TimeUnit.SECONDS);
+   }
+
+
+
+   /*******************************************************************************
+    ** Release the sweeper lease if this node holds it.
+    *******************************************************************************/
+   private void releaseLease() throws QException
+   {
+      ClusterLeases.Holder lease = ClusterLeases.as(registry.getNodeId());
+      ClusterSystem.call(qInstance, () -> lease.release(ClusterSweeper.LEASE_NAME));
+   }
+
+
+
+   /*******************************************************************************
+    ** The settings, or the last ones read when they cannot be read.
+    *******************************************************************************/
+   private ClusterSettings loadSettings()
+   {
+      try
+      {
+         return (ClusterSystem.call(qInstance, ClusterSettings::load));
+      }
+      catch(Exception e)
+      {
+         LOG.warn("Could not read the cluster settings; keeping the current ones", e);
+         return (settings);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Remove the JVM hook, unless the JVM is already shutting down.
+    *******************************************************************************/
+   private void removeShutdownHook()
+   {
+      Thread hook = shutdownHook;
+      if(hook != null && Thread.currentThread() != hook)
+      {
+         try
+         {
+            Runtime.getRuntime().removeShutdownHook(hook);
+         }
+         catch(IllegalStateException e)
+         {
+            LOG.debug("The JVM is shutting down; the cluster hook stays");
+         }
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/runtime/ClusterRuntimeService.java

@@ -0,0 +1,119 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.runtime;
+
+
+import com.kingsrook.qbits.cluster.ClusterQBitConfig;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QRuntimeServiceInterface;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData;
+
+
+/*******************************************************************************
+ ** Starts and stops this node's cluster runtime with the application.
+ ** QApplicationLauncher runs it from the instance's runtime services (the
+ ** producer registers it); a host that starts services itself (for example
+ ** with QApplicationJavalinServer) calls start(qInstance) after the server
+ ** starts, and stop() at shutdown.
+ *******************************************************************************/
+public class ClusterRuntimeService implements QRuntimeServiceInterface
+{
+   private static final QLogger LOG = QLogger.getLogger(ClusterRuntimeService.class);
+
+   private ClusterRuntime runtime;
+
+
+
+   /*******************************************************************************
+    ** Name of the service.
+    *******************************************************************************/
+   @Override
+   public String getName()
+   {
+      return ("cluster");
+   }
+
+
+
+   /*******************************************************************************
+    ** Register this node and start heartbeats.
+    *******************************************************************************/
+   @Override
+   public void start(QInstance qInstance) throws QException
+   {
+      ClusterQBitConfig config = findConfig(qInstance);
+      if(config == null)
+      {
+         LOG.warn("Cluster runtime service started, but no produced Cluster QBit was found in the QInstance");
+         return;
+      }
+      runtime = new ClusterRuntime(qInstance, config);
+      runtime.start();
+   }
+
+
+
+   /*******************************************************************************
+    ** Mark this node stopped.
+    *******************************************************************************/
+   @Override
+   public void stop()
+   {
+      if(runtime != null)
+      {
+         runtime.stop();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Getter for the runtime (null before start).
+    *******************************************************************************/
+   public ClusterRuntime getRuntime()
+   {
+      return (runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** The config of the instance's Cluster qbit, or null.
+    *******************************************************************************/
+   static ClusterQBitConfig findConfig(QInstance qInstance)
+   {
+      if(qInstance == null || qInstance.getQBits() == null)
+      {
+         return (null);
+      }
+      for(QBitMetaData qBit : qInstance.getQBits().values())
+      {
+         if(qBit.getConfig() instanceof ClusterQBitConfig config)
+         {
+            return (config);
+         }
+      }
+      return (null);
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/runtime/ClusterSweeper.java

@@ -0,0 +1,180 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.runtime;
+
+
+import java.io.Serializable;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import com.kingsrook.qbits.cluster.model.ClusterSettings;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qqq.backend.core.actions.tables.ConditionalUpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.logging.QLogger;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.ConditionalUpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.HEARTBEAT_AT;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.ID;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.MODIFY_DATE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.NODE;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.NODE_ID;
+import static com.kingsrook.qbits.cluster.model.ClusterTables.STATUS;
+import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
+
+
+/*******************************************************************************
+ ** The lost-node sweep, run by the one node holding the "cluster-sweeper" lease:
+ ** - a node that is UP or DRAINING with a heartbeat older than
+ **   lost_after_seconds becomes LOST (compare-and-set on its status and
+ **   heartbeat, so a node that beats in between stays UP), and raises the
+ **   cluster.node.lost alert when alert_on_lost is true;
+ ** - STOPPED and LOST rows whose last heartbeat is older than
+ **   retain_stopped_days are deleted (their alerts resolved).
+ ** Every step is idempotent: a sweep interrupted by a crash is simply redone
+ ** by the next leader, so the purge needs no other durable mechanism.
+ *******************************************************************************/
+public class ClusterSweeper
+{
+   public static final String LEASE_NAME = "cluster-sweeper";
+
+   private static final QLogger LOG = QLogger.getLogger(ClusterSweeper.class);
+
+   private static final List<String> LIVE_STATUSES  = List.of(NodeStatus.UP.name(), NodeStatus.DRAINING.name());
+   private static final List<String> ENDED_STATUSES = List.of(NodeStatus.STOPPED.name(), NodeStatus.LOST.name());
+
+   private final QInstance qInstance;
+
+
+
+   /*******************************************************************************
+    ** What one sweep did.
+    **
+    ** @param lostNodeIds nodes marked LOST
+    ** @param purged rows deleted
+    *******************************************************************************/
+   public record Result(List<String> lostNodeIds, int purged)
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Constructor.
+    *******************************************************************************/
+   public ClusterSweeper(QInstance qInstance)
+   {
+      this.qInstance = qInstance;
+   }
+
+
+
+   /*******************************************************************************
+    ** One sweep with the given settings (the caller holds the lease).
+    *******************************************************************************/
+   public Result sweep(ClusterSettings settings) throws QException
+   {
+      return (ClusterSystem.call(qInstance, () ->
+      {
+         Instant now = new ConditionalUpdateAction().databaseNow(NODE);
+         return (new Result(markLost(settings, now), purge(settings, now)));
+      }));
+   }
+
+
+
+   /*******************************************************************************
+    ** Mark stale live nodes LOST.
+    *******************************************************************************/
+   private List<String> markLost(ClusterSettings settings, Instant now) throws QException
+   {
+      Instant cutoff = now.minusSeconds(settings.lostAfterSeconds());
+      QQueryFilter stale = new QQueryFilter()
+         .withCriteria(new QFilterCriteria(STATUS, QCriteriaOperator.IN, LIVE_STATUSES))
+         .withCriteria(new QFilterCriteria(HEARTBEAT_AT, QCriteriaOperator.LESS_THAN, cutoff));
+
+      List<String> lost = new ArrayList<>();
+      for(QRecord node : new QueryAction().execute(new QueryInput(NODE).withFilter(stale)).getRecords())
+      {
+         QQueryFilter guard = new QQueryFilter()
+            .withCriteria(new QFilterCriteria(ID, QCriteriaOperator.EQUALS, node.getValue(ID)))
+            .withCriteria(new QFilterCriteria(STATUS, QCriteriaOperator.IN, LIVE_STATUSES))
+            .withCriteria(new QFilterCriteria(HEARTBEAT_AT, QCriteriaOperator.LESS_THAN, cutoff));
+         int count = new ConditionalUpdateAction().execute(new ConditionalUpdateInput(NODE).withFilter(guard)
+            .withValue(STATUS, NodeStatus.LOST.name())
+            .withValue(MODIFY_DATE, now));
+         if(count == 1)
+         {
+            String  nodeId        = node.getValueString(NODE_ID);
+            Instant lastHeartbeat = node.getValueInstant(HEARTBEAT_AT);
+            long    seconds       = lastHeartbeat == null ? settings.lostAfterSeconds() : Duration.between(lastHeartbeat, now).toSeconds();
+            LOG.warn("Cluster node lost: no heartbeat", logPair("nodeId", nodeId), logPair("secondsSinceHeartbeat", seconds), logPair("previousStatus", node.getValueString(STATUS)));
+            lost.add(nodeId);
+            if(settings.alertOnLost())
+            {
+               ClusterAlerts.raiseLost(node, seconds);
+            }
+         }
+      }
+      return (lost);
+   }
+
+
+
+   /*******************************************************************************
+    ** Delete old STOPPED / LOST rows.
+    *******************************************************************************/
+   private int purge(ClusterSettings settings, Instant now) throws QException
+   {
+      Instant cutoff = now.minus(Duration.ofDays(settings.retainStoppedDays()));
+      QQueryFilter old = new QQueryFilter()
+         .withCriteria(new QFilterCriteria(STATUS, QCriteriaOperator.IN, ENDED_STATUSES))
+         .withCriteria(new QFilterCriteria(HEARTBEAT_AT, QCriteriaOperator.LESS_THAN, cutoff));
+
+      List<Serializable> ids = new ArrayList<>();
+      for(QRecord node : new QueryAction().execute(new QueryInput(NODE).withFilter(old)).getRecords())
+      {
+         ids.add(node.getValue(ID));
+         if(NodeStatus.LOST.name().equals(node.getValueString(STATUS)))
+         {
+            ClusterAlerts.resolveLost(node.getValueString(NODE_ID), "The lost node's row was removed after " + settings.retainStoppedDays() + " days.");
+         }
+      }
+      if(ids.isEmpty())
+      {
+         return (0);
+      }
+      new DeleteAction().execute(new DeleteInput(NODE).withPrimaryKeys(ids));
+      LOG.info("Purged old cluster node rows", logPair("count", ids.size()), logPair("retainDays", settings.retainStoppedDays()));
+      return (ids.size());
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/runtime/ClusterSystem.java

@@ -0,0 +1,100 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.runtime;
+
+
+import com.kingsrook.qqq.backend.core.context.CapturedContext;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+
+
+/*******************************************************************************
+ ** Runs the qbit's own bookkeeping on the cluster_ tables as the system user
+ ** (never a person's work), then restores the caller's context.
+ *******************************************************************************/
+public final class ClusterSystem
+{
+
+   /*******************************************************************************
+    ** A body that returns a value and may throw.
+    *******************************************************************************/
+   @FunctionalInterface
+   public interface Body<T>
+   {
+      /*******************************************************************************
+       ** Run the body.
+       *******************************************************************************/
+      T run() throws QException;
+   }
+
+
+
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private ClusterSystem()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** Run in the current instance as the system user.
+    *******************************************************************************/
+   public static <T> T call(Body<T> body) throws QException
+   {
+      QInstance qInstance = QContext.getQInstance();
+      if(qInstance == null)
+      {
+         throw (new QException("The Cluster qbit needs a QContext with an instance"));
+      }
+      return (call(qInstance, body));
+   }
+
+
+
+   /*******************************************************************************
+    ** Run in the given instance as the system user (no transaction).
+    *******************************************************************************/
+   public static <T> T call(QInstance qInstance, Body<T> body) throws QException
+   {
+      CapturedContext original = QContext.capture();
+      try
+      {
+         QContext.init(qInstance, new QSystemUserSession(qInstance));
+         return (body.run());
+      }
+      finally
+      {
+         if(original.qInstance() == null)
+         {
+            QContext.clear();
+         }
+         else
+         {
+            QContext.init(original);
+         }
+      }
+   }
+}

src/main/java/com/kingsrook/qbits/cluster/widgets/ClusterNodesWidgetRenderer.java

@@ -0,0 +1,228 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster.widgets;
+
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.ConditionalUpdateAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.utils.StringUtils;
+
+
+/*******************************************************************************
+ ** The Nodes widget (a standard TABLE widget, full width): every node that is
+ ** up, draining or lost, grouped by role (a node with several roles appears
+ ** under each), with its status and how long ago it last sent a heartbeat.
+ ** Stopped nodes are left out; they stay in the Nodes table until purged.
+ ** Reads in the viewer's own session (the widget needs the cluster read
+ ** permission, which also reads the table).
+ *******************************************************************************/
+public class ClusterNodesWidgetRenderer extends AbstractWidgetRenderer
+{
+   public static final String NAME = "clusterNodes";
+
+   static final String ROLE      = "role";
+   static final String NODE      = "node";
+   static final String STATUS    = "status";
+   static final String HEARTBEAT = "heartbeat";
+
+
+
+   /*******************************************************************************
+    ** Render the table.
+    *******************************************************************************/
+   @Override
+   public RenderWidgetOutput render(RenderWidgetInput input) throws QException
+   {
+      List<QRecord> nodes = new QueryAction().execute(new QueryInput(ClusterTables.NODE)
+         .withFilter(new QQueryFilter(new QFilterCriteria(ClusterTables.STATUS, QCriteriaOperator.NOT_EQUALS, NodeStatus.STOPPED.name()))
+            .withOrderBy(new QFilterOrderBy(ClusterTables.NODE_ID)))).getRecords();
+      Instant now = new ConditionalUpdateAction().databaseNow(ClusterTables.NODE);
+
+      List<Map<String, Object>> rows = new ArrayList<>();
+      for(QNodeRoles.Role role : QNodeRoles.Role.values())
+      {
+         for(QRecord node : nodes)
+         {
+            if(roles(node).contains(role.name()))
+            {
+               rows.add(row(role, node, now));
+            }
+         }
+      }
+
+      TableData table = new TableData(null, List.of(
+         column("Role", ROLE), column("Node", NODE), column("Status", STATUS), column("Last heartbeat", HEARTBEAT)), rows)
+         .withNoRowsFoundHTML("No nodes are running. A node appears here once it starts with the Cluster qbit.")
+         .withRowsPerPage(50)
+         .withHidePaginationDropdown(true);
+      return (new RenderWidgetOutput(table));
+   }
+
+
+
+   /*******************************************************************************
+    ** One row.
+    *******************************************************************************/
+   static Map<String, Object> row(QNodeRoles.Role role, QRecord node, Instant now)
+   {
+      Map<String, Object> row = new LinkedHashMap<>();
+      row.put(ROLE, roleLabel(role));
+      row.put(NODE, nodeText(node));
+      row.put(STATUS, statusText(node));
+      row.put(HEARTBEAT, age(node.getValueInstant(ClusterTables.HEARTBEAT_AT), now));
+      return (row);
+   }
+
+
+
+   /*******************************************************************************
+    ** The node id, with where it runs and its version folded in (four columns
+    ** keep the table readable at every width): "web-7f9 (k8s-2, website-backend, 0.1.110)".
+    *******************************************************************************/
+   static String nodeText(QRecord node)
+   {
+      String details = Stream.of(node.getValueString(ClusterTables.K8S_NODE_NAME), node.getValueString(ClusterTables.ZONE), node.getValueString(ClusterTables.DEPLOYMENT),
+            node.getValueString(ClusterTables.VERSION))
+         .filter(StringUtils::hasContent).collect(Collectors.joining(", "));
+      String nodeId = node.getValueString(ClusterTables.NODE_ID);
+      return (details.isEmpty() ? nodeId : nodeId + " (" + details + ")");
+   }
+
+
+
+   /*******************************************************************************
+    ** "Up", "Draining", "Up, drain requested", "Lost".
+    *******************************************************************************/
+   static String statusText(QRecord node)
+   {
+      String status = node.getValueString(ClusterTables.STATUS);
+      String label  = status;
+      for(NodeStatus value : NodeStatus.values())
+      {
+         if(value.name().equals(status))
+         {
+            label = value.getPossibleValueLabel();
+         }
+      }
+      if(NodeStatus.UP.name().equals(status) && Boolean.TRUE.equals(node.getValueBoolean(ClusterTables.DRAIN_REQUESTED)))
+      {
+         label += ", drain requested";
+      }
+      return (label == null ? "" : label);
+   }
+
+
+
+   /*******************************************************************************
+    ** "12 seconds ago", "3 minutes ago", "2 hours ago", "4 days ago".
+    *******************************************************************************/
+   static String age(Instant at, Instant now)
+   {
+      if(at == null)
+      {
+         return ("never");
+      }
+      long seconds = Math.max(0, Duration.between(at, now).toSeconds());
+      if(seconds < 120)
+      {
+         return (plural(seconds, "second"));
+      }
+      if(seconds < 7200)
+      {
+         return (plural(seconds / 60, "minute"));
+      }
+      if(seconds < 172800)
+      {
+         return (plural(seconds / 3600, "hour"));
+      }
+      return (plural(seconds / 86400, "day"));
+   }
+
+
+
+   /*******************************************************************************
+    ** "1 second ago", "5 seconds ago".
+    *******************************************************************************/
+   private static String plural(long count, String unit)
+   {
+      return (count + " " + unit + (count == 1 ? "" : "s") + " ago");
+   }
+
+
+
+   /*******************************************************************************
+    ** "Web", "Worker", "Scheduler", "ESB".
+    *******************************************************************************/
+   static String roleLabel(QNodeRoles.Role role)
+   {
+      return (role == QNodeRoles.Role.ESB ? "ESB" : role.name().charAt(0) + role.name().substring(1).toLowerCase(Locale.ROOT));
+   }
+
+
+
+   /*******************************************************************************
+    ** The node's role names.
+    *******************************************************************************/
+   private static Set<String> roles(QRecord node)
+   {
+      String roles = node.getValueString(ClusterTables.ROLES);
+      if(roles == null)
+      {
+         return (Set.of());
+      }
+      return (Stream.of(roles.split(",")).map(String::trim).map(s -> s.toUpperCase(Locale.ROOT)).collect(Collectors.toSet()));
+   }
+
+
+
+   /*******************************************************************************
+    ** A flexible-width text column.
+    *******************************************************************************/
+   private static TableData.Column column(String header, String accessor)
+   {
+      return (new TableData.Column("default", header, accessor, null, "left"));
+   }
+}

src/main/resources/db/liquibase/qbit-cluster/cluster-defaults.changelog.sql

@@ -0,0 +1,54 @@
+-- liquibase formatted sql
+-- Copyright 2026 Kingsrook, LLC. Licensed under the Apache License, Version 2.0.
+-- https://www.apache.org/licenses/LICENSE-2.0
+
+-- qbit-cluster starter defaults: the four cluster settings, so the Settings table shows what
+-- the runtime uses from the first deploy. Include it AFTER cluster.changelog.sql:
+--
+--   - include: { file: db/liquibase/qbit-cluster/cluster-defaults.changelog.sql }
+--
+-- Rules for this file:
+--   * insert-only: a row is inserted only when its key is absent, so an administrator's edits
+--     always survive and nothing is ever updated;
+--   * every rollback is empty: a key-based delete could remove a row the host already had;
+--   * every changeset has a precondition on its table, so a host that has not created the
+--     table yet skips it (MARK_RAN) instead of failing its deploy;
+--   * portable SQL (PostgreSQL and H2): INSERT ... SELECT ... WHERE NOT EXISTS.
+-- The values equal ClusterSettings.DEFAULTS, which the runtime also uses when a row is missing,
+-- so deploying this file never changes behavior. ClusterChangelogTest runs it twice.
+
+-- changeset qbit-cluster:defaults-1
+-- preconditions onFail:MARK_RAN onError:MARK_RAN
+-- precondition-sql-check expectedResult:0 SELECT COUNT(*) FROM cluster_setting WHERE setting_key = 'heartbeat_seconds'
+-- comment: How often each node records that it is alive.
+INSERT INTO cluster_setting (setting_key, setting_value, description, create_date, modify_date)
+SELECT 'heartbeat_seconds', '15', 'How often, in seconds, each node records that it is alive.', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
+WHERE NOT EXISTS (SELECT 1 FROM cluster_setting WHERE setting_key = 'heartbeat_seconds');
+-- rollback empty
+
+-- changeset qbit-cluster:defaults-2
+-- preconditions onFail:MARK_RAN onError:MARK_RAN
+-- precondition-sql-check expectedResult:0 SELECT COUNT(*) FROM cluster_setting WHERE setting_key = 'lost_after_seconds'
+-- comment: When a silent node is marked Lost.
+INSERT INTO cluster_setting (setting_key, setting_value, description, create_date, modify_date)
+SELECT 'lost_after_seconds', '60', 'A node with no heartbeat for this many seconds is marked Lost.', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
+WHERE NOT EXISTS (SELECT 1 FROM cluster_setting WHERE setting_key = 'lost_after_seconds');
+-- rollback empty
+
+-- changeset qbit-cluster:defaults-3
+-- preconditions onFail:MARK_RAN onError:MARK_RAN
+-- precondition-sql-check expectedResult:0 SELECT COUNT(*) FROM cluster_setting WHERE setting_key = 'retain_stopped_days'
+-- comment: How long rows of stopped and lost nodes are kept.
+INSERT INTO cluster_setting (setting_key, setting_value, description, create_date, modify_date)
+SELECT 'retain_stopped_days', '7', 'Days to keep the rows of stopped and lost nodes before they are deleted.', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
+WHERE NOT EXISTS (SELECT 1 FROM cluster_setting WHERE setting_key = 'retain_stopped_days');
+-- rollback empty
+
+-- changeset qbit-cluster:defaults-4
+-- preconditions onFail:MARK_RAN onError:MARK_RAN
+-- precondition-sql-check expectedResult:0 SELECT COUNT(*) FROM cluster_setting WHERE setting_key = 'alert_on_lost'
+-- comment: Whether a lost node raises the cluster.node.lost alert.
+INSERT INTO cluster_setting (setting_key, setting_value, description, create_date, modify_date)
+SELECT 'alert_on_lost', 'true', 'Whether a lost node raises an alert (true or false).', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP
+WHERE NOT EXISTS (SELECT 1 FROM cluster_setting WHERE setting_key = 'alert_on_lost');
+-- rollback empty

src/main/resources/db/liquibase/qbit-cluster/cluster.changelog.sql

@@ -0,0 +1,82 @@
+-- liquibase formatted sql
+-- Copyright 2026 Kingsrook, LLC. Licensed under the Apache License, Version 2.0.
+-- https://www.apache.org/licenses/LICENSE-2.0
+
+-- qbit-cluster schema (QRun-IO/qqq docs/specs/2026-10-15-cluster-ha-design.md section E).
+-- Load this file from the host's master changelog, then the defaults:
+--
+--   - include: { file: db/liquibase/qbit-cluster/cluster.changelog.sql }
+--   - include: { file: db/liquibase/qbit-cluster/cluster-defaults.changelog.sql }
+--
+-- Rules for this file: the host runs it, the qbit never runs DDL; additive only (nothing is
+-- ever dropped or rewritten); every statement is idempotent (IF NOT EXISTS); no foreign keys;
+-- no extensions, owners or grants, so the application's own (non-superuser) owner role can run
+-- it. PostgreSQL first; H2 in PostgreSQL mode runs it too (ClusterChangelogTest).
+-- Column names are the snake_case form of the field names (ClusterTables).
+-- Rollbacks drop a table only when the rollback is asked for explicitly; these tables hold
+-- operational state only (nodes, leases, settings), never business data.
+
+-- changeset qbit-cluster:1
+-- comment: One row per running (or recently running) node of the application: who it is, where it runs, its roles, its heartbeat and status, and any drain request.
+CREATE TABLE IF NOT EXISTS cluster_node (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   node_id VARCHAR(250) NOT NULL,
+   pod_name VARCHAR(250),
+   k8s_node_name VARCHAR(250),
+   zone VARCHAR(100),
+   deployment VARCHAR(250),
+   roles VARCHAR(100),
+   version VARCHAR(100),
+   started_at TIMESTAMP,
+   heartbeat_at TIMESTAMP,
+   status VARCHAR(20),
+   worker_threads INTEGER,
+   scheduler_active BOOLEAN,
+   esb_triggers_active BOOLEAN,
+   async_jobs_running INTEGER,
+   heap_used_mb INTEGER,
+   heap_max_mb INTEGER,
+   drain_requested BOOLEAN DEFAULT FALSE,
+   drain_requested_by VARCHAR(250),
+   drain_reason VARCHAR(500),
+   create_date TIMESTAMP,
+   modify_date TIMESTAMP,
+   CONSTRAINT pk_cluster_node PRIMARY KEY (id),
+   CONSTRAINT uq_cluster_node_node_id UNIQUE (node_id)
+);
+-- rollback DROP TABLE IF EXISTS cluster_node;
+
+-- changeset qbit-cluster:2
+-- comment: The lost-node sweep and purge find nodes by status and heartbeat age.
+CREATE INDEX IF NOT EXISTS ix_cluster_node_status_heartbeat ON cluster_node (status, heartbeat_at);
+-- rollback DROP INDEX IF EXISTS ix_cluster_node_status_heartbeat;
+
+-- changeset qbit-cluster:3
+-- comment: Named leases for singletons that are not Quartz jobs (ClusterLeases). Every change is a compare-and-set on version.
+CREATE TABLE IF NOT EXISTS cluster_lease (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   name VARCHAR(100) NOT NULL,
+   holder_node_id VARCHAR(250),
+   expires_at TIMESTAMP,
+   version INTEGER NOT NULL DEFAULT 0,
+   acquired_at TIMESTAMP,
+   create_date TIMESTAMP,
+   modify_date TIMESTAMP,
+   CONSTRAINT pk_cluster_lease PRIMARY KEY (id),
+   CONSTRAINT uq_cluster_lease_name UNIQUE (name)
+);
+-- rollback DROP TABLE IF EXISTS cluster_lease;
+
+-- changeset qbit-cluster:4
+-- comment: The cluster's settings (heartbeat_seconds, lost_after_seconds, retain_stopped_days, alert_on_lost), seeded by cluster-defaults.changelog.sql.
+CREATE TABLE IF NOT EXISTS cluster_setting (
+   id INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL,
+   setting_key VARCHAR(100) NOT NULL,
+   setting_value VARCHAR(250) NOT NULL,
+   description VARCHAR(500),
+   create_date TIMESTAMP,
+   modify_date TIMESTAMP,
+   CONSTRAINT pk_cluster_setting PRIMARY KEY (id),
+   CONSTRAINT uq_cluster_setting_setting_key UNIQUE (setting_key)
+);
+-- rollback DROP TABLE IF EXISTS cluster_setting;

src/main/resources/help/qbit-cluster.yaml

@@ -0,0 +1,167 @@
+bundle: qbit-cluster
+version: 0.1.0
+layer: QBIT
+entries:
+- key: app:cluster;slot:help
+  summary: The servers (nodes) running this application, what each one does, and whether each one is healthy.
+  content: |
+    When the application runs on more than one server, each running copy is a **node**. Every node records itself in **{{label:cluster_node}}** when it starts and then sends a heartbeat every few seconds, so this page shows which nodes are up, which roles they have (Web, Worker, Scheduler, ESB) and when each was last heard from.
+
+    If a node goes quiet for longer than the configured time, another node marks it **Lost** and raises an alert. To take a node out of service gently, select it in **{{label:cluster_node}}** and use **{{label:clusterDrainNode}}**.
+
+    **{{label:cluster_lease}}** shows which node currently runs each one-at-a-time background task, and **{{label:cluster_setting}}** holds the timings.
+- key: widget:clusterNodes;slot:label
+  summary: Every node that is up, draining or lost, grouped by role, with how long ago it last sent a heartbeat.
+  content: |
+    A node with several roles appears once under each. Stopped nodes are not shown here; they stay in the Nodes table until they are cleaned up. If a heartbeat is older than a minute or so, the node is probably in trouble.
+- key: possibleValueSource:cluster_node_status;slot:help
+  summary: Where a node is in its life.
+  content: |
+    - **Up:** running and sending heartbeats.
+    - **Draining:** finishing what it is doing; it takes no new requests or background work.
+    - **Lost:** stopped sending heartbeats without shutting down cleanly (a crash, a forced kill, a network problem).
+    - **Stopped:** shut down cleanly.
+
+- key: table:cluster_node;slot:help
+  summary: One row for each node of the application, kept up to date by the node itself.
+  content: |
+    Each node adds or refreshes its own row when it starts, then updates **{{label:cluster_node.heartbeatAt}}** and its live numbers every heartbeat. Rows cannot be added or edited here; they reflect what the nodes report.
+
+    Use **{{label:clusterDrainNode}}** to ask a node to stop taking new work, and **{{label:clusterCancelDrain}}** to withdraw that request before the node acts on it. Rows of stopped and lost nodes are deleted automatically after the number of days in the retain_stopped_days setting; you may also delete an old row yourself.
+
+    **How rows get created:** automatically, by each node when it starts. If the list is empty, no node with the Cluster feature has started against this database yet.
+- key: table:cluster_node;section:identity
+  summary: Which node this is, what it does, and whether it is healthy.
+- key: table:cluster_node;section:placement
+  summary: Where the node runs and which version of the application it runs.
+- key: table:cluster_node;section:work
+  summary: What work the node takes on and how much memory it uses.
+- key: table:cluster_node;section:drain
+  summary: Whether someone has asked this node to drain, who, and why.
+- key: table:cluster_node;section:dates
+  summary: When the node started and when its row was created and last changed.
+- key: table:cluster_node;field:nodeId
+  summary: The node's unique name, usually its pod or host name.
+  content: |
+    Taken from the QQQ_NODE_ID setting, else the HOSTNAME (in Kubernetes, the pod name), else the server's host name.
+- key: table:cluster_node;field:podName
+  summary: The Kubernetes pod running this node, when the deployment provides it.
+  content: |
+    From the QQQ_POD_NAME environment variable (the Kubernetes Downward API). Empty outside Kubernetes.
+- key: table:cluster_node;field:k8sNodeName
+  summary: The Kubernetes server (worker machine) the pod is scheduled on, when provided.
+  content: |
+    From the QQQ_K8S_NODE_NAME environment variable. Useful to see whether several nodes share one machine.
+- key: table:cluster_node;field:zone
+  summary: The availability zone or site the node runs in, when provided.
+  content: |
+    From the QQQ_ZONE environment variable.
+- key: table:cluster_node;field:deployment
+  summary: The deployment (group of identical nodes) this node belongs to, when provided.
+  content: |
+    From the QQQ_DEPLOYMENT environment variable, for example website-backend or website-backend-worker.
+- key: table:cluster_node;field:roles
+  summary: 'What this node does: WEB, WORKER, SCHEDULER and/or ESB, as a comma list.'
+  content: |
+    - **WEB:** serves people and API requests.
+    - **WORKER:** runs durable background jobs.
+    - **SCHEDULER:** runs scheduled processes.
+    - **ESB:** runs message triggers.
+
+    Set with the QQQ_NODE_ROLES environment variable; a node without it does everything.
+- key: table:cluster_node;field:version
+  summary: The application version this node runs.
+  content: |
+    The version the application reports (or the QQQ_APP_VERSION environment variable); otherwise the QQQ version, shown as "qqq" and a number. During a rolling update, nodes with two versions are normal for a few minutes.
+- key: table:cluster_node;field:startedAt
+  summary: When this node last started.
+- key: table:cluster_node;field:heartbeatAt
+  summary: When this node last reported that it is alive.
+  content: |
+    Updated every heartbeat (15 seconds by default). For a stopped node, the time it stopped. A node whose heartbeat is older than the lost_after_seconds setting is marked Lost.
+- key: table:cluster_node;field:status
+  summary: 'Where the node is in its life: {{values:cluster_node_status}}.'
+- key: table:cluster_node;field:workerThreads
+  summary: How many durable-job worker threads the node runs; 0 means it runs no background jobs.
+  content: |
+    Empty when the node uses the application's default.
+- key: table:cluster_node;field:schedulerActive
+  summary: Whether this node runs scheduled processes.
+- key: table:cluster_node;field:esbTriggersActive
+  summary: Whether this node runs message (ESB) triggers.
+- key: table:cluster_node;field:asyncJobsRunning
+  summary: How many process steps are running on this node right now.
+  content: |
+    A draining node waits for these to finish before it stops.
+- key: table:cluster_node;field:heapUsedMb
+  summary: Memory the node's Java process is using, in megabytes, at the last heartbeat.
+- key: table:cluster_node;field:heapMaxMb
+  summary: The most memory the node's Java process may use, in megabytes.
+- key: table:cluster_node;field:drainRequested
+  summary: Whether someone has asked this node to drain.
+  content: |
+    Set by **{{label:clusterDrainNode}}** and cleared by **{{label:clusterCancelDrain}}** or when the node restarts. The node acts on it at its next heartbeat.
+- key: table:cluster_node;field:drainRequestedBy
+  summary: Who asked this node to drain.
+- key: table:cluster_node;field:drainReason
+  summary: The reason given when the drain was requested.
+
+- key: table:cluster_lease;slot:help
+  summary: Which node currently runs each background task that must run on only one node at a time.
+  content: |
+    A lease is held by one node until it expires; the holder renews it while it is healthy. When the holder stops or is lost, the lease expires and another node takes it over. The built-in **cluster-sweeper** lease decides which node marks silent nodes as Lost and cleans up old rows.
+
+    To hand a lease over at once, clear **{{label:cluster_lease.holderNodeId}}** or set **{{label:cluster_lease.expiresAt}}** in the past; another node takes it within one heartbeat.
+
+    **How rows get created:** automatically, the first time any node asks for a lease by name. If the list is empty, no node has started yet.
+- key: table:cluster_lease;section:identity
+  summary: The lease, who holds it, and until when.
+- key: table:cluster_lease;section:dates
+  summary: When the current holder got the lease, its change counter, and the row dates.
+- key: table:cluster_lease;field:name
+  summary: The name of the task the lease is for, such as cluster-sweeper.
+- key: table:cluster_lease;field:holderNodeId
+  summary: The node that holds the lease; empty when it was released.
+- key: table:cluster_lease;field:expiresAt
+  summary: When the lease runs out unless its holder renews it.
+- key: table:cluster_lease;field:version
+  summary: A counter that goes up on every change, so two nodes can never take the lease at the same moment.
+- key: table:cluster_lease;field:acquiredAt
+  summary: When the current holder took the lease.
+
+- key: table:cluster_setting;slot:help
+  summary: Timings for heartbeats, lost-node detection, clean-up and alerts.
+  content: |
+    - **heartbeat_seconds** (15): how often each node reports that it is alive.
+    - **lost_after_seconds** (60): a node silent this long is marked Lost. Keep it at least three heartbeats.
+    - **retain_stopped_days** (7): rows of stopped and lost nodes are deleted after this many days.
+    - **alert_on_lost** (true): whether a lost node raises the "Cluster node lost" alert.
+
+    Changes take effect at each node's next heartbeat. A missing or invalid row uses the default shown above.
+
+    **How rows get created:** the defaults are added by the database setup when the application is installed. If the list is empty, the defaults above still apply; you can add a row to change one.
+- key: table:cluster_setting;section:identity
+  summary: The setting and its value.
+- key: table:cluster_setting;section:dates
+  summary: When the setting was added and last changed.
+- key: table:cluster_setting;field:settingKey
+  summary: Which setting this row holds, such as the heartbeat interval in seconds.
+- key: table:cluster_setting;field:settingValue
+  summary: The value; a whole number for the seconds and days settings, true or false for the alert setting.
+- key: table:cluster_setting;field:description
+  summary: What the setting does.
+
+- key: process:clusterDrainNode;slot:help
+  summary: Ask the selected nodes to stop taking new requests and work and finish what they are doing.
+  content: |
+    Select one or more nodes that are up and run this. The Review screen shows which nodes will be asked and which are skipped and why, and warns you if no other node would remain up.
+
+    Each node sees the request at its next heartbeat (within about 15 seconds). It then reports not ready, so the load balancer stops sending it people's requests, stops picking up background work, and lets running steps finish. It keeps running until it is stopped or restarted, which is what you usually do next.
+
+    Draining cannot be undone once the node has started. Before that, use **{{label:clusterCancelDrain}}**.
+- key: process:clusterDrainNode;field:clusterDrainReason
+  summary: Why you are draining the node, for example "maintenance on this server". Optional.
+- key: process:clusterCancelDrain;slot:help
+  summary: Withdraw a drain request that the node has not acted on yet.
+  content: |
+    Only nodes with a drain request that are still up can be changed; the Review screen lists any others with the reason. A node that has already started draining cannot be undrained; restart it instead.

src/main/resources/qbit-cluster.properties

@@ -0,0 +1 @@
+version=${project.version}

src/test/java/com/kingsrook/qbits/cluster/BaseClusterTest.java

@@ -0,0 +1,286 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.runtime.ClusterNodeInfo;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntime;
+import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallback;
+import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
+import com.kingsrook.qqq.backend.core.cluster.QNodeLifecycle;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.exceptions.QException;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
+import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.core.model.session.QUser;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore;
+import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** A memory-backend instance with the Cluster qbit (under an Admin app), a
+ ** recording alerts sink, this JVM as node "node-a", and helpers.
+ *******************************************************************************/
+public class BaseClusterTest
+{
+   public static final String BACKEND = "memory";
+   public static final String NODE_A  = "node-a";
+
+   protected QInstance         qInstance;
+   protected ClusterQBitConfig config;
+
+   private final List<ClusterRuntime> runtimes = new ArrayList<>();
+
+
+
+   /*******************************************************************************
+    ** Fresh store, instance and node identity.
+    *******************************************************************************/
+   @BeforeEach
+   void setUpCluster() throws Exception
+   {
+      MemoryRecordStore.fullReset();
+      QNodeLifecycle.resetForTests();
+      RecordingAlertSink.clear();
+      setNode(NODE_A);
+
+      qInstance = new QInstance();
+      qInstance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class));
+      qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK));
+      qInstance.addApp(new QAppMetaData().withName("admin").withLabel("Admin"));
+      AlertsInstanceMetaData.ofOrWithNew(qInstance).withSink(new QCodeReference(RecordingAlertSink.class));
+      config = new ClusterQBitConfig().withBackendName(BACKEND).withParentAppName("admin").withAppVersion("1.2.3");
+      new ClusterQBitProducer().withQBitConfig(config).produce(qInstance);
+      new QInstanceEnricher(qInstance).enrich();
+      QContext.init(qInstance, new QSystemUserSession(qInstance));
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop runtimes, clear thread locals and statics.
+    *******************************************************************************/
+   @AfterEach
+   void tearDownCluster()
+   {
+      runtimes.forEach(ClusterRuntime::stop);
+      QNodeLifecycle.resetForTests();
+      QNodeRoles.set(null);
+      QContext.clear();
+      MemoryRecordStore.fullReset();
+   }
+
+
+
+   /*******************************************************************************
+    ** Make this JVM the given node (all roles).
+    *******************************************************************************/
+   protected static void setNode(String nodeId)
+   {
+      QNodeRoles.set(new QNodeRoles.Settings(EnumSet.allOf(QNodeRoles.Role.class), true, true, true, 4, nodeId));
+   }
+
+
+
+   /*******************************************************************************
+    ** A runtime for the current node id (registered, no background thread),
+    ** stopped after the test.
+    *******************************************************************************/
+   protected ClusterRuntime runtime(Map<String, String> environment) throws QException
+   {
+      ClusterRuntime runtime = new ClusterRuntime(qInstance, new ClusterNodeInfo(config, environment::get));
+      runtime.initialize();
+      runtimes.add(runtime);
+      return (runtime);
+   }
+
+
+
+   /*******************************************************************************
+    ** A runtime with an empty environment.
+    *******************************************************************************/
+   protected ClusterRuntime runtime() throws QException
+   {
+      return (runtime(Map.of()));
+   }
+
+
+
+   /*******************************************************************************
+    ** All rows of a table.
+    *******************************************************************************/
+   protected List<QRecord> all(String table) throws QException
+   {
+      return (new QueryAction().execute(new QueryInput(table)).getRecords());
+   }
+
+
+
+   /*******************************************************************************
+    ** A node row by node id (or null).
+    *******************************************************************************/
+   protected QRecord node(String nodeId) throws QException
+   {
+      List<QRecord> records = new QueryAction().execute(new QueryInput(ClusterTables.NODE)
+         .withFilter(new QQueryFilter(new QFilterCriteria(ClusterTables.NODE_ID, QCriteriaOperator.EQUALS, nodeId)))).getRecords();
+      return (records.isEmpty() ? null : records.get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Insert a row.
+    *******************************************************************************/
+   protected QRecord insert(String table, QRecord record) throws QException
+   {
+      QRecord inserted = new InsertAction().execute(new InsertInput(table).withRecord(record)).getRecords().get(0);
+      assertThat(inserted.getErrors()).isNullOrEmpty();
+      return (inserted);
+   }
+
+
+
+   /*******************************************************************************
+    ** A node row as another JVM would have written it.
+    *******************************************************************************/
+   protected QRecord otherNode(String nodeId, String status, Instant heartbeatAt, String roles) throws QException
+   {
+      return (insert(ClusterTables.NODE, new QRecord()
+         .withValue(ClusterTables.NODE_ID, nodeId)
+         .withValue(ClusterTables.STATUS, status)
+         .withValue(ClusterTables.ROLES, roles)
+         .withValue(ClusterTables.STARTED_AT, heartbeatAt)
+         .withValue(ClusterTables.HEARTBEAT_AT, heartbeatAt)));
+   }
+
+
+
+   /*******************************************************************************
+    ** Set fields on a row directly (simulating time passing, or another node).
+    *******************************************************************************/
+   protected void setFields(String table, Serializable id, Map<String, Serializable> values) throws QException
+   {
+      QRecord record = new QRecord().withValue(ClusterTables.ID, id);
+      values.forEach(record::setValue);
+      new UpdateAction().execute(new UpdateInput(table).withRecord(record));
+   }
+
+
+
+   /*******************************************************************************
+    ** Log in as a person with the given permissions.
+    *******************************************************************************/
+   protected QSession login(String fullName, String... permissions)
+   {
+      QSession session = new QSession().withUser(new QUser().withIdReference(fullName.toLowerCase().replace(' ', '.')).withFullName(fullName));
+      for(String permission : permissions)
+      {
+         session.withPermission(permission);
+      }
+      QContext.setQSession(session);
+      return (session);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run a record action on the given ids; returns the Result lines as
+    ** "STATUS: count message".
+    *******************************************************************************/
+   protected List<String> runAction(String processName, List<Serializable> ids, Map<String, Serializable> values) throws QException
+   {
+      RunProcessInput input = new RunProcessInput()
+         .withProcessName(processName)
+         .withFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP)
+         .withCallback(new QProcessCallback()
+         {
+            /*******************************************************************************
+             ** The selected records.
+             *******************************************************************************/
+            @Override
+            public QQueryFilter getQueryFilter()
+            {
+               return (new QQueryFilter().withCriteria(ClusterTables.ID, QCriteriaOperator.IN, ids));
+            }
+
+
+
+            /*******************************************************************************
+             ** The form values.
+             *******************************************************************************/
+            @Override
+            public Map<String, Serializable> getFieldValues(List<QFieldMetaData> fields)
+            {
+               return (values);
+            }
+         });
+      values.forEach(input::addValue);
+      RunProcessOutput output = new RunProcessAction().execute(input);
+      if(output.getException().isPresent())
+      {
+         throw (new QException(output.getException().get()));
+      }
+      @SuppressWarnings("unchecked")
+      List<ProcessSummaryLineInterface> lines = (List<ProcessSummaryLineInterface>) output.getValue(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
+      List<String> text = new ArrayList<>();
+      for(ProcessSummaryLineInterface line : lines == null ? List.<ProcessSummaryLineInterface>of() : lines)
+      {
+         text.add(line.getStatus() + ": " + (line instanceof ProcessSummaryLine summary ? summary.getCount() + " " + summary.getMessage() : String.valueOf(line)));
+      }
+      return (text);
+   }
+}

src/test/java/com/kingsrook/qbits/cluster/ClusterChangelogTest.java

@@ -0,0 +1,303 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.cluster.model.ClusterSettings;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qbits.cluster.runtime.ClusterNodeInfo;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntime;
+import com.kingsrook.qbits.cluster.runtime.ClusterSweeper;
+import com.kingsrook.qqq.backend.core.cluster.QNodeLifecycle;
+import com.kingsrook.qqq.backend.core.cluster.QNodeRoles;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
+import liquibase.Liquibase;
+import liquibase.database.Database;
+import liquibase.database.DatabaseFactory;
+import liquibase.database.jvm.JdbcConnection;
+import liquibase.resource.ClassLoaderResourceAccessor;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** The shipped changelogs through real Liquibase on H2 (PostgreSQL mode), then
+ ** the runtime and the lease compare-and-set as real SQL through the RDBMS
+ ** module against the tables they created (so the DDL and the metadata agree).
+ *******************************************************************************/
+class ClusterChangelogTest
+{
+   static final String SCHEMA   = "db/liquibase/qbit-cluster/cluster.changelog.sql";
+   static final String DEFAULTS = "db/liquibase/qbit-cluster/cluster-defaults.changelog.sql";
+
+
+
+   /*******************************************************************************
+    ** Clear statics.
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      QNodeLifecycle.resetForTests();
+      QNodeRoles.set(null);
+      QContext.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Schema then defaults, twice: tables, the seeded settings (equal to the
+    ** runtime defaults); a second update runs nothing; an edit survives.
+    *******************************************************************************/
+   @Test
+   void schemaAndDefaultsAreIdempotentAndInsertOnly() throws Exception
+   {
+      try(Connection connection = DriverManager.getConnection(url("seed")))
+      {
+         update(connection, SCHEMA);
+         update(connection, DEFAULTS);
+         for(Map.Entry<String, List<String>> setting : ClusterSettings.DEFAULTS.entrySet())
+         {
+            assertThat(scalar(connection, "SELECT setting_value FROM cluster_setting WHERE setting_key = '" + setting.getKey() + "'")).isEqualTo(setting.getValue().get(0));
+            assertThat(scalar(connection, "SELECT description FROM cluster_setting WHERE setting_key = '" + setting.getKey() + "'")).isEqualTo(setting.getValue().get(1));
+         }
+         try(Statement statement = connection.createStatement())
+         {
+            statement.execute("UPDATE cluster_setting SET setting_value = '30' WHERE setting_key = 'heartbeat_seconds'");
+            statement.execute("DELETE FROM cluster_setting WHERE setting_key = 'alert_on_lost'");
+         }
+         update(connection, SCHEMA);
+         update(connection, DEFAULTS);
+         assertThat(scalar(connection, "SELECT setting_value FROM cluster_setting WHERE setting_key = 'heartbeat_seconds'")).isEqualTo("30");
+         assertThat(scalar(connection, "SELECT COUNT(*) FROM cluster_setting")).isEqualTo("3");
+         assertThat(scalar(connection, "SELECT COUNT(*) FROM databasechangelog")).isEqualTo("8");
+         assertThat(scalar(connection, "SELECT COUNT(*) FROM cluster_node")).isEqualTo("0");
+         assertThat(scalar(connection, "SELECT COUNT(*) FROM cluster_lease")).isEqualTo("0");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Defaults on a database without the tables mark their changesets ran.
+    *******************************************************************************/
+   @Test
+   void defaultsWithoutTablesMarkRan() throws Exception
+   {
+      try(Connection connection = DriverManager.getConnection(url("empty")))
+      {
+         update(connection, DEFAULTS);
+         assertThat(scalar(connection, "SELECT COUNT(*) FROM databasechangelog WHERE exectype = 'MARK_RAN'")).isEqualTo("4");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Every defaults changeset is guarded, insert-only, with an empty rollback;
+    ** the schema has no foreign keys and nothing a non-owner role cannot run.
+    *******************************************************************************/
+   @Test
+   void changelogRules() throws Exception
+   {
+      String defaults = text(DEFAULTS);
+      for(String part : defaults.split("(?m)^(?=-- changeset )"))
+      {
+         if(part.startsWith("-- changeset "))
+         {
+            assertThat(part).contains("-- preconditions onFail:MARK_RAN onError:MARK_RAN").contains("NOT EXISTS").contains("-- rollback empty").doesNotContain("UPDATE ");
+         }
+      }
+      String schema = statements(text(SCHEMA)).toString().toUpperCase();
+      assertThat(schema).doesNotContain("FOREIGN KEY", "REFERENCES", "EXTENSION", "GRANT ", "OWNER TO", "SUPERUSER");
+      assertThat(schema).contains("CREATE TABLE IF NOT EXISTS CLUSTER_NODE", "CREATE TABLE IF NOT EXISTS CLUSTER_LEASE", "CREATE TABLE IF NOT EXISTS CLUSTER_SETTING");
+   }
+
+
+
+   /*******************************************************************************
+    ** The runtime, settings, sweep and leases on the RDBMS module (real SQL)
+    ** against the changelog's tables.
+    *******************************************************************************/
+   @Test
+   void runtimeAndLeasesOnRealSql() throws Exception
+   {
+      String database = "rdbms" + System.nanoTime();
+      try(Connection connection = DriverManager.getConnection(lowerCaseUrl(database), "sa", ""))
+      {
+         try(Statement statement = connection.createStatement())
+         {
+            for(String sql : statements(text(SCHEMA)))
+            {
+               statement.execute(sql);
+            }
+            for(String sql : statements(text(DEFAULTS)))
+            {
+               statement.execute(sql);
+            }
+         }
+
+         RDBMSBackendMetaData backend = new RDBMSBackendMetaData().withName("rdbms").withVendor("h2").withHostName("mem").withDatabaseName(database).withUsername("sa");
+         backend.setJdbcUrl(lowerCaseUrl(database));
+         QInstance qInstance = new QInstance();
+         qInstance.addBackend(backend);
+         qInstance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK));
+         ClusterQBitConfig config = new ClusterQBitConfig().withBackendName("rdbms");
+         new ClusterQBitProducer().withQBitConfig(config).produce(qInstance);
+         new QInstanceEnricher(qInstance).enrich();
+         QContext.init(qInstance, new QSystemUserSession(qInstance));
+         QNodeRoles.set(new QNodeRoles.Settings(EnumSet.of(QNodeRoles.Role.WEB), true, false, false, 0, "sql-node"));
+
+         ClusterRuntime runtime = new ClusterRuntime(qInstance, new ClusterNodeInfo(config, Map.of(ClusterNodeInfo.ENV_POD_NAME, "pod-1", ClusterNodeInfo.ENV_K8S_NODE, "wk-21")::get));
+         runtime.initialize();
+         assertThat(runtime.getSettings()).isEqualTo(ClusterSettings.defaultSettings());
+         assertThat(scalar(connection, "SELECT status FROM cluster_node WHERE node_id = 'sql-node'")).isEqualTo("UP");
+         assertThat(scalar(connection, "SELECT k8s_node_name FROM cluster_node WHERE node_id = 'sql-node'")).isEqualTo("wk-21");
+         assertThat(scalar(connection, "SELECT roles FROM cluster_node WHERE node_id = 'sql-node'")).isEqualTo("WEB");
+
+         try(Statement statement = connection.createStatement())
+         {
+            statement.execute("INSERT INTO cluster_node (node_id, status, roles, heartbeat_at) VALUES ('silent', 'UP', 'WORKER', DATEADD('MINUTE', -10, CURRENT_TIMESTAMP))");
+         }
+         runtime.tick();
+         assertThat(scalar(connection, "SELECT status FROM cluster_node WHERE node_id = 'silent'")).isEqualTo(NodeStatus.LOST.name());
+         assertThat(scalar(connection, "SELECT holder_node_id FROM cluster_lease WHERE name = '" + ClusterSweeper.LEASE_NAME + "'")).isEqualTo("sql-node");
+
+         ClusterLeases.Holder other = ClusterLeases.as("other-node");
+         assertThat(other.tryAcquire(ClusterSweeper.LEASE_NAME, Duration.ofSeconds(60))).isFalse();
+         try(Statement statement = connection.createStatement())
+         {
+            statement.execute("UPDATE cluster_lease SET expires_at = DATEADD('SECOND', -1, CURRENT_TIMESTAMP)");
+         }
+         assertThat(other.tryAcquire(ClusterSweeper.LEASE_NAME, Duration.ofSeconds(60))).isTrue();
+         assertThat(ClusterLeases.isLeader(ClusterSweeper.LEASE_NAME)).isFalse();
+         assertThat(scalar(connection, "SELECT version FROM cluster_lease")).isEqualTo("2");
+
+         runtime.stop();
+         assertThat(scalar(connection, "SELECT status FROM cluster_node WHERE node_id = 'sql-node'")).isEqualTo(NodeStatus.STOPPED.name());
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** An in-memory H2 in PostgreSQL mode for Liquibase, kept until the JVM ends.
+    *******************************************************************************/
+   private static String url(String name)
+   {
+      return ("jdbc:h2:mem:" + name + System.nanoTime() + ";MODE=PostgreSQL;DEFAULT_NULL_ORDERING=HIGH;DB_CLOSE_DELAY=-1");
+   }
+
+
+
+   /*******************************************************************************
+    ** An in-memory H2 in PostgreSQL mode with lower-case names, as PostgreSQL
+    ** folds them (the RDBMS module quotes identifiers).
+    *******************************************************************************/
+   private static String lowerCaseUrl(String name)
+   {
+      return ("jdbc:h2:mem:" + name + ";MODE=PostgreSQL;DATABASE_TO_UPPER=FALSE;DEFAULT_NULL_ORDERING=HIGH;DB_CLOSE_DELAY=-1");
+   }
+
+
+
+   /*******************************************************************************
+    ** The SQL statements of a formatted-SQL changelog (comments dropped).
+    *******************************************************************************/
+   private static List<String> statements(String changelog)
+   {
+      List<String>  statements = new java.util.ArrayList<>();
+      StringBuilder current    = new StringBuilder();
+      for(String line : changelog.split("\n"))
+      {
+         if(line.trim().startsWith("--") || line.isBlank())
+         {
+            continue;
+         }
+         current.append(line).append('\n');
+         if(line.trim().endsWith(";"))
+         {
+            statements.add(current.toString().trim().replaceAll(";$", ""));
+            current.setLength(0);
+         }
+      }
+      return (statements);
+   }
+
+
+
+   /*******************************************************************************
+    ** Run a changelog with real Liquibase (not closed: that would close the
+    ** test connection).
+    *******************************************************************************/
+   private static void update(Connection connection, String changelog) throws Exception
+   {
+      Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(connection));
+      new Liquibase(changelog, new ClassLoaderResourceAccessor(), database).update("");
+      connection.commit();
+   }
+
+
+
+   /*******************************************************************************
+    ** One value as text.
+    *******************************************************************************/
+   private static String scalar(Connection connection, String sql) throws Exception
+   {
+      try(Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sql))
+      {
+         return (resultSet.next() ? resultSet.getString(1) : null);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A classpath resource as text.
+    *******************************************************************************/
+   private static String text(String resource) throws Exception
+   {
+      try(InputStream stream = ClusterChangelogTest.class.getClassLoader().getResourceAsStream(resource))
+      {
+         assertThat(stream).as(resource).isNotNull();
+         return (new String(stream.readAllBytes(), StandardCharsets.UTF_8));
+      }
+   }
+}

src/test/java/com/kingsrook/qbits/cluster/ClusterDrainTest.java

@@ -0,0 +1,181 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qbits.cluster.processes.ClusterNodeActionSteps;
+import com.kingsrook.qbits.cluster.processes.ClusterNodeActions;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntime;
+import com.kingsrook.qbits.cluster.widgets.ClusterNodesWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
+import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
+import com.kingsrook.qqq.backend.core.cluster.QNodeLifecycle;
+import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Drain node and Cancel drain, end to end: the record action sets the
+ ** request, the node's heartbeat picks it up and calls
+ ** QNodeLifecycle.startDraining, and the row becomes DRAINING.
+ *******************************************************************************/
+class ClusterDrainTest extends BaseClusterTest
+{
+   private static final String DRAIN = "cluster.drain";
+
+
+
+   /*******************************************************************************
+    ** The request flows to QNodeLifecycle with who and why.
+    *******************************************************************************/
+   @Test
+   void drainRequestReachesTheLifecycle() throws Exception
+   {
+      ClusterRuntime runtime = runtime();
+      otherNode("node-b", NodeStatus.UP.name(), Instant.now(), "WEB");
+      Serializable id = node(NODE_A).getValue(ClusterTables.ID);
+
+      login("Pat Admin", DRAIN);
+      List<String> lines = runAction(ClusterNodeActions.DRAIN_NODE, List.of(id), Map.of(ClusterNodeActionSteps.FIELD_REASON, "kernel patch"));
+      assertThat(lines).anyMatch(line -> line.startsWith("OK: 1 ") && line.contains("asked to drain"));
+
+      QRecord requested = node(NODE_A);
+      assertThat(requested.getValueBoolean(ClusterTables.DRAIN_REQUESTED)).isTrue();
+      assertThat(requested.getValueString(ClusterTables.DRAIN_REQUESTED_BY)).isEqualTo("Pat Admin");
+      assertThat(requested.getValueString(ClusterTables.DRAIN_REASON)).isEqualTo("kernel patch");
+      assertThat(requested.getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+      assertThat(QNodeLifecycle.isDraining()).isFalse();
+
+      runtime.tick();
+      assertThat(QNodeLifecycle.isDraining()).isTrue();
+      assertThat(QNodeLifecycle.getDrainReason()).isEqualTo("requested by Pat Admin: kernel patch");
+      assertThat(node(NODE_A).getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.DRAINING.name());
+
+      lines = runAction(ClusterNodeActions.CANCEL_DRAIN, List.of(id), Map.of());
+      assertThat(lines).anyMatch(line -> line.contains("already started draining"));
+      assertThat(node(NODE_A).getValueBoolean(ClusterTables.DRAIN_REQUESTED)).isTrue();
+   }
+
+
+
+   /*******************************************************************************
+    ** Cancel before the node acts: the request is cleared and nothing drains.
+    *******************************************************************************/
+   @Test
+   void cancelBeforeTheNodeActs() throws Exception
+   {
+      ClusterRuntime runtime = runtime();
+      otherNode("node-b", NodeStatus.UP.name(), Instant.now(), "WEB");
+      Serializable id = node(NODE_A).getValue(ClusterTables.ID);
+      login("Pat Admin", DRAIN);
+      runAction(ClusterNodeActions.DRAIN_NODE, List.of(id), Map.of());
+      assertThat(node(NODE_A).getValueString(ClusterTables.DRAIN_REASON)).isNull();
+
+      List<String> lines = runAction(ClusterNodeActions.CANCEL_DRAIN, List.of(id), Map.of());
+      assertThat(lines).anyMatch(line -> line.startsWith("OK: 1 ") && line.contains("cancelled"));
+      assertThat(node(NODE_A).getValueBoolean(ClusterTables.DRAIN_REQUESTED)).isFalse();
+      assertThat(node(NODE_A).getValueString(ClusterTables.DRAIN_REQUESTED_BY)).isNull();
+
+      runtime.tick();
+      assertThat(QNodeLifecycle.isDraining()).isFalse();
+      assertThat(node(NODE_A).getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+   }
+
+
+
+   /*******************************************************************************
+    ** Only UP nodes without a request are drained; others are skipped, with why.
+    *******************************************************************************/
+   @Test
+   void skipsNodesThatCannotBeDrained() throws Exception
+   {
+      QRecord lost    = otherNode("node-lost", NodeStatus.LOST.name(), Instant.now(), "WEB");
+      QRecord stopped = otherNode("node-stopped", NodeStatus.STOPPED.name(), Instant.now(), "WEB");
+      QRecord up      = otherNode("node-up", NodeStatus.UP.name(), Instant.now(), "WEB");
+      login("Pat Admin", DRAIN);
+
+      List<String> lines = runAction(ClusterNodeActions.DRAIN_NODE, List.of(lost.getValue(ClusterTables.ID), stopped.getValue(ClusterTables.ID), up.getValue(ClusterTables.ID)), Map.of());
+      assertThat(lines).anyMatch(line -> line.startsWith("OK: 1 "));
+      assertThat(lines).anyMatch(line -> line.startsWith("INFO: 2 ") && line.contains("Only a node that is up"));
+      assertThat(node("node-lost").getValueBoolean(ClusterTables.DRAIN_REQUESTED)).isNotEqualTo(true);
+
+      lines = runAction(ClusterNodeActions.DRAIN_NODE, List.of(up.getValue(ClusterTables.ID)), Map.of());
+      assertThat(lines).anyMatch(line -> line.contains("already requested"));
+
+      lines = runAction(ClusterNodeActions.CANCEL_DRAIN, List.of(lost.getValue(ClusterTables.ID)), Map.of());
+      assertThat(lines).anyMatch(line -> line.contains("No drain was requested"));
+   }
+
+
+
+   /*******************************************************************************
+    ** The Review warns when no node would remain up.
+    *******************************************************************************/
+   @Test
+   void warnsWhenNoNodeWouldRemainUp() throws Exception
+   {
+      runtime();
+      login("Pat Admin", DRAIN);
+      Serializable id = node(NODE_A).getValue(ClusterTables.ID);
+      List<String> lines = new java.util.ArrayList<>();
+      ClusterNodeActionSteps.DrainTransform transform = new ClusterNodeActionSteps.DrainTransform();
+      com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput input = new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput();
+      input.setRecords(List.of(node(NODE_A)));
+      transform.runOnePage(input, new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput());
+      transform.getProcessSummary(null, false).forEach(line -> lines.add(String.valueOf(((com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine) line).getMessage())));
+      assertThat(lines).anyMatch(line -> line.contains("No other node is up"));
+      assertThat(id).isNotNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** The actions need the drain permission; the app and widget need read.
+    *******************************************************************************/
+   @Test
+   void permissions()
+   {
+      RunProcessInput input = new RunProcessInput();
+      login("Read Only", "cluster.read");
+      assertThat(PermissionsHelper.hasProcessPermission(input, ClusterNodeActions.DRAIN_NODE)).isFalse();
+      assertThat(PermissionsHelper.hasProcessPermission(input, ClusterNodeActions.CANCEL_DRAIN)).isFalse();
+      assertThat(PermissionsHelper.hasAppPermission(input, "cluster")).isTrue();
+      assertThat(PermissionsHelper.hasWidgetPermission(input, ClusterNodesWidgetRenderer.NAME)).isTrue();
+      assertThat(PermissionsHelper.hasTablePermission(input, ClusterTables.NODE, TablePermissionSubType.READ)).isTrue();
+      assertThat(PermissionsHelper.hasTablePermission(input, ClusterTables.SETTING, TablePermissionSubType.EDIT)).isFalse();
+
+      login("Operator", DRAIN);
+      assertThat(PermissionsHelper.hasProcessPermission(input, ClusterNodeActions.DRAIN_NODE)).isTrue();
+      assertThat(PermissionsHelper.hasAppPermission(input, "cluster")).isFalse();
+
+      login("Admin", "cluster.read", "cluster.write");
+      assertThat(PermissionsHelper.hasTablePermission(input, ClusterTables.SETTING, TablePermissionSubType.EDIT)).isTrue();
+   }
+}

src/test/java/com/kingsrook/qbits/cluster/ClusterLeasesTest.java

@@ -0,0 +1,185 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** Lease acquire, renew, expire, steal and release (memory backend; the same
+ ** compare-and-set runs as SQL in ClusterChangelogTest).
+ *******************************************************************************/
+class ClusterLeasesTest extends BaseClusterTest
+{
+   private static final Duration TTL = Duration.ofSeconds(60);
+
+
+
+   /*******************************************************************************
+    ** The full life of a lease between two nodes.
+    *******************************************************************************/
+   @Test
+   void acquireRenewExpireStealRelease() throws Exception
+   {
+      ClusterLeases.Holder a = ClusterLeases.as("node-a");
+      ClusterLeases.Holder b = ClusterLeases.as("node-b");
+
+      assertThat(a.tryAcquire("job", TTL)).isTrue();
+      assertThat(a.isLeader("job")).isTrue();
+      assertThat(lease().getValueInteger(ClusterTables.LEASE_VERSION)).isEqualTo(1);
+
+      assertThat(b.tryAcquire("job", TTL)).isFalse();
+      assertThat(b.isLeader("job")).isFalse();
+      assertThat(b.renew("job", TTL)).isFalse();
+      assertThat(b.release("job")).isFalse();
+
+      Instant firstExpiry = lease().getValueInstant(ClusterTables.EXPIRES_AT);
+      Instant acquired    = lease().getValueInstant(ClusterTables.ACQUIRED_AT);
+      Thread.sleep(5);
+      assertThat(a.renew("job", TTL)).isTrue();
+      assertThat(lease().getValueInstant(ClusterTables.EXPIRES_AT)).isAfter(firstExpiry);
+      assertThat(lease().getValueInstant(ClusterTables.ACQUIRED_AT)).isEqualTo(acquired);
+      assertThat(a.tryAcquire("job", TTL)).isTrue();
+      assertThat(lease().getValueInteger(ClusterTables.LEASE_VERSION)).isEqualTo(3);
+
+      expire();
+      assertThat(a.isLeader("job")).isFalse();
+      assertThat(b.tryAcquire("job", TTL)).isTrue();
+      assertThat(lease().getValueString(ClusterTables.HOLDER_NODE_ID)).isEqualTo("node-b");
+      assertThat(a.isLeader("job")).isFalse();
+      assertThat(a.renew("job", TTL)).isFalse();
+      assertThat(b.isLeader("job")).isTrue();
+
+      assertThat(b.release("job")).isTrue();
+      assertThat(lease().getValueString(ClusterTables.HOLDER_NODE_ID)).isNull();
+      assertThat(b.isLeader("job")).isFalse();
+      assertThat(a.tryAcquire("job", TTL)).isTrue();
+      assertThat(all(ClusterTables.LEASE)).hasSize(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** Many nodes race for an expired lease at once: exactly one wins.
+    *******************************************************************************/
+   @Test
+   void exactlyOneWinsARace() throws Exception
+   {
+      assertThat(ClusterLeases.as("node-a").tryAcquire("job", TTL)).isTrue();
+      expire();
+
+      int                  racers  = 12;
+      AtomicInteger        winners = new AtomicInteger();
+      CountDownLatch       go      = new CountDownLatch(1);
+      ExecutorService      pool    = Executors.newFixedThreadPool(racers);
+      List<Future<?>>      futures = new ArrayList<>();
+      for(int i = 0; i < racers; i++)
+      {
+         String nodeId = "racer-" + i;
+         futures.add(pool.submit(() ->
+         {
+            QContext.init(qInstance, new QSystemUserSession(qInstance));
+            go.await();
+            if(ClusterLeases.as(nodeId).tryAcquire("job", TTL))
+            {
+               winners.incrementAndGet();
+            }
+            return (null);
+         }));
+      }
+      go.countDown();
+      for(Future<?> future : futures)
+      {
+         future.get();
+      }
+      pool.shutdown();
+      assertThat(winners.get()).isEqualTo(1);
+      assertThat(lease().getValueString(ClusterTables.HOLDER_NODE_ID)).startsWith("racer-");
+      assertThat(lease().getValueInteger(ClusterTables.LEASE_VERSION)).isEqualTo(2);
+   }
+
+
+
+   /*******************************************************************************
+    ** The static API acts for this node.
+    *******************************************************************************/
+   @Test
+   void staticApiUsesThisNode() throws Exception
+   {
+      assertThat(ClusterLeases.tryAcquire("singleton", TTL)).isTrue();
+      assertThat(ClusterLeases.isLeader("singleton")).isTrue();
+      assertThat(lease().getValueString(ClusterTables.HOLDER_NODE_ID)).isEqualTo(NODE_A);
+      assertThat(ClusterLeases.renew("singleton", TTL)).isTrue();
+      assertThat(ClusterLeases.release("singleton")).isTrue();
+      assertThat(ClusterLeases.isLeader("singleton")).isFalse();
+   }
+
+
+
+   /*******************************************************************************
+    ** Bad arguments.
+    *******************************************************************************/
+   @Test
+   void badArguments()
+   {
+      assertThatThrownBy(() -> ClusterLeases.as(" ")).isInstanceOf(IllegalArgumentException.class);
+      assertThatThrownBy(() -> ClusterLeases.tryAcquire("x", Duration.ZERO)).isInstanceOf(IllegalArgumentException.class);
+      assertThatThrownBy(() -> ClusterLeases.renew("x", null)).isInstanceOf(IllegalArgumentException.class);
+   }
+
+
+
+   /*******************************************************************************
+    ** The one lease row.
+    *******************************************************************************/
+   private QRecord lease() throws Exception
+   {
+      return (all(ClusterTables.LEASE).get(0));
+   }
+
+
+
+   /*******************************************************************************
+    ** Let the lease run out.
+    *******************************************************************************/
+   private void expire() throws Exception
+   {
+      setFields(ClusterTables.LEASE, lease().getValue(ClusterTables.ID), Map.of(ClusterTables.EXPIRES_AT, Instant.now().minusSeconds(1)));
+   }
+}

src/test/java/com/kingsrook/qbits/cluster/ClusterQBitProducerTest.java

@@ -0,0 +1,174 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import com.kingsrook.qbits.cluster.health.ClusterHeartbeatHealthCheck;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qbits.cluster.processes.ClusterNodeActions;
+import com.kingsrook.qbits.cluster.runtime.ClusterAlerts;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntimeService;
+import com.kingsrook.qbits.cluster.widgets.ClusterNodesWidgetRenderer;
+import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
+import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
+import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
+import com.kingsrook.qqq.backend.core.instances.QInstanceValidator;
+import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType;
+import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfigValidationException;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability;
+import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
+import com.kingsrook.qqq.backend.core.testing.HelpBundleAssertions;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** The producer defines the tables, actions, widget, app, alert type, health
+ ** check, runtime service and help, and the result validates.
+ *******************************************************************************/
+class ClusterQBitProducerTest extends BaseClusterTest
+{
+
+   /*******************************************************************************
+    ** Everything is defined.
+    *******************************************************************************/
+   @Test
+   void definesEverything()
+   {
+      assertThat(qInstance.getTable(ClusterTables.NODE)).isNotNull();
+      assertThat(qInstance.getTable(ClusterTables.LEASE)).isNotNull();
+      assertThat(qInstance.getTable(ClusterTables.SETTING)).isNotNull();
+      assertThat(qInstance.getTable(ClusterTables.NODE).getFields()).containsKeys(ClusterTables.NODE_ID, ClusterTables.POD_NAME, ClusterTables.K8S_NODE_NAME,
+         ClusterTables.ZONE, ClusterTables.DEPLOYMENT, ClusterTables.ROLES, ClusterTables.VERSION, ClusterTables.STARTED_AT, ClusterTables.HEARTBEAT_AT,
+         ClusterTables.STATUS, ClusterTables.WORKER_THREADS, ClusterTables.SCHEDULER_ACTIVE, ClusterTables.ESB_TRIGGERS_ACTIVE, ClusterTables.ASYNC_JOBS_RUNNING,
+         ClusterTables.HEAP_USED_MB, ClusterTables.HEAP_MAX_MB, ClusterTables.DRAIN_REQUESTED, ClusterTables.DRAIN_REQUESTED_BY, ClusterTables.DRAIN_REASON,
+         ClusterTables.CREATE_DATE, ClusterTables.MODIFY_DATE);
+      assertThat(qInstance.getTable(ClusterTables.NODE).getField(ClusterTables.K8S_NODE_NAME).getBackendName()).isEqualTo("k8s_node_name");
+      assertThat(qInstance.getTable(ClusterTables.NODE).getField(ClusterTables.STATUS).getPossibleValueSourceName()).isEqualTo(NodeStatus.NAME);
+      assertThat(qInstance.getPossibleValueSource(NodeStatus.NAME)).isNotNull();
+      assertThat(qInstance.getTable(ClusterTables.NODE).isCapabilityEnabled(qInstance.getBackend(BACKEND), Capability.TABLE_INSERT)).isFalse();
+      assertThat(qInstance.getTable(ClusterTables.SETTING).isCapabilityEnabled(qInstance.getBackend(BACKEND), Capability.TABLE_UPDATE)).isTrue();
+
+      assertThat(qInstance.getProcess(ClusterNodeActions.DRAIN_NODE).getTableName()).isEqualTo(ClusterTables.NODE);
+      assertThat(qInstance.getProcess(ClusterNodeActions.CANCEL_DRAIN).getTableName()).isEqualTo(ClusterTables.NODE);
+      assertThat(qInstance.getWidget(ClusterNodesWidgetRenderer.NAME).getGridColumns()).isEqualTo(12);
+
+      QAppMetaData app = qInstance.getApp("cluster");
+      assertThat(app.getWidgets()).containsExactly(ClusterNodesWidgetRenderer.NAME);
+      assertThat(app.getChildren()).extracting(child -> child.getName()).containsExactly(ClusterTables.NODE, ClusterTables.LEASE, ClusterTables.SETTING);
+      assertThat(qInstance.getApp("admin").getChildren()).extracting(child -> child.getName()).contains("cluster");
+
+      assertThat(QAlertTypeMetaData.of(qInstance, ClusterAlerts.NODE_LOST)).isNotNull();
+      assertThat(HealthChecksMetaData.ofOrWithNew(qInstance).getChecks()).anyMatch(check -> ClusterHeartbeatHealthCheck.NAME.equals(check.getName()));
+      assertThat(qInstance.getRuntimeServices()).extracting(reference -> reference.getName()).containsOnlyOnce(ClusterRuntimeService.class.getName());
+      ////////////////////////////////////////////////////////////////////
+      // the version comes from the build (filtered qbit-cluster.properties) //
+      ////////////////////////////////////////////////////////////////////
+      assertThat(ClusterQBitProducer.version()).matches("[0-9]+[.][0-9]+[.][0-9]+(-[A-Za-z0-9.]+)?");
+      if(System.getProperty("qbit.artifactVersion") != null)
+      {
+         assertThat(ClusterQBitProducer.version()).isEqualTo(System.getProperty("qbit.artifactVersion"));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The produced metadata passes the instance validator.
+    *******************************************************************************/
+   @Test
+   void validates() throws Exception
+   {
+      new QInstanceValidator().validate(qInstance);
+   }
+
+
+
+   /*******************************************************************************
+    ** No processes in the navigation: the app lists only the three tables.
+    *******************************************************************************/
+   @Test
+   void actionsAreNotInTheNavigation()
+   {
+      List<String> appChildren = new ArrayList<>();
+      qInstance.getApps().values().forEach(app -> app.getChildren().forEach(child -> appChildren.add(child.getName())));
+      assertThat(appChildren).doesNotContain(ClusterNodeActions.DRAIN_NODE, ClusterNodeActions.CANCEL_DRAIN);
+   }
+
+
+
+   /*******************************************************************************
+    ** The shipped help covers every table, field, process, widget and possible
+    ** value source of the qbit.
+    *******************************************************************************/
+   @Test
+   void helpIsComplete()
+   {
+      Set<String> owned = Set.of("cluster", ClusterTables.NODE, ClusterTables.LEASE, ClusterTables.SETTING, ClusterNodeActions.DRAIN_NODE, ClusterNodeActions.CANCEL_DRAIN);
+      HelpBundleAssertions.forInstance(qInstance).bundles(ClusterQBitProducer.HELP_BUNDLE).owning(target -> owned.contains(target.name())).assertComplete();
+   }
+
+
+
+   /*******************************************************************************
+    ** A bad config fails before anything is added.
+    *******************************************************************************/
+   @Test
+   void invalidConfigFails()
+   {
+      QInstance instance = new QInstance();
+      instance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class));
+      assertThatThrownBy(() -> new ClusterQBitProducer().withQBitConfig(new ClusterQBitConfig()).produce(instance)).isInstanceOf(QBitConfigValidationException.class);
+      assertThatThrownBy(() -> new ClusterQBitProducer().withQBitConfig(new ClusterQBitConfig().withBackendName(BACKEND).withParentAppName("nope")).produce(instance))
+         .isInstanceOf(QBitConfigValidationException.class);
+      assertThatThrownBy(() -> new ClusterQBitProducer().produce(instance)).hasMessageContaining("required");
+      assertThat(instance.getTable(ClusterTables.NODE)).isNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** Without a parent app, placement is left to the host; the runtime service
+    ** can be left out.
+    *******************************************************************************/
+   @Test
+   void noParentNoService() throws Exception
+   {
+      QInstance instance = new QInstance();
+      instance.addBackend(new QBackendMetaData().withName(BACKEND).withBackendType(MemoryBackendModule.class));
+      instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("mock").withType(QAuthenticationType.MOCK));
+      new ClusterQBitProducer().withQBitConfig(new ClusterQBitConfig().withBackendName(BACKEND).withAppName("nodes").withRegisterRuntimeService(false)).produce(instance);
+      new QInstanceEnricher(instance).enrich();
+      assertThat(instance.getApp("nodes")).isNotNull();
+      assertThat(instance.getRuntimeServices()).noneMatch(reference -> ClusterRuntimeService.class.getName().equals(reference.getName()));
+      assertThat(new ClusterQBitProducer().getNamespace()).isEqualTo("cluster");
+   }
+}

src/test/java/com/kingsrook/qbits/cluster/ClusterRegistryTest.java

@@ -0,0 +1,208 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Map;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qbits.cluster.runtime.ClusterAlerts;
+import com.kingsrook.qbits.cluster.runtime.ClusterNodeInfo;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntime;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntimeService;
+import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
+import com.kingsrook.qqq.backend.core.cluster.QNodeLifecycle;
+import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** The node row: register (upsert), heartbeat, status changes, stop.
+ *******************************************************************************/
+class ClusterRegistryTest extends BaseClusterTest
+{
+
+   /*******************************************************************************
+    ** Start inserts the row with the environment, roles and live fields.
+    *******************************************************************************/
+   @Test
+   void registerInsertsTheRow() throws Exception
+   {
+      runtime(Map.of(ClusterNodeInfo.ENV_POD_NAME, "web-7f9-abc", ClusterNodeInfo.ENV_K8S_NODE, "k8s-wk-21", ClusterNodeInfo.ENV_ZONE, "galaxy",
+         ClusterNodeInfo.ENV_DEPLOYMENT, "website-backend"));
+
+      QRecord node = node(NODE_A);
+      assertThat(node.getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+      assertThat(node.getValueString(ClusterTables.POD_NAME)).isEqualTo("web-7f9-abc");
+      assertThat(node.getValueString(ClusterTables.K8S_NODE_NAME)).isEqualTo("k8s-wk-21");
+      assertThat(node.getValueString(ClusterTables.ZONE)).isEqualTo("galaxy");
+      assertThat(node.getValueString(ClusterTables.DEPLOYMENT)).isEqualTo("website-backend");
+      assertThat(node.getValueString(ClusterTables.ROLES)).isEqualTo("WEB,WORKER,SCHEDULER,ESB");
+      assertThat(node.getValueString(ClusterTables.VERSION)).isEqualTo("1.2.3");
+      assertThat(node.getValueInteger(ClusterTables.WORKER_THREADS)).isEqualTo(4);
+      assertThat(node.getValueBoolean(ClusterTables.SCHEDULER_ACTIVE)).isTrue();
+      assertThat(node.getValueInteger(ClusterTables.ASYNC_JOBS_RUNNING)).isZero();
+      assertThat(node.getValueInteger(ClusterTables.HEAP_USED_MB)).isPositive();
+      assertThat(node.getValueInstant(ClusterTables.STARTED_AT)).isNotNull();
+      assertThat(node.getValueInstant(ClusterTables.HEARTBEAT_AT)).isNotNull();
+      assertThat(node.getValueBoolean(ClusterTables.DRAIN_REQUESTED)).isFalse();
+   }
+
+
+
+   /*******************************************************************************
+    ** A restart with the same node id updates the one row and clears an old
+    ** drain request; no duplicate.
+    *******************************************************************************/
+   @Test
+   void restartUpdatesTheSameRow() throws Exception
+   {
+      QRecord old = otherNode(NODE_A, NodeStatus.STOPPED.name(), Instant.now().minus(1, ChronoUnit.HOURS), "WEB");
+      setFields(ClusterTables.NODE, old.getValue(ClusterTables.ID), Map.of(ClusterTables.DRAIN_REQUESTED, true, ClusterTables.DRAIN_REQUESTED_BY, "Pat"));
+
+      runtime();
+      assertThat(all(ClusterTables.NODE)).hasSize(1);
+      QRecord node = node(NODE_A);
+      assertThat(node.getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+      assertThat(node.getValueBoolean(ClusterTables.DRAIN_REQUESTED)).isFalse();
+      assertThat(node.getValueString(ClusterTables.DRAIN_REQUESTED_BY)).isNull();
+      assertThat(node.getValueInstant(ClusterTables.STARTED_AT)).isAfter(Instant.now().minus(1, ChronoUnit.MINUTES));
+   }
+
+
+
+   /*******************************************************************************
+    ** A heartbeat moves heartbeat_at forward and keeps the row UP.
+    *******************************************************************************/
+   @Test
+   void heartbeatUpdatesTheRow() throws Exception
+   {
+      ClusterRuntime runtime = runtime();
+      QRecord        node    = node(NODE_A);
+      Instant        old     = Instant.now().minus(30, ChronoUnit.SECONDS);
+      setFields(ClusterTables.NODE, node.getValue(ClusterTables.ID), Map.of(ClusterTables.HEARTBEAT_AT, old, ClusterTables.HEAP_USED_MB, 0));
+
+      runtime.tick();
+      QRecord after = node(NODE_A);
+      assertThat(after.getValueInstant(ClusterTables.HEARTBEAT_AT)).isAfter(old);
+      assertThat(after.getValueInteger(ClusterTables.HEAP_USED_MB)).isPositive();
+      assertThat(after.getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+   }
+
+
+
+   /*******************************************************************************
+    ** A deleted row comes back on the next heartbeat.
+    *******************************************************************************/
+   @Test
+   void missingRowIsRegisteredAgain() throws Exception
+   {
+      ClusterRuntime runtime = runtime();
+      new DeleteAction().execute(new DeleteInput(ClusterTables.NODE).withPrimaryKeys(java.util.List.of(node(NODE_A).getValue(ClusterTables.ID))));
+      assertThat(node(NODE_A)).isNull();
+      runtime.tick();
+      assertThat(node(NODE_A)).isNotNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** A node marked LOST that beats again is UP and its alert resolved.
+    *******************************************************************************/
+   @Test
+   void lostNodeThatBeatsAgainRecovers() throws Exception
+   {
+      ClusterRuntime runtime = runtime();
+      setFields(ClusterTables.NODE, node(NODE_A).getValue(ClusterTables.ID), Map.of(ClusterTables.STATUS, NodeStatus.LOST.name()));
+      runtime.tick();
+      assertThat(node(NODE_A).getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+      assertThat(RecordingAlertSink.RESOLVED).anyMatch(r -> ClusterAlerts.NODE_LOST.equals(r.getAlertTypeName()) && NODE_A.equals(r.getDedupKey()));
+   }
+
+
+
+   /*******************************************************************************
+    ** A drain from anywhere (SIGTERM) marks DRAINING; stop marks STOPPED and
+    ** heartbeats stop.
+    *******************************************************************************/
+   @Test
+   void drainThenStop() throws Exception
+   {
+      ClusterRuntime runtime = runtime();
+      QNodeLifecycle.startDraining("shutdown");
+      assertThat(node(NODE_A).getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.DRAINING.name());
+
+      runtime.tick();
+      assertThat(node(NODE_A).getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.DRAINING.name());
+
+      runtime.stop();
+      QRecord stopped = node(NODE_A);
+      assertThat(stopped.getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.STOPPED.name());
+      assertThat(runtime.getRegistry().isStopped()).isTrue();
+      assertThat(ClusterRuntime.current()).isNull();
+
+      Instant at = stopped.getValueInstant(ClusterTables.HEARTBEAT_AT);
+      runtime.tick();
+      runtime.stop();
+      assertThat(node(NODE_A).getValueInstant(ClusterTables.HEARTBEAT_AT)).isEqualTo(at);
+   }
+
+
+
+   /*******************************************************************************
+    ** The runtime service finds the config, starts a heartbeat thread and stops.
+    *******************************************************************************/
+   @Test
+   void runtimeServiceStartsAndStops() throws Exception
+   {
+      ClusterRuntimeService service = new ClusterRuntimeService();
+      assertThat(service.getName()).isEqualTo("cluster");
+      service.start(qInstance);
+      assertThat(service.getRuntime()).isNotNull();
+      assertThat(ClusterRuntime.current()).isSameAs(service.getRuntime());
+      assertThat(node(NODE_A).getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+      service.stop();
+      assertThat(node(NODE_A).getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.STOPPED.name());
+
+      ClusterRuntimeService without = new ClusterRuntimeService();
+      without.start(new com.kingsrook.qqq.backend.core.model.metadata.QInstance());
+      without.stop();
+      assertThat(without.getRuntime()).isNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** The version falls back to QQQ_APP_VERSION, then the QQQ version.
+    *******************************************************************************/
+   @Test
+   void versionFallbacks()
+   {
+      ClusterQBitConfig noVersion = new ClusterQBitConfig().withBackendName(BACKEND);
+      assertThat(new ClusterNodeInfo(noVersion, Map.of(ClusterNodeInfo.ENV_APP_VERSION, "9.9.9")::get).version()).isEqualTo("9.9.9");
+      assertThat(new ClusterNodeInfo(noVersion, name -> null).version()).startsWith("qqq ");
+   }
+}

src/test/java/com/kingsrook/qbits/cluster/ClusterSettingsWidgetHealthTest.java

@@ -0,0 +1,164 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.cluster.health.ClusterHeartbeatHealthCheck;
+import com.kingsrook.qbits.cluster.model.ClusterSettings;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntime;
+import com.kingsrook.qbits.cluster.widgets.ClusterNodesWidgetRenderer;
+import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
+import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
+import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
+import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
+import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
+import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
+import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Settings (defaults, table values, validation), the Nodes widget and the
+ ** heartbeat health check.
+ *******************************************************************************/
+class ClusterSettingsWidgetHealthTest extends BaseClusterTest
+{
+
+   /*******************************************************************************
+    ** Defaults, values from the table, and bad values falling back.
+    *******************************************************************************/
+   @Test
+   void settings() throws Exception
+   {
+      assertThat(ClusterSettings.load()).isEqualTo(ClusterSettings.defaultSettings());
+      assertThat(ClusterSettings.defaultSettings()).isEqualTo(new ClusterSettings(15, 60, 7, true));
+      ClusterSettings.DEFAULTS.forEach((key, valueAndDescription) -> assertThat(ClusterSettings.validate(key, valueAndDescription.get(0))).isNull());
+
+      insert(ClusterTables.SETTING, new QRecord().withValue(ClusterTables.SETTING_KEY, ClusterSettings.HEARTBEAT_SECONDS).withValue(ClusterTables.SETTING_VALUE, "5"));
+      insert(ClusterTables.SETTING, new QRecord().withValue(ClusterTables.SETTING_KEY, ClusterSettings.ALERT_ON_LOST).withValue(ClusterTables.SETTING_VALUE, "FALSE"));
+      assertThat(ClusterSettings.load()).isEqualTo(new ClusterSettings(5, 60, 7, false));
+
+      assertThat(ClusterSettings.from(Map.of(ClusterSettings.HEARTBEAT_SECONDS, "30", ClusterSettings.LOST_AFTER_SECONDS, "20")).lostAfterSeconds()).isEqualTo(90);
+      assertThat(ClusterSettings.from(Map.of(ClusterSettings.RETAIN_STOPPED_DAYS, "x", ClusterSettings.ALERT_ON_LOST, "maybe"))).isEqualTo(ClusterSettings.defaultSettings());
+      assertThat(ClusterSettings.from(Map.of(ClusterSettings.HEARTBEAT_SECONDS, "0")).heartbeatSeconds()).isEqualTo(15);
+   }
+
+
+
+   /*******************************************************************************
+    ** The table rejects values the runtime could not use, on insert and update.
+    *******************************************************************************/
+   @Test
+   void settingValuesAreValidated() throws Exception
+   {
+      QRecord bad = new InsertAction().execute(new InsertInput(ClusterTables.SETTING).withRecord(new QRecord()
+         .withValue(ClusterTables.SETTING_KEY, ClusterSettings.LOST_AFTER_SECONDS).withValue(ClusterTables.SETTING_VALUE, "soon"))).getRecords().get(0);
+      assertThat(bad.getErrors()).isNotEmpty();
+      assertThat(all(ClusterTables.SETTING)).isEmpty();
+
+      QRecord custom = insert(ClusterTables.SETTING, new QRecord().withValue(ClusterTables.SETTING_KEY, "my_note").withValue(ClusterTables.SETTING_VALUE, "anything"));
+      assertThat(custom.getErrors()).isNullOrEmpty();
+
+      QRecord good = insert(ClusterTables.SETTING, new QRecord().withValue(ClusterTables.SETTING_KEY, ClusterSettings.ALERT_ON_LOST).withValue(ClusterTables.SETTING_VALUE, "true"));
+      QRecord updated = new UpdateAction().execute(new UpdateInput(ClusterTables.SETTING).withRecord(new QRecord()
+         .withValue(ClusterTables.ID, good.getValue(ClusterTables.ID)).withValue(ClusterTables.SETTING_VALUE, "yes"))).getRecords().get(0);
+      assertThat(updated.getErrors()).isNotEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** The widget lists live nodes by role (a node under each of its roles),
+    ** leaves stopped ones out, and shows the heartbeat age.
+    *******************************************************************************/
+   @Test
+   void widgetGroupsNodesByRole() throws Exception
+   {
+      Instant now = Instant.now();
+      otherNode("web-1", NodeStatus.UP.name(), now.minus(5, ChronoUnit.SECONDS), "WEB");
+      otherNode("worker-1", NodeStatus.UP.name(), now.minus(3, ChronoUnit.MINUTES), "WORKER,SCHEDULER,ESB");
+      otherNode("gone", NodeStatus.STOPPED.name(), now, "WEB");
+      QRecord lost = otherNode("lost-1", NodeStatus.LOST.name(), now.minus(5, ChronoUnit.HOURS), "WEB");
+      setFields(ClusterTables.NODE, otherNode("web-2", NodeStatus.UP.name(), now, "WEB").getValue(ClusterTables.ID), Map.of(ClusterTables.DRAIN_REQUESTED, true));
+
+      login("Viewer", "cluster.read");
+      TableData table = (TableData) new ClusterNodesWidgetRenderer().render(new RenderWidgetInput()).getWidgetData();
+      List<Map<String, Object>> rows = table.getRows();
+      assertThat(rows).extracting(row -> row.get("role") + "/" + row.get("node"))
+         .containsExactly("Web/lost-1", "Web/web-1", "Web/web-2", "Worker/worker-1", "Scheduler/worker-1", "ESB/worker-1");
+      assertThat(rows.get(0).get("status")).isEqualTo("Lost");
+      assertThat(rows.get(0).get("heartbeat")).isEqualTo("5 hours ago");
+      assertThat(rows.get(2).get("status")).isEqualTo("Up, drain requested");
+      assertThat(rows.get(3).get("heartbeat")).isEqualTo("3 minutes ago");
+      assertThat(table.getColumns()).as("four columns: readable at every width").hasSize(4);
+      assertThat(lost).isNotNull();
+
+      assertThat(ClusterNodesWidgetRenderer.class).isNotNull();
+   }
+
+
+
+   /*******************************************************************************
+    ** The health check: unknown without a runtime, up when current, degraded
+    ** when late, down when lost or missing.
+    *******************************************************************************/
+   @Test
+   void healthCheck() throws Exception
+   {
+      ClusterHeartbeatHealthCheck check = new ClusterHeartbeatHealthCheck();
+      HealthCheckInput            input = new HealthCheckInput().withQInstance(qInstance).withDeadline(Instant.now().plusSeconds(30));
+      assertThat(check.run(input).getStatus().name()).isEqualTo("UNKNOWN");
+
+      ClusterRuntime runtime = runtime();
+      assertThat(status(check.run(input))).isEqualTo("UP");
+
+      Object id = node(NODE_A).getValue(ClusterTables.ID);
+      setFields(ClusterTables.NODE, (java.io.Serializable) id, Map.of(ClusterTables.HEARTBEAT_AT, Instant.now().minusSeconds(45)));
+      assertThat(status(check.run(input))).isEqualTo("DEGRADED");
+
+      setFields(ClusterTables.NODE, (java.io.Serializable) id, Map.of(ClusterTables.HEARTBEAT_AT, Instant.now().minusSeconds(600)));
+      assertThat(status(check.run(input))).isEqualTo("DOWN");
+
+      runtime.tick();
+      assertThat(status(check.run(input))).isEqualTo("UP");
+      runtime.stop();
+   }
+
+
+
+   /*******************************************************************************
+    ** An outcome's status name.
+    *******************************************************************************/
+   private static String status(HealthCheckOutcome outcome)
+   {
+      return (outcome.getStatus().name());
+   }
+}

src/test/java/com/kingsrook/qbits/cluster/ClusterSweeperTest.java

@@ -0,0 +1,136 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.Map;
+import com.kingsrook.qbits.cluster.model.ClusterSettings;
+import com.kingsrook.qbits.cluster.model.ClusterTables;
+import com.kingsrook.qbits.cluster.model.NodeStatus;
+import com.kingsrook.qbits.cluster.runtime.ClusterAlerts;
+import com.kingsrook.qbits.cluster.runtime.ClusterRuntime;
+import com.kingsrook.qbits.cluster.runtime.ClusterSweeper;
+import com.kingsrook.qqq.backend.core.model.data.QRecord;
+import org.junit.jupiter.api.Test;
+import static org.assertj.core.api.Assertions.assertThat;
+
+
+/*******************************************************************************
+ ** Lost-node detection, the alert, and the purge of old rows.
+ *******************************************************************************/
+class ClusterSweeperTest extends BaseClusterTest
+{
+
+   /*******************************************************************************
+    ** A silent node becomes LOST and raises the alert; a fresh one stays UP;
+    ** a second sweep changes nothing more.
+    *******************************************************************************/
+   @Test
+   void staleNodeBecomesLost() throws Exception
+   {
+      Instant now = Instant.now();
+      otherNode("node-b", NodeStatus.UP.name(), now.minus(5, ChronoUnit.MINUTES), "WEB");
+      otherNode("node-c", NodeStatus.DRAINING.name(), now.minus(2, ChronoUnit.MINUTES), "WORKER");
+      otherNode("node-d", NodeStatus.UP.name(), now.minus(10, ChronoUnit.SECONDS), "WEB");
+
+      ClusterSweeper.Result result = new ClusterSweeper(qInstance).sweep(ClusterSettings.defaultSettings());
+      assertThat(result.lostNodeIds()).containsExactlyInAnyOrder("node-b", "node-c");
+      assertThat(node("node-b").getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.LOST.name());
+      assertThat(node("node-c").getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.LOST.name());
+      assertThat(node("node-d").getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+
+      assertThat(RecordingAlertSink.RAISED).hasSize(2);
+      assertThat(RecordingAlertSink.RAISED).allMatch(signal -> ClusterAlerts.NODE_LOST.equals(signal.getAlertTypeName()));
+      assertThat(RecordingAlertSink.RAISED).extracting(signal -> signal.getDedupKey()).containsExactlyInAnyOrder("node-b", "node-c");
+      assertThat(RecordingAlertSink.RAISED.get(0).getRecordRefs()).hasSize(1);
+
+      assertThat(new ClusterSweeper(qInstance).sweep(ClusterSettings.defaultSettings()).lostNodeIds()).isEmpty();
+      assertThat(RecordingAlertSink.RAISED).hasSize(2);
+   }
+
+
+
+   /*******************************************************************************
+    ** alert_on_lost = false marks LOST without an alert.
+    *******************************************************************************/
+   @Test
+   void noAlertWhenTurnedOff() throws Exception
+   {
+      otherNode("node-b", NodeStatus.UP.name(), Instant.now().minus(5, ChronoUnit.MINUTES), "WEB");
+      new ClusterSweeper(qInstance).sweep(ClusterSettings.from(Map.of(ClusterSettings.ALERT_ON_LOST, "false")));
+      assertThat(node("node-b").getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.LOST.name());
+      assertThat(RecordingAlertSink.RAISED).isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** STOPPED and LOST rows older than retain_stopped_days are deleted (the lost
+    ** one's alert resolved); younger ones and live ones stay.
+    *******************************************************************************/
+   @Test
+   void purgesOldRows() throws Exception
+   {
+      Instant now = Instant.now();
+      otherNode("old-stopped", NodeStatus.STOPPED.name(), now.minus(8, ChronoUnit.DAYS), "WEB");
+      otherNode("old-lost", NodeStatus.LOST.name(), now.minus(9, ChronoUnit.DAYS), "WEB");
+      otherNode("young-stopped", NodeStatus.STOPPED.name(), now.minus(12, ChronoUnit.HOURS), "WEB");
+      otherNode("live", NodeStatus.UP.name(), now, "WEB");
+
+      ClusterSweeper.Result result = new ClusterSweeper(qInstance).sweep(ClusterSettings.defaultSettings());
+      assertThat(result.purged()).isEqualTo(2);
+      assertThat(all(ClusterTables.NODE)).extracting(r -> r.getValueString(ClusterTables.NODE_ID)).containsExactlyInAnyOrder("young-stopped", "live");
+      assertThat(RecordingAlertSink.RESOLVED).anyMatch(r -> "old-lost".equals(r.getDedupKey()));
+
+      assertThat(new ClusterSweeper(qInstance).sweep(ClusterSettings.from(Map.of(ClusterSettings.RETAIN_STOPPED_DAYS, "1"))).purged()).isEqualTo(0);
+      setFields(ClusterTables.NODE, node("young-stopped").getValue(ClusterTables.ID), Map.of(ClusterTables.HEARTBEAT_AT, now.minus(2, ChronoUnit.DAYS)));
+      assertThat(new ClusterSweeper(qInstance).sweep(ClusterSettings.from(Map.of(ClusterSettings.RETAIN_STOPPED_DAYS, "1"))).purged()).isEqualTo(1);
+   }
+
+
+
+   /*******************************************************************************
+    ** The tick sweeps on the node that holds the sweeper lease, and only there.
+    *******************************************************************************/
+   @Test
+   void onlyTheLeaderSweeps() throws Exception
+   {
+      ClusterRuntime runtime = runtime();
+      insert(ClusterTables.LEASE, new QRecord().withValue(ClusterTables.NAME, ClusterSweeper.LEASE_NAME).withValue(ClusterTables.HOLDER_NODE_ID, "node-z")
+         .withValue(ClusterTables.EXPIRES_AT, Instant.now().plus(1, ChronoUnit.HOURS)).withValue(ClusterTables.LEASE_VERSION, 1));
+      otherNode("node-b", NodeStatus.UP.name(), Instant.now().minus(5, ChronoUnit.MINUTES), "WEB");
+
+      runtime.tick();
+      assertThat(node("node-b").getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.UP.name());
+
+      QRecord lease = all(ClusterTables.LEASE).get(0);
+      setFields(ClusterTables.LEASE, lease.getValue(ClusterTables.ID), Map.of(ClusterTables.EXPIRES_AT, Instant.now().minus(1, ChronoUnit.SECONDS)));
+      runtime.tick();
+      assertThat(node("node-b").getValueString(ClusterTables.STATUS)).isEqualTo(NodeStatus.LOST.name());
+      assertThat(all(ClusterTables.LEASE).get(0).getValueString(ClusterTables.HOLDER_NODE_ID)).isEqualTo(NODE_A);
+
+      runtime.stop();
+      assertThat(all(ClusterTables.LEASE).get(0).getValueString(ClusterTables.HOLDER_NODE_ID)).isNull();
+   }
+}

src/test/java/com/kingsrook/qbits/cluster/RecordingAlertSink.java

@@ -0,0 +1,83 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2026.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package com.kingsrook.qbits.cluster;
+
+
+import java.time.Instant;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import com.kingsrook.qqq.backend.core.alerts.AlertResolution;
+import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
+import com.kingsrook.qqq.backend.core.alerts.AlertSinkInterface;
+
+
+/*******************************************************************************
+ ** An alerts sink that records what it receives (tests).
+ *******************************************************************************/
+public class RecordingAlertSink implements AlertSinkInterface
+{
+   public static final List<AlertSignal>     RAISED   = new CopyOnWriteArrayList<>();
+   public static final List<AlertResolution> RESOLVED = new CopyOnWriteArrayList<>();
+
+
+
+   /*******************************************************************************
+    ** Forget everything.
+    *******************************************************************************/
+   public static void clear()
+   {
+      RAISED.clear();
+      RESOLVED.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a raise.
+    *******************************************************************************/
+   @Override
+   public void raise(AlertSignal signal)
+   {
+      RAISED.add(signal);
+   }
+
+
+
+   /*******************************************************************************
+    ** Record a resolve.
+    *******************************************************************************/
+   @Override
+   public void resolve(AlertResolution resolution)
+   {
+      RESOLVED.add(resolution);
+   }
+
+
+
+   /*******************************************************************************
+    ** Ignore heartbeats.
+    *******************************************************************************/
+   @Override
+   public void heartbeat(String heartbeatKey, Instant at)
+   {
+   }
+}