Compare commits

..
8 Commits
Author SHA1 Message Date
forge-recovery 32e6146e08 ci: enroll shared Gitea lifecycle
QRun-IO/ci-policy: ci / fork-guard (push) Skipped
QRun-IO/ci-policy: ci / head-profile (push) Skipped
QRun-IO/ci-policy: ci / application-template-0 (push) Skipped
QRun-IO/ci-policy: ci / gradle-1 (push) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (push) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (push) Skipped
QRun-IO/ci-policy: ci / node-8 (push) Skipped
QRun-IO/ci-policy: ci / node-9 (push) Skipped
QRun-IO/ci-policy: ci / node-10 (push) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (push) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (push) Skipped
QRun-IO/ci-policy: ci / source-config-13 (push) Skipped
QRun-IO/ci-policy: ci / source-config-14 (push) Skipped
QRun-IO/ci-policy: ci / android-0 (push) Skipped
QRun-IO/ci-policy: ci / original-mcp (push) Skipped
QRun-IO/ci-policy: ci / not-enrolled (push) Skipped
QRun-IO/ci-policy: native / fork-deny (push) Skipped
QRun-IO/ci-policy: native / dotnet-0 (push) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 27s
QRun-IO/ci-policy: native / native-result (push) Successful in 8s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (push) Successful in 26s
QRun-IO/ci-policy: ci / repository-policy (push) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (push) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (push) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (push) Skipped
Shared Java Maven CI with Docker / verify (push) Successful in 1m38s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (push) Successful in 1m54s
Shared Java Maven CI with Docker / gate (push) Successful in 21s
qrun-release / release (push) Successful in 23m31s
qrun-release / postpublish (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (push) Successful in 19s
QRun-IO/ci-policy: ci / java-docker-3 (push) Successful in 23m32s
QRun-IO/ci-policy: ci / ci-result (push) Successful in 13s
2026-11-02 22:11:14 +00:00
18b3b64839 ci: add the isolated postpublish job to the QRun release caller
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / head-profile (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-10 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-11 (pull_request) Skipped
QRun-IO/ci-policy: ci / repository-policy-12 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-13 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-14 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
Shared Java Maven CI with Docker / secrets (push) Successful in 13s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 12s
QRun-IO/ci-policy: ci / repository-policy (pull_request) Skipped
QRun-IO/ci-policy: ci / java-core-2-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-docker-3-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-6-head (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-7-head (pull_request) Skipped
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 4s
Shared Java Maven CI with Docker / verify (push) Successful in 1m17s
Shared Java Maven CI with Docker / gate (push) Successful in 10s
qrun-release / release (push) Successful in 2h34m17s
qrun-release / postpublish (push) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 1m11s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 11s
QRun-IO/ci-policy: ci / java-docker-3 (pull_request) Successful in 2h35m28s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 6s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 15:47:02 +00:00
4993c0f837 ci: move the QRun release caller to the thin v2 contract
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / application-template-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-1 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-frontend-browser-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-6 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-7 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-playwright-8 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-9 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: native / fork-deny (pull_request) Skipped
QRun-IO/ci-policy: native / dotnet-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / ci-result (pull_request) Failing after 3s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Canceled after 0s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Canceled after 0s
QRun-IO/ci-policy: ci / java-docker-2 (pull_request) Canceled after 1h22m31s
Shared Java Maven CI with Docker / verify (push) Canceled after 0s
qrun-release / release (push) Canceled after 1h22m35s
Shared Java Maven CI with Docker / gate (push) Canceled after 0s
Shared Java Maven CI with Docker / secrets (push) Successful in 23s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 22s
QRun-IO/ci-policy: native / native-result (pull_request) Successful in 8s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 06:16:36 +00:00
forge-recoveryandCodex 710f480549 ci: add the identical QRun release caller
QRun-IO/ci-policy: ci / fork-guard (pull_request) Skipped
QRun-IO/ci-policy: ci / gradle-0 (pull_request) Skipped
QRun-IO/ci-policy: ci / node-3 (pull_request) Skipped
QRun-IO/ci-policy: ci / java-maven-2 (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / secrets (pull_request) Successful in 12s
QRun-IO/ci-policy: ci / node-4 (pull_request) Skipped
QRun-IO/ci-policy: ci / source-config-5 (pull_request) Skipped
QRun-IO/ci-policy: ci / not-enrolled (pull_request) Skipped
QRun-IO/ci-policy: Shared Java Maven CI with Docker / verify (pull_request) Successful in 1m19s
QRun-IO/ci-policy: Shared Java Maven CI with Docker / gate (pull_request) Successful in 12s
QRun-IO/ci-policy: ci / java-docker-1 (pull_request) Successful in 14m10s
QRun-IO/ci-policy: ci / ci-result (pull_request) Successful in 5s
Co-Authored-By: Codex <noreply@openai.com>
2026-11-02 03:26:03 +00:00
james.maesandCodex 5f87b54ae3 ci: enroll in the shared QQQ orb (#41)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 19:50:20 -05:00
james.maesandCodex 2d24bf8bf2 ci: use shared orb pipelines and source scanning (#36)
Co-authored-by: Codex <noreply@openai.com>
2026-10-21 10:43:00 -05:00
james.maes ba23ccebbe Merge pull request #11 from QRun-IO/develop
release: bring 0.1.1 (TagRuleEngine PVS fix) into main
2026-10-12 20:09:19 -05:00
CircleCI 29a39a491c Bump version to 0.1.0 [skip ci] 2026-10-12 21:31:19 +00:00
23 changed files with 158 additions and 902 deletions

No files matched your search

+11 -57
View File
@@ -1,60 +1,14 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.6.9
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
qqq-orb: kingsrook/qqq-orb@0.8
workflows:
test_only:
jobs:
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
filters:
branches:
ignore: /(develop|release\/.*)/
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: snapshot
filters:
branches:
only: [develop]
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release_candidate
filters:
branches:
only: [/release\/.*/]
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [qqq-maven-registry-credentials]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [/^v\d+\.\d+\.\d+$/]
setup:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
+25
View File
@@ -0,0 +1,25 @@
# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
name: qrun-release
on:
push: {}
workflow_dispatch: {}
permissions:
contents: read
actions: read
jobs:
release:
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1
with:
runner: linux-small
docker-runner: container-build
browsers: true
browser-runner: browser-build
postpublish:
if: ${{ needs.release.outputs.capture == 'true' }}
needs: release
permissions:
contents: write
actions: write
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
with:
runner: linux-small
+23
View File
@@ -0,0 +1,23 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
-25
View File
@@ -1,25 +0,0 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
on:
workflow_dispatch:
inputs:
dry_run:
description: Print the plan without merging or creating branches
type: boolean
default: false
pull_request:
types: [closed]
branches: [develop]
permissions:
contents: write
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+16
View File
@@ -0,0 +1,16 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release-no-hotfix
publish:
snapshot: true
release_candidate: true
release: true
hotfix: true
asciidoc: false
+1 -1
View File
@@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository.
## Dependencies
- Java 21.
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.5` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.4` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
- Tests: see pom.xml.
## Release
-7
View File
@@ -4,14 +4,7 @@ All notable changes to this project are documented here. The format follows Keep
## [0.1.1] - Unreleased
### Changed
- Require released QQQ BOM 4.1.0-RC.5; revalidation hooks default to core Standard Webhooks signing. Per-hook `signatureFormat=LEGACY` preserves the prior header and signature bytes.
- Use core `SafeHttpClient` for revalidation requests: HTTPS/public targets by default, explicit `allowPlainHttp`/`internalAllowlist` exceptions, bounded responses and checked DNS pinning. Fan-out preserves Host/TLS identity. Host schema migration and receiver migration are described in README and `docs/revalidation-hook-migration.md`.
### Fixed
- Tag rules accept ignored fields in `onlyWhenFieldsChange`, such as `!viewCount`, so counter-only updates do not purge rendered pages. Plain allow-lists still work, and ignored fields take precedence in mixed lists. No schema change is needed.
- Purge actions warn when providers are NONE or hostnames are OFF, and report requests as recorded but not sent. The home explains recorded-only purges and that this version does not run origin header checks.
- Tag rule `{prefix.field}` placeholders now resolve the joined table from the `<prefix>Id` field's possible-value source, so prefixed tables (`cms_blog`) work; a missing joined table is logged instead of silently skipped.
## [0.1.0] - Unreleased
+1 -60
View File
@@ -31,21 +31,7 @@ Design: QRun-IO/qqq `docs/specs/2026-10-12-qbit-cdn-design.md`.
NONE. See `docs/cloudfront-expected-config.md`.
- **Permissions:** `cdn.view`, `cdn.purge`, `cdn.purgeAll`, `cdn.admin`.
## Tag rules
`cdn_table_tag_rule.onlyWhenFieldsChange` is a comma-separated list of QQQ field
names. Plain entries (for example `title,slug`) purge only when at least one
listed field changes. Prefix a field with `!` to ignore it: `!viewCount` prevents
a view-count-only update from purging, while a title change still purges.
With `!viewCount, !lastIndexedAt`, any change outside those fields counts.
Mixed lists keep the allow-list and exclude ignored fields, even if also listed
plainly: `title,viewCount,!viewCount` counts only title changes.
Names are trimmed and case-sensitive. Partial updates retain omitted old values;
only unequal old and new values count. This filter does not restrict inserts or
deletes. The separate `publishedField` filter still applies to all changes.
### Purge-path placeholders
## Purge-path placeholders
`cdn_table_tag_rule` templates (`tagTemplates`, `invalidationPaths`) can use `{field}` for the changed record's own fields and `{prefix.field}` for a joined record, e.g. `/blog/{blog.slug}/{slug}*`.
@@ -72,36 +58,6 @@ CloudFront credentials: `<secretRef>_ACCESS_KEY_ID` / `<secretRef>_SECRET_ACCESS
Drift: the OpenTofu `expected_config` JSON in `CDN_EXPECTED_CONFIG` (or the
variable named by the provider setting `expectedConfigEnv`).
## Revalidation hooks
Requires QQQ **4.1.0-RC.5** (the default `qqq-snapshot` profile imports the released BOM;
its historical property name does not select a core SNAPSHOT).
New `cdn_revalidation_hook` rows default `signatureFormat` to `STANDARD_WEBHOOKS`;
existing rows with a null format keep the `LEGACY` signature. Signed requests use core `WebhookSigner` and
`WebhookSecret`: `webhook-id`, `webhook-timestamp`, and `webhook-signature`.
Rows on `LEGACY` (or null) keep sending
`X-Revalidate-Signature: sha256=<lowercase hex HMAC-SHA256 of the body>`.
Legacy keys remain the exact UTF-8 environment value, with no trimming or decoding.
Standard keys must contain at least 16 bytes after core parsing (trimmed raw text,
or base64-decoded `whsec_` text). `secretRef` names the environment variable;
as before, no resolved secret means no signature headers.
Core `SafeHttpClient` now enforces HTTPS and rejects internal addresses by default.
For an internal HTTP pod endpoint explicitly set `allowPlainHttp=true` and
`internalAllowlist` to its exact hostname, a `.suffix`, IP, or CIDR (comma/whitespace
separated). Metadata and unspecified addresses remain blocked. Every DNS address
is checked before fan-out; the original hostname is retained for Host/TLS, and
connections use the checked addresses without resolving again. Redirects fail,
responses are bounded to 64 KiB, and `timeoutMs` is a positive per-request total
limit (default 5000). These transport rules apply to both signature formats.
Host apps must migrate `cdn_revalidation_hook` to add `signatureFormat` (string),
`allowPlainHttp` (boolean), and `internalAllowlist` (text). Backfill `LEGACY` for
old receivers; configure internal network exceptions explicitly before dispatch.
See [receiver migration](docs/revalidation-hook-migration.md) for the inspected
KoF22 receiver and exact changes required.
## Build
```sh
@@ -111,18 +67,3 @@ KoF22 receiver and exact changes required.
## License
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
### Before a CDN is live
With no enabled CDN provider (including providers of type `NONE`), purges are
recorded only, not sent. A real provider also needs a `LIVE` or `LOCKED` hostname
with purging enabled. Purge From CDN, Purge Everything, and Retry Purge show a
Review warning and a recorded-only Result when sending is unavailable. Requests
for enabled providers remain visible with `SKIPPED_NONE_PROVIDER` or
`SKIPPED_STAGE_OFF`; with no enabled provider, no new request row is created.
The home Connection and Health cards explain the unconfigured state, and purge
activity labels recorded-only requests when disconnected.
This version does not run origin header checks. The Origin headers card says
nothing has been checked until the host application writes results to the
standard `cdn_origin_check` table; those results continue to appear normally.
-68
View File
@@ -1,68 +0,0 @@
# Revalidation hook migration (F11 / qbit-cdn #15)
Reference: core source read directly at tag `v4.1.0-RC.5`, including
`WebhookSigner`, `WebhookSecret`, `WebhookVerifier`, `SafeHttpClient`,
`SafeHttpPolicy`, and `AddressRules`. No temporary core bridge is needed.
## Receiver found locally
The plain `Website-Frontend` and `Website-Publisher` checkouts have no match for
`X-Revalidate-Signature`. The implementation is in
`R:/Git.Local/KofTwentyTwo/Website-Frontend-wt-isr`, branch `feature/cdn-isr`,
HEAD `476318b` at inspection (working files inspected):
- `src/lib/revalidate.ts:5`: names `x-revalidate-signature`.
- `src/lib/revalidate.ts:9`: `signBody` returns `sha256=` plus hex HMAC-SHA256
over the body using the raw string key (the calculation is line 10).
- `src/lib/revalidate.ts:14`: `verifySignature` compares this value in constant
time; line 21 lowercases the incoming legacy header.
- `src/app/api/revalidate/route.ts:30`: reads `CDN_REVALIDATE_SECRET`.
- `src/app/api/revalidate/route.ts:36`: reads the body before JSON parsing.
- `src/app/api/revalidate/route.ts:40`: reads only the legacy signature header
and calls the verifier; standard requests currently return 401.
- `src/__tests__/revalidate-route.test.ts:12`: test requests send only the legacy header.
These receiver files were inspected, not modified.
## Deployment and receiver changes
1. Add the three hook columns documented in README. Existing rows (null
`signatureFormat`) keep the LEGACY signature; switch a hook to `STANDARD_WEBHOOKS`
after its receiver verifies Standard Webhooks. New rows default to `STANDARD_WEBHOOKS`.
For internal HTTP endpoints also explicitly allow HTTP and the intended host.
2. Update `src/lib/revalidate.ts:14` to verify Standard Webhooks: require all
three headers, validate an integer Unix timestamp within 300 seconds of the
receiver clock (reject malformed/non-finite values), then HMAC-SHA256 the
bytes `<webhook-id>.<webhook-timestamp>.<raw body>` and compare decoded
signatures in constant time. Accept any matching `v1,<base64>` entry in the
space-separated signature list. Do not lowercase base64 signatures.
Match core secret parsing: trim text; decode `whsec_` base64 keys, otherwise
UTF-8; use at least 16 key bytes. Rotate short existing keys on both sides.
3. At `src/app/api/revalidate/route.ts:36`, retain raw body bytes for verification;
at line 40 pass all three headers to the new verifier before parsing JSON.
Update its header import/documentation (lines 5 and 22). During migration the
receiver may explicitly support both formats; do not fall back to legacy
when standard headers are present but invalid.
4. Update `src/__tests__/revalidate-route.test.ts:12` and helper tests to cover
valid standard requests, altered body/id/timestamp, missing headers, stale
or future timestamps, malformed timestamps, encoded keys, and legacy mode.
5. Deploy the receiver, then switch its hook to `STANDARD_WEBHOOKS`.
The JSON payload remains `{"tags":[...]}`. One UUID identifies a hook invocation,
shared across its fan-out requests; each request gets its own current timestamp.
A later dispatcher invocation generates a new UUID. Revalidation remains
idempotent, so receivers must not rely on this ID to deduplicate purge retries;
any optional deduplication must remain local to each pod, not suppress other pods.
## Compatibility and verification
The legacy HMAC helper remains solely for the explicitly supported legacy format,
not as a temporary replacement for core. Golden tests use fixed digests calculated
independently of that helper, including a short key and a whitespace/prefix key.
Standard requests are checked using RC.5 `WebhookVerifier`; transport tests cover
HTTP/private-address refusal, explicit local exceptions and mixed-address fan-out.
The test/collaborator constructor now takes `(secrets, resolver)`; callers no longer
inject `java.net.http.HttpClient`. The production no-argument constructor remains.
The only existing test adaptation outside the runner tests is `PurgePipelineTest`
using this constructor. The RC.5 bump required no unrelated code fixes.
+4 -6
View File
@@ -18,19 +18,17 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
<revision>0.1.0-RC.1</revision>
<revision>0.1.0</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<jacoco.version>0.8.15</jacoco.version>
<aws.sdk.version>2.41.34</aws.sdk.version>
<aws.sdk.version>2.41.10</aws.sdk.version>
</properties>
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.1.0</version>
<version>2.0.0</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -194,7 +192,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<qqq.snapshot.version>4.1.0-RC.6</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -61,14 +61,14 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
** A provider row as shown.
*******************************************************************************/
public record ProviderView(Integer id, String name, String type, String distributionId, String dnsTarget, String originHost,
String lastHealthStatus, String lastHealthMessage, Instant lastSuccessfulTest, Long credentialAgeDays, Boolean enabled)
String lastHealthStatus, String lastHealthMessage, Instant lastSuccessfulTest, Long credentialAgeDays)
{
/*******************************************************************************
** Whether this is a real CDN (not the NONE placeholder).
*******************************************************************************/
public boolean isConnected()
{
return (Boolean.TRUE.equals(enabled) && !CdnProviderType.NONE.name().equals(type));
return (!CdnProviderType.NONE.name().equals(type));
}
}
@@ -119,7 +119,7 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
LocalDate rotated = ValueUtils.getValueAsLocalDate(row.getValue("credentialRotatedDate"));
providers.add(new ProviderView(id, row.getValueString("name"), row.getValueString("providerType"), row.getValueString("zoneOrDistributionId"),
row.getValueString("dnsTarget"), row.getValueString("originHost"), row.getValueString("lastHealthStatus"), row.getValueString("lastHealthMessage"),
ValueUtils.getValueAsInstant(row.getValue("lastSuccessfulTestDate")), rotated == null ? null : ChronoUnit.DAYS.between(rotated, today), row.getValueBoolean("enabled")));
ValueUtils.getValueAsInstant(row.getValue("lastSuccessfulTestDate")), rotated == null ? null : ChronoUnit.DAYS.between(rotated, today)));
if(CdnProviderType.CLOUDFRONT.name().equals(row.getValueString("providerType")))
{
PathPlannerSettings settings = PathPlannerSettings.from(CdnPurgeDispatcher.configOf(row));
@@ -80,7 +80,7 @@ public final class CdnHomeView
String text = "All CDN checks are OK.";
if("NOT_CONNECTED".equals(state))
{
text = "No CDN provider configured: purges are recorded only.";
text = "No CDN connected yet.";
}
else if("ATTENTION".equals(state))
{
@@ -99,7 +99,7 @@ public final class CdnHomeView
open(html, "Connection");
if(!data.isConnected())
{
empty(html, "No CDN provider configured: purges are recorded only. The CloudFront distribution is created in the cutover window.");
empty(html, "No CDN connected yet. The CloudFront distribution is created in the cutover window.");
}
for(CdnHomeData.ProviderView provider : data.providers())
{
@@ -182,7 +182,6 @@ public final class CdnHomeView
Instant created = ValueUtils.getValueAsInstant(row.getValue("createDate"));
html.append("<li><a href=\"").append(esc(appPath + "/" + CdnTables.PURGE_REQUEST + "/" + row.getValue("id"))).append("\">#")
.append(esc(String.valueOf(row.getValue("id")))).append("</a> ").append(esc(row.getValueString("status")))
.append(data.isConnected() ? "" : " - recorded only (not sent)")
.append(created == null ? "" : ", " + WHEN.format(created)).append("</li>");
}
html.append("</ul>");
@@ -225,7 +224,7 @@ public final class CdnHomeView
open(html, "Origin headers");
if(data.originChecks().isEmpty())
{
empty(html, "Origin header checks are not run by this version; nothing has been checked.");
empty(html, "No origin header check has run yet.");
}
for(QRecord check : data.originChecks())
{
@@ -1,66 +0,0 @@
/*
* Copyright 2026 Kingsrook, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.cdn.model;
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
/*******************************************************************************
** Signature format selected on each revalidation hook.
*******************************************************************************/
public enum CdnSignatureFormat implements PossibleValueEnum<String>
{
STANDARD_WEBHOOKS("STANDARD_WEBHOOKS", "Standard Webhooks"),
LEGACY("LEGACY", "Legacy X-Revalidate-Signature");
private final String id;
private final String label;
/*******************************************************************************
** Constructor.
*******************************************************************************/
CdnSignatureFormat(String id, String label)
{
this.id = id;
this.label = label;
}
/*******************************************************************************
** Getter for possibleValueId.
*******************************************************************************/
@Override
public String getPossibleValueId()
{
return (id);
}
/*******************************************************************************
** Getter for possibleValueLabel.
*******************************************************************************/
@Override
public String getPossibleValueLabel()
{
return (label);
}
}
@@ -24,7 +24,6 @@ import com.kingsrook.qbits.cdn.spi.CdnProviderType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
@@ -55,12 +54,11 @@ public final class CdnTables
public static final List<String> ALL = List.of(PROVIDER, HOSTNAME, CACHE_POLICY, TABLE_TAG_RULE, PURGE_REQUEST, REVALIDATION_HOOK,
EDGE_STAT_DAILY, ORIGIN_CHECK, DRIFT_FINDING);
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
public static final String PVS_STAGE = "cdn_hostname_stage";
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
public static final String PVS_SIGNATURE_FORMAT = "cdn_signature_format";
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
public static final String PVS_STAGE = "cdn_hostname_stage";
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
@@ -97,7 +95,6 @@ public final class CdnTables
QPossibleValueSource.newForEnum(PVS_PURGE_STATUS, CdnPurgeStatus.values()),
QPossibleValueSource.newForEnum(PVS_PURGE_SOURCE, CdnPurgeSource.values()),
QPossibleValueSource.newForEnum(PVS_POLICY_CLASS, CdnPolicyClass.values()),
QPossibleValueSource.newForEnum(PVS_SIGNATURE_FORMAT, CdnSignatureFormat.values()),
QPossibleValueSource.newForTable(PROVIDER),
QPossibleValueSource.newForTable(HOSTNAME)));
}
@@ -274,8 +271,7 @@ public final class CdnTables
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withMaxLength(100))
.withField(new QFieldMetaData("tagTemplates", QFieldType.TEXT))
.withField(new QFieldMetaData("invalidationPaths", QFieldType.TEXT))
.withField(new QFieldMetaData("onlyWhenFieldsChange", QFieldType.STRING).withMaxLength(1000)
.withHelpContent(new QHelpContent("Comma-separated, case-sensitive field names. Plain entries allow only those changes; !viewCount ignores viewCount changes. With only ignored entries, any other changed field counts. In mixed lists, ignored fields never count. Inserts and deletes are unaffected by this filter.")))
.withField(new QFieldMetaData("onlyWhenFieldsChange", QFieldType.STRING).withMaxLength(1000))
.withField(new QFieldMetaData("publishedField", QFieldType.STRING).withMaxLength(100))
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)));
}
@@ -322,9 +318,6 @@ public final class CdnTables
.withField(new QFieldMetaData("url", QFieldType.STRING).withIsRequired(true).withMaxLength(500))
.withField(new QFieldMetaData("fanOutDns", QFieldType.BOOLEAN).withDefaultValue(false))
.withField(new QFieldMetaData("secretRef", QFieldType.STRING).withMaxLength(100))
.withField(new QFieldMetaData("signatureFormat", QFieldType.STRING).withPossibleValueSourceName(PVS_SIGNATURE_FORMAT).withDefaultValue(CdnSignatureFormat.STANDARD_WEBHOOKS.name()))
.withField(new QFieldMetaData("allowPlainHttp", QFieldType.BOOLEAN).withDefaultValue(false))
.withField(new QFieldMetaData("internalAllowlist", QFieldType.TEXT))
.withField(new QFieldMetaData("timeoutMs", QFieldType.INTEGER).withDefaultValue(5000))
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)));
}
@@ -167,56 +167,6 @@ public final class CdnProcesses
/*******************************************************************************
** Summary counts for requests that cannot be sent, shared by all actions.
*******************************************************************************/
private static class UnsentSummary
{
private final ProcessSummaryLine absent = line(Status.WARNING,
"record has no enabled CDN provider", "records have no enabled CDN provider",
"record had no enabled CDN provider (no purge request recorded)", "records had no enabled CDN provider (no purge requests recorded)");
private final ProcessSummaryLine none = line(Status.WARNING,
"No CDN provider configured: purges are recorded only, not sent.", "No CDN provider configured: purges are recorded only, not sent.",
"purge request was recorded (not sent: no CDN provider)", "purge requests were recorded (not sent: no CDN provider)");
private final ProcessSummaryLine off = line(Status.WARNING,
"CDN hostnames are not live: purges are recorded only, not sent.", "CDN hostnames are not live: purges are recorded only, not sent.",
"purge request was recorded (not sent: CDN hostnames are not live)", "purge requests were recorded (not sent: CDN hostnames are not live)");
/*******************************************************************************
** Count a skipped request; returns whether it cannot be sent.
*******************************************************************************/
boolean add(CdnPurgeStatus status)
{
if(status == CdnPurgeStatus.SKIPPED_NONE_PROVIDER)
{
none.incrementCount(1);
return (true);
}
if(status == CdnPurgeStatus.SKIPPED_STAGE_OFF)
{
off.incrementCount(1);
return (true);
}
return (false);
}
/*******************************************************************************
** Include only reasons encountered by this action.
*******************************************************************************/
void append(ArrayList<ProcessSummaryLineInterface> lines)
{
absent.addSelfToListIfAnyCount(lines);
none.addSelfToListIfAnyCount(lines);
off.addSelfToListIfAnyCount(lines);
}
}
/*******************************************************************************
** Scheduled: send due purges and poll in-progress ones.
*******************************************************************************/
@@ -239,7 +189,6 @@ public final class CdnProcesses
*******************************************************************************/
public static class PurgeRecordsTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
{
private final UnsentSummary unsent = new UnsentSummary();
private final ProcessSummaryLine records = line(Status.OK, "record will be purged", "records will be purged",
"record was purged", "records were purged");
private final ProcessSummaryLine noRule = line(Status.INFO, "record has no CDN tag rule (no change)", "records have no CDN tag rule (no change)",
@@ -258,7 +207,6 @@ public final class CdnProcesses
String table = input.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
List<QRecord> rules = rules(table);
TagRuleEngine engine = new TagRuleEngine();
CdnPurgeStatus sending = CdnPurgeService.sendingStatus(null);
for(QRecord record : input.getRecords())
{
List<String> tags = new ArrayList<>();
@@ -278,10 +226,7 @@ public final class CdnProcesses
recordPaths.forEach(path -> paths.add(path.path()));
record.setValue("cdnTags", String.join("\n", tags));
record.setValue("cdnPaths", com.kingsrook.qbits.cdn.purge.PurgeText.paths(recordPaths));
if(!unsent.add(sending))
{
records.incrementCountAndAddPrimaryKey(record.getValue("id"));
}
records.incrementCountAndAddPrimaryKey(record.getValue("id"));
output.getRecords().add(record);
}
}
@@ -304,7 +249,6 @@ public final class CdnProcesses
lines.add(pathLine);
}
noRule.addSelfToListIfAnyCount(lines);
unsent.append(lines);
return (lines);
}
}
@@ -312,11 +256,10 @@ public final class CdnProcesses
/*******************************************************************************
** Purge From CDN, execute: record requests with hubs included.
** Purge From CDN, execute: one purge request (sent at once, hubs included).
*******************************************************************************/
public static class PurgeRecordsLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
{
private final UnsentSummary unsent = new UnsentSummary();
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
"purge request was recorded (sent within seconds)", "purge requests were recorded (sent within seconds)");
@@ -334,17 +277,7 @@ public final class CdnProcesses
List<QRecord> rows = CdnPurgeService.enqueue(new CdnPurgeService.Request(CdnPurgeKind.TAG.name(),
com.kingsrook.qbits.cdn.purge.PurgeText.tags(record.getValueString("cdnTags")),
com.kingsrook.qbits.cdn.purge.PurgeText.paths(record.getValueString("cdnPaths")), true, CdnPurgeSource.RECORD_ACTION, table, record.getValue("id")));
if(rows.isEmpty())
{
unsent.absent.incrementCount(1);
}
for(QRecord row : rows)
{
if(!unsent.add(CdnPurgeStatus.valueOf(row.getValueString("status"))))
{
requested.incrementCount(1);
}
}
requested.incrementCount(rows.size());
output.getRecords().add(record);
}
}
@@ -363,7 +296,6 @@ public final class CdnProcesses
{
transform.noRule.addSelfToListIfAnyCount(lines);
}
unsent.append(lines);
return (lines);
}
}
@@ -375,7 +307,6 @@ public final class CdnProcesses
*******************************************************************************/
public static class RetryTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
{
private final UnsentSummary unsent = new UnsentSummary();
private final ProcessSummaryLine retry = line(Status.OK, "purge request will be sent again", "purge requests will be sent again",
"purge request was queued again", "purge requests were queued again");
private final ProcessSummaryLine skip = line(Status.INFO, "purge request has not failed (no change)", "purge requests have not failed (no change)",
@@ -394,10 +325,7 @@ public final class CdnProcesses
String status = record.getValueString("status");
if(CdnPurgeStatus.FAILED.name().equals(status) || CdnPurgeStatus.DEAD.name().equals(status) || CdnPurgeStatus.DEFERRED_BUDGET.name().equals(status))
{
if(!unsent.add(CdnPurgeService.sendingStatus(record.getValue("cdnProviderId"))))
{
retry.incrementCountAndAddPrimaryKey(record.getValue("id"));
}
retry.incrementCountAndAddPrimaryKey(record.getValue("id"));
output.getRecords().add(record);
}
else
@@ -418,7 +346,6 @@ public final class CdnProcesses
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
retry.addSelfToListIfAnyCount(lines);
skip.addSelfToListIfAnyCount(lines);
unsent.append(lines);
return (lines);
}
}
@@ -426,16 +353,10 @@ public final class CdnProcesses
/*******************************************************************************
** Retry Purge, execute: queue live providers or record why sending is skipped.
** Retry Purge, execute: back to PENDING (sent on the next dispatcher pass).
*******************************************************************************/
public static class RetryLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
public static class RetryLoad extends AbstractLoadStep
{
private final UnsentSummary unsent = new UnsentSummary();
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
"purge request was queued again", "purge requests were queued again");
/*******************************************************************************
** Reset the rows.
*******************************************************************************/
@@ -445,12 +366,7 @@ public final class CdnProcesses
List<QRecord> updates = new ArrayList<>();
for(QRecord record : input.getRecords())
{
CdnPurgeStatus status = CdnPurgeService.sendingStatus(record.getValue("cdnProviderId"));
if(!unsent.add(status))
{
requested.incrementCount(1);
}
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("status", status.name())
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("status", CdnPurgeStatus.PENDING.name())
.withValue("source", CdnPurgeSource.RETRY.name()).withValue("attempts", 0).withValue("nextAttemptDate", null)
.withValue("errorMessage", null).withValue("completedDate", null).withValue("modifyDate", CdnRuntime.clock().instant()));
}
@@ -459,24 +375,6 @@ public final class CdnProcesses
output.getRecords().addAll(new UpdateAction().execute(new UpdateInput(CdnTables.PURGE_REQUEST).withRecords(updates)).getRecords());
}
}
/*******************************************************************************
** Result lines from the requests recorded during execution.
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
requested.addSelfToListIfAnyCount(lines);
if(getTransformStep() instanceof RetryTransform transform)
{
transform.skip.addSelfToListIfAnyCount(lines);
}
unsent.append(lines);
return (lines);
}
}
@@ -486,7 +384,6 @@ public final class CdnProcesses
*******************************************************************************/
public static class PurgeHostnameTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
{
private final UnsentSummary unsent = new UnsentSummary();
private final ProcessSummaryLine providers = line(Status.WARNING,
"CDN distribution will be purged completely (every hostname on it)", "CDN distributions will be purged completely (every hostname on them)",
"CDN distribution purge was recorded", "CDN distribution purges were recorded");
@@ -505,10 +402,7 @@ public final class CdnProcesses
Serializable providerId = record.getValue("cdnProviderId");
if(providerId != null && seen.add(providerId))
{
if(!unsent.add(CdnPurgeService.sendingStatus(providerId)))
{
providers.incrementCountAndAddPrimaryKey(record.getValue("id"));
}
providers.incrementCountAndAddPrimaryKey(record.getValue("id"));
output.getRecords().add(record);
}
}
@@ -524,7 +418,6 @@ public final class CdnProcesses
{
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
providers.addSelfToListIfAnyCount(lines);
unsent.append(lines);
return (lines);
}
}
@@ -534,14 +427,8 @@ public final class CdnProcesses
/*******************************************************************************
** Purge Everything, execute.
*******************************************************************************/
public static class PurgeHostnameLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
public static class PurgeHostnameLoad extends AbstractLoadStep
{
private final UnsentSummary unsent = new UnsentSummary();
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
"CDN distribution purge was recorded", "CDN distribution purges were recorded");
/*******************************************************************************
** Record an ALL purge for each provider.
*******************************************************************************/
@@ -550,36 +437,11 @@ public final class CdnProcesses
{
for(QRecord record : input.getRecords())
{
List<QRecord> rows = CdnPurgeService.enqueueFor(record.getValue("cdnProviderId"), new CdnPurgeService.Request(CdnPurgeKind.ALL.name(), List.of(), List.of(), true,
CdnPurgeService.enqueueFor(record.getValue("cdnProviderId"), new CdnPurgeService.Request(CdnPurgeKind.ALL.name(), List.of(), List.of(), true,
CdnPurgeSource.MANUAL, CdnTables.HOSTNAME, record.getValue("id")));
if(rows.isEmpty())
{
unsent.absent.incrementCount(1);
}
for(QRecord row : rows)
{
if(!unsent.add(CdnPurgeStatus.valueOf(row.getValueString("status"))))
{
requested.incrementCount(1);
}
}
output.getRecords().add(record);
}
}
/*******************************************************************************
** Result lines from the requests recorded during execution.
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
{
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
requested.addSelfToListIfAnyCount(lines);
unsent.append(lines);
return (lines);
}
}
@@ -120,36 +120,6 @@ public final class CdnPurgeService
/*******************************************************************************
** Sending eligibility for one provider, or all enabled providers when null.
** A pending status means at least one provider can send; otherwise the
** status gives the same reason used when recording a new request.
*******************************************************************************/
public static CdnPurgeStatus sendingStatus(Serializable providerId) throws QException
{
QQueryFilter filter = new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true));
if(providerId != null)
{
filter.addCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, providerId));
}
CdnPurgeStatus status = CdnPurgeStatus.SKIPPED_NONE_PROVIDER;
for(QRecord provider : new QueryAction().execute(new QueryInput(CdnTables.PROVIDER).withFilter(filter)).getRecords())
{
CdnPurgeStatus candidate = initialStatus(provider);
if(candidate == CdnPurgeStatus.PENDING)
{
return (candidate);
}
if(candidate == CdnPurgeStatus.SKIPPED_STAGE_OFF)
{
status = candidate;
}
}
return (status);
}
/*******************************************************************************
** Status of a new request for a provider.
*******************************************************************************/
@@ -21,18 +21,16 @@ import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.net.InetAddress;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.HexFormat;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Function;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.kingsrook.qbits.cdn.model.CdnSignatureFormat;
import com.kingsrook.qbits.cdn.model.CdnTables;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -43,20 +41,17 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.StringUtils;
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpClient;
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpPolicy;
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSigner;
/*******************************************************************************
** Calls every enabled cdn_revalidation_hook before the CDN is purged, so the
** edge never refetches a stale page (design spec section 4.5 step 3).
**
** Request: POST {"tags":[...]} signed with core Standard Webhooks headers.
** LEGACY keeps X-Revalidate-Signature with a hex HMAC of the body. The
** secretRef names an environment variable. SafeHttpClient checks and pins
** every address; fan-out preserves the original Host and TLS server name.
** Request: POST {"tags":[...]} with header
** "X-Revalidate-Signature: sha256=<hex HMAC-SHA256 of the body>", keyed by
** the secret in the environment variable the hook's secretRef names. With
** fanOutDns, the hook's host is resolved and every address is called (one
** per pod behind a headless Service). Every call must answer 2xx.
*******************************************************************************/
public class RevalidationHookRunner
{
@@ -64,8 +59,9 @@ public class RevalidationHookRunner
private static final ObjectMapper JSON = new ObjectMapper();
private final Function<String, String> secrets;
private final Resolver resolver;
private final HttpClient http;
private final Function<String, String> secrets;
private final Resolver resolver;
@@ -87,7 +83,7 @@ public class RevalidationHookRunner
*******************************************************************************/
public RevalidationHookRunner()
{
this(System::getenv,
this(HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(), System::getenv,
host -> java.util.Arrays.stream(InetAddress.getAllByName(host)).map(InetAddress::getHostAddress).toList());
}
@@ -96,8 +92,9 @@ public class RevalidationHookRunner
/*******************************************************************************
** Constructor with collaborators.
*******************************************************************************/
public RevalidationHookRunner(Function<String, String> secrets, Resolver resolver)
public RevalidationHookRunner(HttpClient http, Function<String, String> secrets, Resolver resolver)
{
this.http = http;
this.secrets = secrets;
this.resolver = resolver;
}
@@ -136,63 +133,42 @@ public class RevalidationHookRunner
byte[] body = JSON.writeValueAsBytes(java.util.Map.of("tags", tags));
String secretRef = hook.getValueString("secretRef");
String secret = StringUtils.hasContent(secretRef) ? secrets.apply(secretRef) : null;
String formatValue = hook.getValueString("signatureFormat");
CdnSignatureFormat format = formatValue == null ? CdnSignatureFormat.LEGACY : CdnSignatureFormat.valueOf(formatValue);
WebhookSecret signingSecret = secret != null && format == CdnSignatureFormat.STANDARD_WEBHOOKS ? WebhookSecret.parse(secret) : null;
String messageId = UUID.randomUUID().toString();
URI uri = URI.create(hook.getValueString("url"));
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
if(timeout != null && timeout <= 0)
List<URI> targets = new ArrayList<>();
if(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns"))))
{
throw (new QException("Revalidation hook timeoutMs must be positive"));
for(String address : resolver.resolve(uri.getHost()))
{
String host = address.contains(":") ? "[" + address + "]" : address;
targets.add(new URI(uri.getScheme(), null, host, uri.getPort(), uri.getPath(), uri.getQuery(), null));
}
}
String allowlist = hook.getValueString("internalAllowlist");
SafeHttpPolicy policy = SafeHttpPolicy.defaults()
.withAllowPlainHttp(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("allowPlainHttp"))))
.withInternalAllowlist(allowlist == null ? List.of() : Arrays.asList(allowlist.split("[,\\s]+")))
.withTotalTimeout(Duration.ofMillis(timeout == null ? 5000 : timeout));
List<String> resolved = resolver.resolve(uri.getHost());
InetAddress[] addresses = new InetAddress[resolved.size()];
for(int i = 0; i < resolved.size(); i++)
else
{
addresses[i] = InetAddress.getByName(resolved.get(i));
targets.add(uri);
}
if(addresses.length == 0)
if(targets.isEmpty())
{
throw (new QException("Revalidation hook " + name + " resolved to no addresses"));
}
try(SafeHttpClient validator = new SafeHttpClient(policy, host -> addresses))
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
for(URI target : targets)
{
validator.checkUrl(uri.toString());
}
boolean fanOut = Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns")));
int calls = fanOut ? addresses.length : 1;
for(int i = 0; i < calls; i++)
{
InetAddress[] targetAddresses = fanOut ? new InetAddress[] {addresses[i]} : addresses;
Map<String, String> headers = new LinkedHashMap<>();
headers.put("Content-Type", "application/json");
HttpRequest.Builder request = HttpRequest.newBuilder(target)
.timeout(Duration.ofMillis(timeout == null ? 5000 : timeout))
.header("Content-Type", "application/json")
.POST(HttpRequest.BodyPublishers.ofByteArray(body));
if(secret != null)
{
if(format == CdnSignatureFormat.LEGACY)
{
headers.put(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
}
else
{
headers.putAll(WebhookSigner.headers(messageId, Instant.now(), body, List.of(signingSecret)));
}
request.header(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
}
try(SafeHttpClient client = new SafeHttpClient(policy, host -> targetAddresses))
HttpResponse<String> response = http.send(request.build(), HttpResponse.BodyHandlers.ofString());
if(response.statusCode() / 100 != 2)
{
int status = client.post(uri.toString(), headers, body).statusCode();
if(status / 100 != 2)
{
throw (new QException("Revalidation hook " + name + " answered " + status + " from " + uri.getHost()));
}
throw (new QException("Revalidation hook " + name + " answered " + response.statusCode() + " from " + target.getHost()));
}
}
return (calls);
return (targets.size());
}
catch(QException e)
{
@@ -212,7 +188,7 @@ public class RevalidationHookRunner
/*******************************************************************************
** Legacy compatibility: hex HMAC-SHA256 over the body with raw UTF-8 key bytes.
** Hex HMAC-SHA256.
*******************************************************************************/
public static String sign(String secret, byte[] body) throws Exception
{
@@ -55,8 +55,8 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
** expires by TTL instead of being invalidated;
** - publishedField: when set, only records that are (or were) published
** count, so draft edits purge nothing;
** - onlyWhenFieldsChange: comma list of watched fields; !field ignores a
** field. With only ignored fields, any other changed field counts.
** - onlyWhenFieldsChange: comma list; an update that changes none of them
** purges nothing.
**
** A dotted placeholder "{blog.slug}" reads field "slug" of the record that
** field "blogId" points to (the xId convention). The joined table is the one
@@ -159,27 +159,7 @@ public class TagRuleEngine
String watched = rule.getValueString("onlyWhenFieldsChange");
if(current != null && old != null && StringUtils.hasContent(watched))
{
Set<String> included = new LinkedHashSet<>();
Set<String> ignored = new LinkedHashSet<>();
for(String entry : watched.split(","))
{
String field = entry.strip();
if(field.startsWith("!"))
{
ignored.add(field.substring(1).strip());
}
else if(StringUtils.hasContent(field))
{
included.add(field);
}
}
if(included.isEmpty() && !ignored.isEmpty())
{
included.addAll(current.getValues().keySet());
included.addAll(old.getValues().keySet());
}
included.removeAll(ignored);
return (included.stream()
return (Arrays.stream(watched.split(",")).map(String::strip).filter(StringUtils::hasContent)
.anyMatch(field -> !Objects.equals(current.getValue(field), old.getValue(field))));
}
return (true);
@@ -58,10 +58,10 @@ class CdnHomeTest
{
CdnTestSupport.setUp();
String html = render(CdnWidgets.HOME_WIDGET);
assertThat(html).contains("No CDN provider configured: purges are recorded only.").contains("No edge statistics yet.").contains("No purges recorded yet.")
.contains("No path-billed provider connected yet.").contains("Origin header checks are not run by this version; nothing has been checked.").contains("Nothing to compare yet.")
assertThat(html).contains("No CDN connected yet.").contains("No edge statistics yet.").contains("No purges recorded yet.")
.contains("No path-billed provider connected yet.").contains("No origin header check has run yet.").contains("Nothing to compare yet.")
.contains("minmax(min(100%,22rem),1fr)");
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN provider configured: purges are recorded only.").contains("href=\"/cdnAdmin\"");
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN connected yet.").contains("href=\"/cdnAdmin\"");
}
@@ -99,7 +99,7 @@ class CdnHomeTest
String html = CdnHomeView.render(data, "/cdnAdmin");
assertThat(html).contains("d1.cloudfront.net").contains(age + " days (rotate every 90) - rotation due").contains("1,500").contains("91.5%")
.contains("<polyline").contains("396 of 396").contains("BEHAVIOR").contains("1 open").contains("/cdnAdmin/cdn_purge_request/")
.contains("www.example.com").doesNotContain("recorded only (not sent)", "nothing has been checked");
.contains("www.example.com");
assertThat(CdnHomeView.renderHealthCard(data, "/cdnAdmin")).contains("Needs attention");
}
@@ -119,31 +119,4 @@ class CdnHomeTest
assertEquals("50.0%", CdnHomeView.percent(new BigDecimal("0.5")));
assertEquals("91.5%", CdnHomeView.percent(new BigDecimal("91.5")));
}
/*******************************************************************************
** NONE and disabled providers do not imply a connection or a sent purge.
*******************************************************************************/
@Test
void recordedOnlyHome() throws Exception
{
for(boolean none : List.of(true, false))
{
CdnTestSupport.setUp();
Integer provider = CdnTestSupport.liveProvider(none ? CdnProviderType.NONE : CdnProviderType.CLOUDFRONT, null);
if(!none)
{
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.PROVIDER)
.withRecord(new QRecord().withValue("id", provider).withValue("enabled", false)));
}
CdnTestSupport.insert(CdnTables.PURGE_REQUEST, new QRecord().withValue("cdnProviderId", provider).withValue("status", "SKIPPED_NONE_PROVIDER"));
assertThat(render(CdnWidgets.HOME_WIDGET)).contains("No CDN provider configured: purges are recorded only.")
.contains("The CloudFront distribution is created in the cutover window.").contains("recorded only (not sent)")
.contains("Origin header checks are not run by this version; nothing has been checked.");
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN provider configured: purges are recorded only.");
}
}
}
@@ -70,16 +70,6 @@ class CdnProcessesTest
RunProcessOutput review = new RunProcessAction().execute(input);
List<ProcessSummaryLineInterface> reviewLines = (List<ProcessSummaryLineInterface>) review.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
for(ProcessSummaryLineInterface reviewLine : reviewLines)
{
ProcessSummaryLine summary = (ProcessSummaryLine) reviewLine;
summary.pickMessage(false);
if(summary.getMessage().contains("purges are recorded only"))
{
assertEquals(com.kingsrook.qqq.backend.core.model.actions.processes.Status.WARNING, summary.getStatus());
}
}
RunProcessInput go = new RunProcessInput();
go.setProcessName(process);
go.setProcessUUID(review.getProcessUUID());
@@ -147,7 +137,6 @@ class CdnProcessesTest
new QRecord().withValue("cdnProviderId", 1).withValue("status", "SUCCEEDED"));
List<List<String>> lines = run(CdnProcesses.RETRY_PURGE, rows.get(0).getValue("id") + "," + rows.get(1).getValue("id"));
assertThat(lines.get(0)).containsExactly("1 purge request will be sent again", "1 purge request has not failed (no change)");
assertThat(lines.get(1)).contains("1 purge request was queued again");
QRecord retried = purges().get(0);
assertEquals("PENDING", retried.getValueString("status"));
assertEquals(0, retried.getValueInteger("attempts"));
@@ -166,99 +155,9 @@ class CdnProcessesTest
CdnTestSupport.insert(CdnTables.PROVIDER, new QRecord().withValue("name", "other").withValue("providerType", "CLOUDFRONT").withValue("enabled", true));
List<List<String>> lines = run(CdnProcesses.PURGE_HOSTNAME, "1");
assertThat(lines.get(0)).containsExactly("1 CDN distribution will be purged completely (every hostname on it)");
assertThat(lines.get(1)).containsExactly("1 CDN distribution purge was recorded");
List<QRecord> rows = purges();
assertThat(rows).hasSize(1);
assertEquals("ALL", rows.get(0).getValueString("kind"));
assertEquals(1, rows.get(0).getValueInteger("cdnProviderId"));
}
/*******************************************************************************
** Every action reports NONE and OFF honestly, including plural results.
*******************************************************************************/
@Test
void recordedOnlySummaries() throws Exception
{
for(boolean none : List.of(true, false))
{
for(String process : List.of(CdnProcesses.PURGE_PREFIX + "post", CdnProcesses.PURGE_HOSTNAME, CdnProcesses.RETRY_PURGE))
{
setUp();
String table = none ? CdnTables.PROVIDER : CdnTables.HOSTNAME;
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(table)
.withRecord(new QRecord().withValue("id", 1).withValue(none ? "providerType" : "stage", none ? "NONE" : "OFF")));
String ids = "1";
if(process.equals(CdnProcesses.PURGE_PREFIX + "post"))
{
CdnTestSupport.insert("post", new QRecord().withValue("slug", "one"), new QRecord().withValue("slug", "two"));
ids = "1,2";
}
else if(process.equals(CdnProcesses.RETRY_PURGE))
{
CdnTestSupport.insert(CdnTables.PURGE_REQUEST,
new QRecord().withValue("cdnProviderId", 1).withValue("status", "FAILED"),
new QRecord().withValue("cdnProviderId", 1).withValue("status", "DEAD"));
ids = "1,2";
}
List<List<String>> lines = run(process, ids);
String warning = none ? "No CDN provider configured" : "CDN hostnames are not live";
assertThat(lines.get(0)).anyMatch(line -> line.contains(warning + ": purges are recorded only, not sent."));
assertThat(lines.get(0)).noneMatch(line -> line.contains("will be purged") || line.contains("will be sent again"));
String reason = none ? "no CDN provider" : "CDN hostnames are not live";
assertThat(lines.get(1)).containsExactly(process.equals(CdnProcesses.PURGE_HOSTNAME)
? "1 purge request was recorded (not sent: " + reason + ")"
: "2 purge requests were recorded (not sent: " + reason + ")");
assertThat(purges()).allMatch(row -> (none ? "SKIPPED_NONE_PROVIDER" : "SKIPPED_STAGE_OFF").equals(row.getValueString("status")));
}
}
}
/*******************************************************************************
** Eligibility considers enabled providers and their purge-enabled hosts.
*******************************************************************************/
@Test
void sendingEligibility() throws Exception
{
CdnTestSupport.setUp();
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_NONE_PROVIDER,
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
CdnTestSupport.liveProvider(CdnProviderType.NONE, null);
Integer live = CdnTestSupport.liveProvider(CdnProviderType.CLOUDFRONT, null);
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.PENDING,
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.HOSTNAME)
.withRecord(new QRecord().withValue("id", 2).withValue("purgeEnabled", false)));
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_STAGE_OFF,
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(live));
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.PROVIDER)
.withRecord(new QRecord().withValue("id", live).withValue("enabled", false)));
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_NONE_PROVIDER,
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
}
/*******************************************************************************
** Without an enabled provider, do not claim a request row was recorded.
*******************************************************************************/
@Test
void absentProvider() throws Exception
{
CdnTestSupport.setUp();
CdnTestSupport.postRule();
CdnTestSupport.insert("post", new QRecord().withValue("slug", "one"));
List<List<String>> lines = run(CdnProcesses.PURGE_PREFIX + "post", "1");
assertThat(lines.get(0)).contains("1 No CDN provider configured: purges are recorded only, not sent.");
assertThat(lines.get(1)).containsExactly("1 record had no enabled CDN provider (no purge request recorded)");
assertThat(purges()).isEmpty();
}
}
@@ -17,25 +17,23 @@
package com.kingsrook.qbits.cdn.purge;
import java.net.http.HttpClient;
import java.util.List;
import com.kingsrook.qbits.cdn.CdnRuntime;
import com.kingsrook.qbits.cdn.CdnTestSupport;
import com.kingsrook.qbits.cdn.CdnTestSupport.FakePathProvider;
import com.kingsrook.qbits.cdn.CdnTestSupport.FakeTagProvider;
import com.kingsrook.qbits.cdn.CdnTestSupport;
import com.kingsrook.qbits.cdn.model.CdnTables;
import com.kingsrook.qbits.cdn.spi.CdnException;
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
import com.kingsrook.qbits.cdn.spi.CdnPurgeKind;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -48,7 +46,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
*******************************************************************************/
class PurgePipelineTest
{
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(name -> null, host -> List.of()));
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of()));
@@ -141,32 +139,6 @@ class PurgePipelineTest
/*******************************************************************************
** The record-change listener skips counter updates but records title changes.
*******************************************************************************/
@Test
void ignoredCounterChangeRecordsNoPurge() throws Exception
{
QContext.getQInstance().getTable("post").withField(new QFieldMetaData("viewCount", QFieldType.INTEGER));
CdnTestSupport.liveProvider(CdnProviderType.CLOUDFRONT, null);
QRecord rule = new QueryAction().execute(new QueryInput(CdnTables.TABLE_TAG_RULE)).getRecords().get(0);
new UpdateAction().execute(new UpdateInput(CdnTables.TABLE_TAG_RULE).withRecord(new QRecord().withValue("id", rule.getValue("id"))
.withValue("onlyWhenFieldsChange", "!viewCount")));
QRecord post = CdnTestSupport.insert("post", new QRecord().withValue("slug", "counter-post").withValue("title", "Old")
.withValue("blogId", 1).withValue("isPublished", true).withValue("viewCount", 10)).get(0);
assertThat(purges()).hasSize(1);
new UpdateAction().execute(new UpdateInput("post").withRecord(new QRecord().withValue("id", post.getValue("id")).withValue("viewCount", 11)));
assertThat(purges()).as("counter-only update adds no purge request").hasSize(1);
new UpdateAction().execute(new UpdateInput("post").withRecord(new QRecord().withValue("id", post.getValue("id")).withValue("title", "New")));
assertThat(purges()).hasSize(2);
assertThat(purges().get(1).getValueString("paths")).contains("/blog/quick-bits/counter-post*");
assertThat(purges().get(1).getValueString("tags")).contains("post:" + post.getValue("id"));
}
/*******************************************************************************
** Retryable failures back off and end DEAD; permanent ones go DEAD at once.
*******************************************************************************/
@@ -18,19 +18,15 @@ package com.kingsrook.qbits.cdn.purge;
import java.net.InetSocketAddress;
import java.net.http.HttpClient;
import java.nio.charset.StandardCharsets;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicInteger;
import com.kingsrook.qbits.cdn.CdnTestSupport;
import com.kingsrook.qbits.cdn.model.CdnTables;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerificationException;
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerifier;
import com.sun.net.httpserver.HttpServer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
@@ -50,7 +46,6 @@ class RevalidationHookRunnerTest
private final List<String> signatures = new CopyOnWriteArrayList<>();
private final AtomicInteger status = new AtomicInteger(200);
private HttpServer server;
private final List<Map<String, String>> headers = new CopyOnWriteArrayList<>();
@@ -64,9 +59,6 @@ class RevalidationHookRunnerTest
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
server.createContext("/api/revalidate", exchange ->
{
Map<String, String> received = new LinkedHashMap<>();
exchange.getRequestHeaders().forEach((key, values) -> received.put(key.toLowerCase(java.util.Locale.ROOT), values.get(0)));
headers.add(received);
bodies.add(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
String signature = exchange.getRequestHeaders().getFirst(RevalidationHookRunner.SIGNATURE_HEADER);
signatures.add(signature == null ? "none" : signature);
@@ -96,21 +88,13 @@ class RevalidationHookRunnerTest
void signsAndFansOut() throws Exception
{
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true).withValue("fanOutDns", true)
.withValue("secretRef", "SECRET").withValue("allowPlainHttp", true).withValue("internalAllowlist", "frontend.invalid").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "SECRET".equals(name) ? "test-secret-at-least-16-bytes" : null,
.withValue("secretRef", "SECRET").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> "SECRET".equals(name) ? "s3cret" : null,
host -> List.of("127.0.0.1", "127.0.0.1"));
assertEquals(2, runner.run(List.of("post:1")));
assertThat(bodies).hasSize(2).allMatch("{\"tags\":[\"post:1\"]}"::equals);
WebhookVerifier verifier = new WebhookVerifier(List.of(WebhookSecret.parse("test-secret-at-least-16-bytes")));
for(int i = 0; i < bodies.size(); i++)
{
verifier.verify(headers.get(i), bodies.get(i).getBytes(StandardCharsets.UTF_8));
}
assertThat(signatures).containsExactly("none", "none");
assertThat(headers.get(0).get("webhook-id")).isEqualTo(headers.get(1).get("webhook-id"));
assertThat(headers.get(0).get("host")).startsWith("frontend.invalid:");
assertThatThrownBy(() -> verifier.verify(headers.get(0), "tampered".getBytes(StandardCharsets.UTF_8)))
.isInstanceOf(WebhookVerificationException.class);
String expected = "sha256=" + RevalidationHookRunner.sign("s3cret", bodies.get(0).getBytes(StandardCharsets.UTF_8));
assertThat(signatures).allMatch(expected::equals);
assertEquals(0, runner.run(List.of()));
}
@@ -124,94 +108,12 @@ class RevalidationHookRunnerTest
{
status.set(500);
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true)
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1").withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of());
assertThatThrownBy(() -> runner.run(List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("500");
assertThat(signatures).containsExactly("none");
QRecord fanOut = new QRecord().withValue("name", "empty").withValue("fanOutDns", true).withValue("url", "http://x.invalid/api/revalidate");
assertThatThrownBy(() -> new RevalidationHookRunner(name -> null, host -> List.of()).call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
}
/*******************************************************************************
** Legacy mode preserves fixed signatures, including whitespace and whsec_ keys.
*******************************************************************************/
@Test
void legacyGoldenVectors() throws Exception
{
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
assertThat(signatures.get(0)).isEqualTo("sha256=540457336a45ef962b165f8732f71ebf327ffa67c0cb1047c446df0231fa0db2");
new RevalidationHookRunner(name -> " whsec_test-key-with-spaces ", host -> List.of("127.0.0.1"))
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
assertThat(signatures.get(1)).isEqualTo("sha256=a4dd022bffe6f799633102c415fdc18e22f078bfc98357f909b5df58d5318b05");
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
.call(hook(), List.of("post:1"));
assertThat(signatures.get(2)).as("a null format (existing rows) keeps the legacy signature").isEqualTo(signatures.get(0));
assertThat(bodies).containsExactly("{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}");
assertThat(headers).allSatisfy(value -> assertThat(value).doesNotContainKeys("webhook-id", "webhook-timestamp", "webhook-signature"));
}
/*******************************************************************************
** An explicit standard format accepts encoded secrets.
*******************************************************************************/
@Test
void explicitStandardFormat() throws Exception
{
WebhookSecret secret = WebhookSecret.generate();
new RevalidationHookRunner(name -> secret.reveal(), host -> List.of("127.0.0.1"))
.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"));
new WebhookVerifier(List.of(secret)).verify(headers.get(0), bodies.get(0).getBytes(StandardCharsets.UTF_8));
}
/*******************************************************************************
** Invalid format and short standard secrets fail before sending.
*******************************************************************************/
@Test
void invalidSigningConfiguration()
{
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "short", host -> List.of("127.0.0.1"));
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("at least 16 bytes");
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "UNKNOWN"), List.of("post:1"))).isInstanceOf(QException.class);
assertThat(bodies).isEmpty();
}
/*******************************************************************************
** Safe defaults reject HTTP, private addresses and mixed fan-out before sending.
*******************************************************************************/
@Test
void safeHttpPolicy()
{
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
assertThatThrownBy(() -> runner.call(hook().withValue("allowPlainHttp", null), List.of("post:1")))
.isInstanceOf(QException.class).hasMessageContaining("Only https");
assertThatThrownBy(() -> runner.call(hook().withValue("internalAllowlist", null), List.of("post:1")))
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
RevalidationHookRunner mixed = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1", "169.254.169.254"));
assertThatThrownBy(() -> mixed.call(hook().withValue("fanOutDns", true), List.of("post:1")))
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
assertThatThrownBy(() -> runner.call(hook().withValue("timeoutMs", 0), List.of("post:1")))
.isInstanceOf(QException.class).hasMessageContaining("positive");
assertThat(bodies).isEmpty();
}
/*******************************************************************************
** A local test hook explicitly opts into HTTP and loopback.
*******************************************************************************/
private QRecord hook()
{
return (new QRecord().withValue("name", "test").withValue("secretRef", "SECRET")
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1")
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
assertThatThrownBy(() -> runner.call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
}
}
@@ -135,67 +135,6 @@ class TagRuleEngineTest
/*******************************************************************************
** Ignore-only rules compare actual values after merging partial updates.
*******************************************************************************/
@Test
void ignoresCounterOnlyUpdates() throws Exception
{
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", " !viewCount, !lastIndexedAt ").withValue("tagTemplates", "post:{id}");
QRecord old = new QRecord().withValue("id", 1).withValue("title", "Old").withValue("viewCount", 10).withValue("lastIndexedAt", "yesterday");
TagRuleEngine engine = new TagRuleEngine();
assertTrue(engine.evaluate(rule, List.of(new QRecord(old)), List.of(old)).isEmpty());
assertTrue(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("viewCount", 11).withValue("lastIndexedAt", "today")), List.of(old)).isEmpty());
assertTrue(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", "Old")), List.of(old)).isEmpty());
assertThat(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", "New").withValue("viewCount", 11)), List.of(old)).tags()).containsExactly("post:1");
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", null)), List.of(old)).isEmpty());
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("newField", "value")), List.of(old)).isEmpty());
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("ViewCount", 11)), List.of(old)).isEmpty());
}
/*******************************************************************************
** Plain lists retain allow-list behavior; ignored entries always win.
*******************************************************************************/
@Test
void appliesAllowListAndMixedRules()
{
QRecord old = new QRecord().withValue("title", "Old").withValue("viewCount", 10);
QRecord counterChange = new QRecord(old).withValue("viewCount", 11);
QRecord titleChange = new QRecord(old).withValue("title", "New");
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", " title, ,slug ");
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
assertTrue(TagRuleEngine.counts(rule, titleChange, old));
assertFalse(TagRuleEngine.counts(rule, new QRecord(old), old));
rule.setValue("onlyWhenFieldsChange", "title, viewCount, !viewCount");
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
assertTrue(TagRuleEngine.counts(rule, titleChange, old));
assertFalse(TagRuleEngine.counts(rule, new QRecord(old).withValue("slug", "new"), old));
rule.setValue("onlyWhenFieldsChange", "viewCount,!viewCount");
assertFalse(TagRuleEngine.counts(rule, titleChange, old));
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
}
/*******************************************************************************
** Changed-field filters never suppress inserts or deletes.
*******************************************************************************/
@Test
void insertsAndDeletesIgnoreFieldFilters() throws Exception
{
QRecord post = new QRecord().withValue("id", 1).withValue("viewCount", 10);
for(String fields : List.of("!viewCount", "title", "title,!viewCount"))
{
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", fields).withValue("tagTemplates", "post:{id}");
assertThat(new TagRuleEngine().evaluate(rule, List.of(post), List.of()).tags()).containsExactly("post:1");
assertThat(new TagRuleEngine().evaluate(rule, List.of(), List.of(post)).tags()).containsExactly("post:1");
}
}
/*******************************************************************************
** Stored text round-trips; invalid lines are dropped.
*******************************************************************************/