Compare commits
28
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3d72a5d05d
|
||
|
|
4fb8d74458 | ||
|
|
aba80ff62f
|
||
|
|
55a2ec2a09 | ||
|
|
1262f567c8
|
||
|
|
bbf1b55eac | ||
|
|
f7c9e0fce3
|
||
|
|
d03c791ca3 | ||
|
|
5a0d3e824a
|
||
|
|
422b34988d
|
||
|
|
3980e05719
|
||
|
|
1cd8810e32
|
||
|
|
d3995734dd
|
||
|
|
8bb2c4eb4f
|
||
|
|
b4d08dc065 | ||
|
|
49c0db82f3
|
||
|
|
675b9fb32c | ||
|
|
37fb879f51 | ||
|
|
ecceaf31ee
|
||
|
|
4546a17d6b | ||
|
|
fb4a536494 | ||
|
|
967c52f8bc
|
||
|
|
5801ff2ebd | ||
|
|
7510392166
|
||
|
|
59fc98ccb2 | ||
|
|
1e2af2acf1
|
||
|
|
b1d7d9543f | ||
|
|
15439d2fba |
No files matched your search
+57
-11
@@ -1,14 +1,60 @@
|
||||
version: 2.1
|
||||
setup: true
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.9
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release\/.*)/
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only: [develop]
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only: [/release\/.*/]
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [/^v\d+\.\d+\.\d+$/]
|
||||
@@ -1,25 +0,0 @@
|
||||
# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
|
||||
name: qrun-release
|
||||
on:
|
||||
push: {}
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
jobs:
|
||||
release:
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
docker-runner: container-build
|
||||
browsers: true
|
||||
browser-runner: browser-build
|
||||
postpublish:
|
||||
if: ${{ needs.release.outputs.capture == 'true' }}
|
||||
needs: release
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
@@ -1,23 +0,0 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -0,0 +1,25 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release-no-hotfix
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
hotfix: true
|
||||
asciidoc: false
|
||||
@@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository.
|
||||
## Dependencies
|
||||
|
||||
- Java 21.
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.4` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.5` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
|
||||
- Tests: see pom.xml.
|
||||
|
||||
## Release
|
||||
|
||||
@@ -4,7 +4,27 @@ All notable changes to this project are documented here. The format follows Keep
|
||||
|
||||
## [0.1.1] - Unreleased
|
||||
|
||||
### Added
|
||||
- Central `cdn.provider.<id>` dependency health checks for enabled providers, every 900 seconds with a 20000 ms timeout, reusing the existing connection test with bounded, non-secret results.
|
||||
- Ship the host-neutral in-app help bundle with QQQ registration, live references, and coverage checks for all CDN metadata and generated purge actions. Hosts can override entries with a HOST bundle.
|
||||
- Origin header check: Check Origin Headers on Hostnames (and a nightly run over hostnames that are not Off) asks the origin for sample pages and writes `cdn_origin_check` rows. The table had no writer before.
|
||||
- Guided empty states: help bundle `help/qbit-cdn.yaml` (app and all nine tables, with "How rows get created") applied by `CdnHelp`, and a Setup card on the CDN home.
|
||||
- "Create default provider" (No CDN), "Create default policies" (two turned-off examples) and "Create default rules" (a tags-only rule per purgeable table) table actions; all insert-only.
|
||||
- `hostnameRole` is documented as a label that changes nothing; cache policies are documented as applying only where the application calls `CdnResponseHeaderPolicy`.
|
||||
|
||||
### Removed
|
||||
- Connection testing from the daily drift step; drift checks and the manual Test Connection action remain.
|
||||
- The `cdnHealthCard` HTML widget, its renderer and dedicated status summary.
|
||||
|
||||
### Changed
|
||||
- Pin the released QQQ BOM to 4.1.0-RC.7 and replace the temporary help loader with the core bundle registry.
|
||||
- Require released QQQ BOM 4.1.0-RC.5; revalidation hooks default to core Standard Webhooks signing. Per-hook `signatureFormat=LEGACY` preserves the prior header and signature bytes.
|
||||
- Use core `SafeHttpClient` for revalidation requests: HTTPS/public targets by default, explicit `allowPlainHttp`/`internalAllowlist` exceptions, bounded responses and checked DNS pinning. Fan-out preserves Host/TLS identity. Host schema migration and receiver migration are described in README and `docs/revalidation-hook-migration.md`.
|
||||
|
||||
|
||||
### Fixed
|
||||
- Tag rules accept ignored fields in `onlyWhenFieldsChange`, such as `!viewCount`, so counter-only updates do not purge rendered pages. Plain allow-lists still work, and ignored fields take precedence in mixed lists. No schema change is needed.
|
||||
- Purge actions warn when providers are NONE or hostnames are OFF, and report requests as recorded but not sent. The home explains recorded-only purges and that this version does not run origin header checks.
|
||||
- Tag rule `{prefix.field}` placeholders now resolve the joined table from the `<prefix>Id` field's possible-value source, so prefixed tables (`cms_blog`) work; a missing joined table is logged instead of silently skipped.
|
||||
|
||||
## [0.1.0] - Unreleased
|
||||
|
||||
@@ -25,13 +25,45 @@ Design: QRun-IO/qqq `docs/specs/2026-10-12-qbit-cdn-design.md`.
|
||||
Purge Everything, Test Connection, Import Edge Stats, Check Config Drift;
|
||||
scheduled: dispatcher, daily stats import, daily drift check.
|
||||
- **Admin -> CDN app:** sections Connection / Settings / Activity / Performance,
|
||||
a home widget with honest empty states, and a Health status card.
|
||||
a home widget with honest empty states.
|
||||
- **Providers:** CloudFront (invalidations, connection test, CloudWatch stats,
|
||||
drift against the OpenTofu `expected_config`), Souin (self-hosted, tags),
|
||||
NONE. See `docs/cloudfront-expected-config.md`.
|
||||
- **Permissions:** `cdn.view`, `cdn.purge`, `cdn.purgeAll`, `cdn.admin`.
|
||||
|
||||
## Purge-path placeholders
|
||||
## Health checks
|
||||
|
||||
Central System Health registers `cdn.provider.<id>` for each enabled
|
||||
`cdn_provider` row: **DEPENDENCY**, component `cdn`, category **CDN**, every
|
||||
**900 seconds**, with a **20000 ms** timeout. It reuses the read-only connection
|
||||
test: failure is DOWN; success without the origin header is DEGRADED; otherwise
|
||||
UP. Providers without connection-test support report UNKNOWN. Disable a check
|
||||
centrally with `health_check.enabled`.
|
||||
|
||||
Details contain only provider type, a known provider status (or UNKNOWN), and
|
||||
whether the origin header is configured. The manual `cdnTestConnection` action
|
||||
and its `lastHealth*` fields remain; the daily scheduled step checks drift only.
|
||||
The host must use QQQ **4.1.0-RC.8** or later for the central health SPI.
|
||||
|
||||
## Setup
|
||||
|
||||
This qbit ships no DDL and seeds nothing from a changelog (a host chooses its own column names). Open the CDN app: the Setup card lists each part with the next step. On a fresh install run **Create default provider** (No CDN: purges are recorded, nothing is sent), **Create default rules** (tags for every purgeable table), and optionally **Create default policies** (two turned-off examples). Add your CDN and hostnames when you have them. Cache policies change a response only where the application calls `CdnResponseHeaderPolicy.decide`. **Check Origin Headers** on a hostname (and the nightly run) asks the origin, the provider's `originHost` when set, for the home page, prefix policy paths and literal tag-rule paths, and records how many have the right Cache-Control.
|
||||
|
||||
## Tag rules
|
||||
|
||||
`cdn_table_tag_rule.onlyWhenFieldsChange` is a comma-separated list of QQQ field
|
||||
names. Plain entries (for example `title,slug`) purge only when at least one
|
||||
listed field changes. Prefix a field with `!` to ignore it: `!viewCount` prevents
|
||||
a view-count-only update from purging, while a title change still purges.
|
||||
With `!viewCount, !lastIndexedAt`, any change outside those fields counts.
|
||||
Mixed lists keep the allow-list and exclude ignored fields, even if also listed
|
||||
plainly: `title,viewCount,!viewCount` counts only title changes.
|
||||
|
||||
Names are trimmed and case-sensitive. Partial updates retain omitted old values;
|
||||
only unequal old and new values count. This filter does not restrict inserts or
|
||||
deletes. The separate `publishedField` filter still applies to all changes.
|
||||
|
||||
### Purge-path placeholders
|
||||
|
||||
`cdn_table_tag_rule` templates (`tagTemplates`, `invalidationPaths`) can use `{field}` for the changed record's own fields and `{prefix.field}` for a joined record, e.g. `/blog/{blog.slug}/{slug}*`.
|
||||
|
||||
@@ -58,6 +90,56 @@ CloudFront credentials: `<secretRef>_ACCESS_KEY_ID` / `<secretRef>_SECRET_ACCESS
|
||||
Drift: the OpenTofu `expected_config` JSON in `CDN_EXPECTED_CONFIG` (or the
|
||||
variable named by the provider setting `expectedConfigEnv`).
|
||||
|
||||
## Revalidation hooks
|
||||
|
||||
Requires QQQ **4.1.0-RC.8** (the default `qqq-snapshot` profile imports the released BOM;
|
||||
its historical property name does not select a core SNAPSHOT).
|
||||
|
||||
New `cdn_revalidation_hook` rows default `signatureFormat` to `STANDARD_WEBHOOKS`;
|
||||
existing rows with a null format keep the `LEGACY` signature. Signed requests use core `WebhookSigner` and
|
||||
`WebhookSecret`: `webhook-id`, `webhook-timestamp`, and `webhook-signature`.
|
||||
Rows on `LEGACY` (or null) keep sending
|
||||
`X-Revalidate-Signature: sha256=<lowercase hex HMAC-SHA256 of the body>`.
|
||||
Legacy keys remain the exact UTF-8 environment value, with no trimming or decoding.
|
||||
Standard keys must contain at least 16 bytes after core parsing (trimmed raw text,
|
||||
or base64-decoded `whsec_` text). `secretRef` names the environment variable;
|
||||
as before, no resolved secret means no signature headers.
|
||||
|
||||
Core `SafeHttpClient` now enforces HTTPS and rejects internal addresses by default.
|
||||
For an internal HTTP pod endpoint explicitly set `allowPlainHttp=true` and
|
||||
`internalAllowlist` to its exact hostname, a `.suffix`, IP, or CIDR (comma/whitespace
|
||||
separated). Metadata and unspecified addresses remain blocked. Every DNS address
|
||||
is checked before fan-out; the original hostname is retained for Host/TLS, and
|
||||
connections use the checked addresses without resolving again. Redirects fail,
|
||||
responses are bounded to 64 KiB, and `timeoutMs` is a positive per-request total
|
||||
limit (default 5000). These transport rules apply to both signature formats.
|
||||
|
||||
Host apps must migrate `cdn_revalidation_hook` to add `signatureFormat` (string),
|
||||
`allowPlainHttp` (boolean), and `internalAllowlist` (text). Backfill `LEGACY` for
|
||||
old receivers; configure internal network exceptions explicitly before dispatch.
|
||||
See [receiver migration](docs/revalidation-hook-migration.md) for the inspected
|
||||
KoF22 receiver and exact changes required.
|
||||
|
||||
## In-app help
|
||||
|
||||
The qbit registers [help/qbit-cdn.yaml](src/main/resources/help/qbit-cdn.yaml)
|
||||
as a `QBIT` bundle, versioned with the current POM version (`0.1.0-SNAPSHOT`).
|
||||
QQQ 4.1.0-RC.7 syncs it at startup when the host installs the help tables;
|
||||
registration is harmless without them. Keep the literal bundle version in step
|
||||
with the POM when releasing.
|
||||
|
||||
Hosts can register their own YAML with `layer: HOST` using
|
||||
`HelpBundles.register(qInstance, "help/my-host.yaml")`. Entries with the same
|
||||
key and role override this bundle; user-edited help retains priority. Use the
|
||||
configured app name and concrete `cdnPurge_<table>` process keys for overrides.
|
||||
The qbit binds the shared `app:cdnAdmin;slot:purgeRecords` text to each configured
|
||||
record action and adapts app entries when the host changes the app name.
|
||||
|
||||
`CdnHelpBundleTest` uses `HelpBundleAssertions` to check the shipped resource and
|
||||
the registered bundle with a custom app name and generated record actions.
|
||||
It checks live references, summaries, all owned tables, fields and processes,
|
||||
and additionally requires help for widgets and possible-value sources.
|
||||
|
||||
## Build
|
||||
|
||||
```sh
|
||||
@@ -67,3 +149,17 @@ variable named by the provider setting `expectedConfigEnv`).
|
||||
## License
|
||||
|
||||
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
||||
|
||||
### Before a CDN is live
|
||||
|
||||
With no enabled CDN provider (including providers of type `NONE`), purges are
|
||||
recorded only, not sent. A real provider also needs a `LIVE` or `LOCKED` hostname
|
||||
with purging enabled. Purge From CDN, Purge Everything, and Retry Purge show a
|
||||
Review warning and a recorded-only Result when sending is unavailable. Requests
|
||||
for enabled providers remain visible with `SKIPPED_NONE_PROVIDER` or
|
||||
`SKIPPED_STAGE_OFF`; with no enabled provider, no new request row is created.
|
||||
The home Connection card explains the unconfigured state, and purge
|
||||
activity labels recorded-only requests when disconnected.
|
||||
|
||||
Run **Check Origin Headers** on a hostname to populate origin-check results.
|
||||
When a scheduler is configured, a nightly job also checks hostnames that are not Off.
|
||||
@@ -0,0 +1,7 @@
|
||||
# Upgrading
|
||||
|
||||
## Unreleased
|
||||
|
||||
- No schema change. Hosts get the help text, the Setup card and the new actions automatically.
|
||||
- The nightly origin check runs on hosts with a scheduler (02:45 UTC) over hostnames that are not Off. It makes HTTPS requests to the provider's origin host (or the hostname) for a few pages; allow that egress or leave hostnames Off.
|
||||
- Run Create default provider on hosts that have no provider if you want purges recorded as would-be purges.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Revalidation hook migration (F11 / qbit-cdn #15)
|
||||
|
||||
Reference: core source read directly at tag `v4.1.0-RC.5`, including
|
||||
`WebhookSigner`, `WebhookSecret`, `WebhookVerifier`, `SafeHttpClient`,
|
||||
`SafeHttpPolicy`, and `AddressRules`. No temporary core bridge is needed.
|
||||
|
||||
## Receiver found locally
|
||||
|
||||
The plain `Website-Frontend` and `Website-Publisher` checkouts have no match for
|
||||
`X-Revalidate-Signature`. The implementation is in
|
||||
`R:/Git.Local/KofTwentyTwo/Website-Frontend-wt-isr`, branch `feature/cdn-isr`,
|
||||
HEAD `476318b` at inspection (working files inspected):
|
||||
|
||||
- `src/lib/revalidate.ts:5`: names `x-revalidate-signature`.
|
||||
- `src/lib/revalidate.ts:9`: `signBody` returns `sha256=` plus hex HMAC-SHA256
|
||||
over the body using the raw string key (the calculation is line 10).
|
||||
- `src/lib/revalidate.ts:14`: `verifySignature` compares this value in constant
|
||||
time; line 21 lowercases the incoming legacy header.
|
||||
- `src/app/api/revalidate/route.ts:30`: reads `CDN_REVALIDATE_SECRET`.
|
||||
- `src/app/api/revalidate/route.ts:36`: reads the body before JSON parsing.
|
||||
- `src/app/api/revalidate/route.ts:40`: reads only the legacy signature header
|
||||
and calls the verifier; standard requests currently return 401.
|
||||
- `src/__tests__/revalidate-route.test.ts:12`: test requests send only the legacy header.
|
||||
|
||||
These receiver files were inspected, not modified.
|
||||
|
||||
## Deployment and receiver changes
|
||||
|
||||
1. Add the three hook columns documented in README. Existing rows (null
|
||||
`signatureFormat`) keep the LEGACY signature; switch a hook to `STANDARD_WEBHOOKS`
|
||||
after its receiver verifies Standard Webhooks. New rows default to `STANDARD_WEBHOOKS`.
|
||||
For internal HTTP endpoints also explicitly allow HTTP and the intended host.
|
||||
2. Update `src/lib/revalidate.ts:14` to verify Standard Webhooks: require all
|
||||
three headers, validate an integer Unix timestamp within 300 seconds of the
|
||||
receiver clock (reject malformed/non-finite values), then HMAC-SHA256 the
|
||||
bytes `<webhook-id>.<webhook-timestamp>.<raw body>` and compare decoded
|
||||
signatures in constant time. Accept any matching `v1,<base64>` entry in the
|
||||
space-separated signature list. Do not lowercase base64 signatures.
|
||||
Match core secret parsing: trim text; decode `whsec_` base64 keys, otherwise
|
||||
UTF-8; use at least 16 key bytes. Rotate short existing keys on both sides.
|
||||
3. At `src/app/api/revalidate/route.ts:36`, retain raw body bytes for verification;
|
||||
at line 40 pass all three headers to the new verifier before parsing JSON.
|
||||
Update its header import/documentation (lines 5 and 22). During migration the
|
||||
receiver may explicitly support both formats; do not fall back to legacy
|
||||
when standard headers are present but invalid.
|
||||
4. Update `src/__tests__/revalidate-route.test.ts:12` and helper tests to cover
|
||||
valid standard requests, altered body/id/timestamp, missing headers, stale
|
||||
or future timestamps, malformed timestamps, encoded keys, and legacy mode.
|
||||
5. Deploy the receiver, then switch its hook to `STANDARD_WEBHOOKS`.
|
||||
|
||||
The JSON payload remains `{"tags":[...]}`. One UUID identifies a hook invocation,
|
||||
shared across its fan-out requests; each request gets its own current timestamp.
|
||||
A later dispatcher invocation generates a new UUID. Revalidation remains
|
||||
idempotent, so receivers must not rely on this ID to deduplicate purge retries;
|
||||
any optional deduplication must remain local to each pod, not suppress other pods.
|
||||
|
||||
## Compatibility and verification
|
||||
|
||||
The legacy HMAC helper remains solely for the explicitly supported legacy format,
|
||||
not as a temporary replacement for core. Golden tests use fixed digests calculated
|
||||
independently of that helper, including a short key and a whitespace/prefix key.
|
||||
Standard requests are checked using RC.5 `WebhookVerifier`; transport tests cover
|
||||
HTTP/private-address refusal, explicit local exceptions and mixed-address fan-out.
|
||||
|
||||
The test/collaborator constructor now takes `(secrets, resolver)`; callers no longer
|
||||
inject `java.net.http.HttpClient`. The production no-argument constructor remains.
|
||||
The only existing test adaptation outside the runner tests is `PurgePipelineTest`
|
||||
using this constructor. The RC.5 bump required no unrelated code fixes.
|
||||
@@ -18,17 +18,19 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<revision>0.1.0</revision>
|
||||
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
|
||||
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
|
||||
<revision>0.1.0-SNAPSHOT</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
<aws.sdk.version>2.41.10</aws.sdk.version>
|
||||
<aws.sdk.version>2.41.34</aws.sdk.version>
|
||||
</properties>
|
||||
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>2.1.0</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
@@ -104,7 +106,34 @@
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<resources>
|
||||
<!-- help bundles carry the artifact version (QQQ help bundle provenance) -->
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<filtering>true</filtering>
|
||||
<includes>
|
||||
<include>help/*.yaml</include>
|
||||
</includes>
|
||||
</resource>
|
||||
<resource>
|
||||
<directory>src/main/resources</directory>
|
||||
<filtering>false</filtering>
|
||||
<excludes>
|
||||
<exclude>help/*.yaml</exclude>
|
||||
</excludes>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<!-- the artifact version, for the help bundle version test -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<systemPropertyVariables combine.children="append">
|
||||
<qbit.artifactVersion>${project.version}</qbit.artifactVersion>
|
||||
</systemPropertyVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
@@ -192,7 +221,7 @@
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
|
||||
<qqq.snapshot.version>4.1.0-RC.8</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -20,10 +20,13 @@ package com.kingsrook.qbits.cdn;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.app.CdnWidgets;
|
||||
import com.kingsrook.qbits.cdn.health.CdnProviderHealthChecks;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnProcesses;
|
||||
import com.kingsrook.qbits.cdn.purge.CdnRecordChangeListener;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
@@ -149,6 +152,8 @@ public class CdnQBitProducer implements QBitMetaDataProducer<CdnQBitConfig>
|
||||
}
|
||||
List<QProcessMetaData> processes = new ArrayList<>(CdnProcesses.processes(qBitConfig));
|
||||
processes.addAll(com.kingsrook.qbits.cdn.processes.CdnProviderProcesses.processes(qBitConfig));
|
||||
processes.addAll(com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses.processes(qBitConfig));
|
||||
processes.addAll(com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses.processes());
|
||||
for(QProcessMetaData process : processes)
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
@@ -160,6 +165,9 @@ public class CdnQBitProducer implements QBitMetaDataProducer<CdnQBitConfig>
|
||||
}
|
||||
output.add(CdnWidgets.app(qBitConfig, tables));
|
||||
output.addSelfToInstance(qInstance);
|
||||
HealthChecksMetaData.ofOrWithNew(qInstance).withSupplier(new HealthCheckSupplierMetaData()
|
||||
.withNamePrefix("cdn.provider").withComponent("cdn").withSupplier(new QCodeReference(CdnProviderHealthChecks.class)));
|
||||
com.kingsrook.qbits.cdn.model.CdnHelp.register(qInstance, qBitConfig);
|
||||
|
||||
if(!qBitConfig.getPurgeableTables().isEmpty())
|
||||
{
|
||||
|
||||
@@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
*******************************************************************************/
|
||||
public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostnames, List<StatView> stats, List<QRecord> recentPurges,
|
||||
int failed24h, int dead24h, int failed7d, int dead7d, List<BudgetView> budgets, List<QRecord> originChecks,
|
||||
Map<String, Integer> openDriftByArea, Instant lastDriftCheck, Instant now)
|
||||
Map<String, Integer> openDriftByArea, Instant lastDriftCheck, Instant now, List<CdnSetupStatus.Item> setup)
|
||||
{
|
||||
public static final int ROTATION_WARNING_DAYS = 75;
|
||||
public static final int ROTATION_DAYS = 90;
|
||||
@@ -61,14 +61,14 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
** A provider row as shown.
|
||||
*******************************************************************************/
|
||||
public record ProviderView(Integer id, String name, String type, String distributionId, String dnsTarget, String originHost,
|
||||
String lastHealthStatus, String lastHealthMessage, Instant lastSuccessfulTest, Long credentialAgeDays)
|
||||
String lastHealthStatus, String lastHealthMessage, Instant lastSuccessfulTest, Long credentialAgeDays, Boolean enabled)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Whether this is a real CDN (not the NONE placeholder).
|
||||
*******************************************************************************/
|
||||
public boolean isConnected()
|
||||
{
|
||||
return (!CdnProviderType.NONE.name().equals(type));
|
||||
return (Boolean.TRUE.equals(enabled) && !CdnProviderType.NONE.name().equals(type));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
LocalDate rotated = ValueUtils.getValueAsLocalDate(row.getValue("credentialRotatedDate"));
|
||||
providers.add(new ProviderView(id, row.getValueString("name"), row.getValueString("providerType"), row.getValueString("zoneOrDistributionId"),
|
||||
row.getValueString("dnsTarget"), row.getValueString("originHost"), row.getValueString("lastHealthStatus"), row.getValueString("lastHealthMessage"),
|
||||
ValueUtils.getValueAsInstant(row.getValue("lastSuccessfulTestDate")), rotated == null ? null : ChronoUnit.DAYS.between(rotated, today)));
|
||||
ValueUtils.getValueAsInstant(row.getValue("lastSuccessfulTestDate")), rotated == null ? null : ChronoUnit.DAYS.between(rotated, today), row.getValueBoolean("enabled")));
|
||||
if(CdnProviderType.CLOUDFRONT.name().equals(row.getValueString("providerType")))
|
||||
{
|
||||
PathPlannerSettings settings = PathPlannerSettings.from(CdnPurgeDispatcher.configOf(row));
|
||||
@@ -237,7 +237,7 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
}
|
||||
|
||||
return (new CdnHomeData(providers, hostnames, stats, recent, failed24, dead24, failed7, dead7, budgets, new ArrayList<>(latestOrigin.values()),
|
||||
drift, lastDrift, now));
|
||||
drift, lastDrift, now, CdnSetupStatus.load()));
|
||||
}
|
||||
|
||||
|
||||
@@ -252,24 +252,6 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Overall state for the Health card: NOT_CONNECTED, ATTENTION or OK.
|
||||
*******************************************************************************/
|
||||
public String overallState()
|
||||
{
|
||||
if(!isConnected())
|
||||
{
|
||||
return ("NOT_CONNECTED");
|
||||
}
|
||||
boolean attention = dead24h > 0 || !openDriftByArea.isEmpty()
|
||||
|| providers.stream().anyMatch(p -> p.isConnected() && ("FAILED".equals(p.lastHealthStatus())
|
||||
|| (p.credentialAgeDays() != null && p.credentialAgeDays() >= ROTATION_WARNING_DAYS)))
|
||||
|| budgets.stream().anyMatch(b -> b.billed() > b.soft());
|
||||
return (attention ? "ATTENTION" : "OK");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Paths billed this month for a provider.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -61,6 +61,7 @@ public final class CdnHomeView
|
||||
{
|
||||
StringBuilder html = new StringBuilder("<div class=\"cdn-home\" style=\"" + GRID + "\">");
|
||||
connection(html, data, appPath);
|
||||
setup(html, data, appPath);
|
||||
performance(html, data);
|
||||
purges(html, data, appPath);
|
||||
budget(html, data);
|
||||
@@ -71,26 +72,6 @@ public final class CdnHomeView
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Health status card.
|
||||
*******************************************************************************/
|
||||
public static String renderHealthCard(CdnHomeData data, String appPath)
|
||||
{
|
||||
String state = data.overallState();
|
||||
String text = "All CDN checks are OK.";
|
||||
if("NOT_CONNECTED".equals(state))
|
||||
{
|
||||
text = "No CDN connected yet.";
|
||||
}
|
||||
else if("ATTENTION".equals(state))
|
||||
{
|
||||
text = "Needs attention: see the CDN home.";
|
||||
}
|
||||
return ("<div style=\"" + CARD + "\"><strong>CDN</strong><p>" + esc(text) + "</p><a href=\"" + esc(appPath) + "\">Open the CDN home</a></div>");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Connection card.
|
||||
*******************************************************************************/
|
||||
@@ -99,7 +80,7 @@ public final class CdnHomeView
|
||||
open(html, "Connection");
|
||||
if(!data.isConnected())
|
||||
{
|
||||
empty(html, "No CDN connected yet. The CloudFront distribution is created in the cutover window.");
|
||||
empty(html, "No CDN provider configured: purges are recorded only. The CloudFront distribution is created in the cutover window.");
|
||||
}
|
||||
for(CdnHomeData.ProviderView provider : data.providers())
|
||||
{
|
||||
@@ -182,6 +163,7 @@ public final class CdnHomeView
|
||||
Instant created = ValueUtils.getValueAsInstant(row.getValue("createDate"));
|
||||
html.append("<li><a href=\"").append(esc(appPath + "/" + CdnTables.PURGE_REQUEST + "/" + row.getValue("id"))).append("\">#")
|
||||
.append(esc(String.valueOf(row.getValue("id")))).append("</a> ").append(esc(row.getValueString("status")))
|
||||
.append(data.isConnected() ? "" : " - recorded only (not sent)")
|
||||
.append(created == null ? "" : ", " + WHEN.format(created)).append("</li>");
|
||||
}
|
||||
html.append("</ul>");
|
||||
@@ -216,6 +198,50 @@ public final class CdnHomeView
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Setup card: each configuration table, whether it is set up, and the next
|
||||
** step, linked to its table.
|
||||
*******************************************************************************/
|
||||
private static void setup(StringBuilder html, CdnHomeData data, String appPath)
|
||||
{
|
||||
if(data.setup() == null || data.setup().isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
open(html, "Setup");
|
||||
long todo = data.setup().stream().filter(i -> i.state() == CdnSetupStatus.State.TODO).count();
|
||||
html.append("<p>").append(todo == 0 ? "Everything the CDN needs is set up." : esc(todo + (todo == 1 ? " item needs" : " items need") + " attention.")).append("</p><ul style=\"list-style:none;margin:0;padding:0;\">");
|
||||
for(CdnSetupStatus.Item item : data.setup())
|
||||
{
|
||||
String color = switch(item.state())
|
||||
{
|
||||
case OK -> "#16a34a";
|
||||
case TODO -> "#dc2626";
|
||||
case DEFAULT -> "#d97706";
|
||||
case OPTIONAL -> "#9ca3af";
|
||||
};
|
||||
String label = switch(item.state())
|
||||
{
|
||||
case OK -> "Set up";
|
||||
case TODO -> "Needs setup";
|
||||
case DEFAULT -> "Using defaults";
|
||||
case OPTIONAL -> "Optional";
|
||||
};
|
||||
html.append("<li style=\"margin:0 0 .75rem;\"><strong><a href=\"").append(esc(appPath + "/" + item.tableName())).append("\">").append(esc(item.label())).append("</a></strong> ")
|
||||
.append("<span style=\"white-space:nowrap;\"><span aria-hidden=\"true\" style=\"width:.55rem;height:.55rem;border-radius:50%;background:").append(color)
|
||||
.append(";display:inline-block;\"></span> ").append(esc(label)).append("</span><div>").append(esc(item.detail())).append("</div>");
|
||||
if(item.nextStep() != null)
|
||||
{
|
||||
html.append("<div style=\"").append(MUTED).append("\">Next: ").append(esc(item.nextStep())).append("</div>");
|
||||
}
|
||||
html.append("</li>");
|
||||
}
|
||||
html.append("</ul>");
|
||||
close(html);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Origin header card.
|
||||
*******************************************************************************/
|
||||
@@ -224,7 +250,7 @@ public final class CdnHomeView
|
||||
open(html, "Origin headers");
|
||||
if(data.originChecks().isEmpty())
|
||||
{
|
||||
empty(html, "No origin header check has run yet.");
|
||||
empty(html, "No origin check yet. Run Check Origin Headers on Hostnames; hostnames that are not Off are also checked every night.");
|
||||
}
|
||||
for(QRecord check : data.originChecks())
|
||||
{
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.app;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.cdn.CdnQBitConfig;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.model.CdnHostnameStage;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Setup card on the CDN home: each configuration table, whether it is set up,
|
||||
** and the next step. Reads the small configuration tables the same way the runtime
|
||||
** does (enabled rows only where the runtime filters on enabled, the application's
|
||||
** purgeable tables from the qbit config).
|
||||
*******************************************************************************/
|
||||
public final class CdnSetupStatus
|
||||
{
|
||||
/*******************************************************************************
|
||||
** OK: in place. TODO: needs doing. DEFAULT: working on defaults. OPTIONAL: off
|
||||
** until set up.
|
||||
*******************************************************************************/
|
||||
public enum State
|
||||
{
|
||||
OK, TODO, DEFAULT, OPTIONAL
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One line of the card.
|
||||
*******************************************************************************/
|
||||
public record Item(String label, String tableName, State state, String detail, String nextStep)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnSetupStatus()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build the items from the tables.
|
||||
*******************************************************************************/
|
||||
public static List<Item> load() throws QException
|
||||
{
|
||||
CdnQBitConfig config = CdnRuntime.getConfig();
|
||||
List<QRecord> providers = all(CdnTables.PROVIDER);
|
||||
List<QRecord> hostnames = all(CdnTables.HOSTNAME);
|
||||
List<QRecord> policies = all(CdnTables.CACHE_POLICY);
|
||||
List<QRecord> rules = all(CdnTables.TABLE_TAG_RULE);
|
||||
List<QRecord> hooks = all(CdnTables.REVALIDATION_HOOK);
|
||||
long checks = new CountAction().execute(new CountInput(CdnTables.ORIGIN_CHECK)).getCount();
|
||||
boolean scheduled = config != null && config.getSchedulerName() != null;
|
||||
return (build(providers, hostnames, policies, rules, hooks, checks, config == null ? List.of() : config.getPurgeableTables(), scheduled));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The items for given rows (no queries; for tests and reuse).
|
||||
*******************************************************************************/
|
||||
public static List<Item> build(List<QRecord> providers, List<QRecord> hostnames, List<QRecord> policies, List<QRecord> rules, List<QRecord> hooks,
|
||||
long checks, List<String> purgeableTables, boolean scheduled)
|
||||
{
|
||||
List<Item> items = new ArrayList<>();
|
||||
|
||||
//////////////
|
||||
// provider //
|
||||
//////////////
|
||||
List<QRecord> enabledProviders = providers.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("enabled"))).toList();
|
||||
List<QRecord> real = enabledProviders.stream().filter(p -> !CdnProviderType.NONE.name().equals(p.getValueString("providerType"))).toList();
|
||||
if(providers.isEmpty())
|
||||
{
|
||||
items.add(new Item("Providers", CdnTables.PROVIDER, State.TODO, "No provider: changes are not even recorded as purges.",
|
||||
"Run Create default provider (No CDN: records purges, sends nothing), or add your CDN with New."));
|
||||
}
|
||||
else if(enabledProviders.isEmpty())
|
||||
{
|
||||
items.add(new Item("Providers", CdnTables.PROVIDER, State.TODO, "Every provider is turned off.", "Turn one on (Enabled)."));
|
||||
}
|
||||
else if(real.isEmpty())
|
||||
{
|
||||
items.add(new Item("Providers", CdnTables.PROVIDER, State.DEFAULT, "Only No CDN: purges are recorded and nothing is sent.",
|
||||
"Add your CDN with New (type, distribution id, and the environment variable that holds its key), then Test Connection."));
|
||||
}
|
||||
else
|
||||
{
|
||||
boolean failing = real.stream().anyMatch(p -> "FAILED".equals(p.getValueString("lastHealthStatus")));
|
||||
items.add(failing
|
||||
? new Item("Providers", CdnTables.PROVIDER, State.TODO, "A provider's last connection test failed.", "Check its key variable and run Test Connection.")
|
||||
: new Item("Providers", CdnTables.PROVIDER, State.OK, real.size() + " CDN provider" + (real.size() == 1 ? "" : "s") + " turned on.", null));
|
||||
}
|
||||
|
||||
///////////////
|
||||
// hostnames //
|
||||
///////////////
|
||||
if(hostnames.isEmpty())
|
||||
{
|
||||
items.add(real.isEmpty()
|
||||
? new Item("Hostnames", CdnTables.HOSTNAME, State.OPTIONAL, "None: nothing sits behind a CDN yet.", "Add each site hostname once a CDN provider exists.")
|
||||
: new Item("Hostnames", CdnTables.HOSTNAME, State.TODO, "A CDN is set up but no hostname uses it, so nothing is purged.", "Add the hostnames behind it, at stage Off, and move them to Headers only, then Live."));
|
||||
}
|
||||
else
|
||||
{
|
||||
long live = hostnames.stream().filter(h -> (CdnHostnameStage.LIVE.name().equals(h.getValueString("stage")) || CdnHostnameStage.LOCKED.name().equals(h.getValueString("stage")))
|
||||
&& Boolean.TRUE.equals(h.getValueBoolean("purgeEnabled"))).count();
|
||||
items.add(live == 0
|
||||
? new Item("Hostnames", CdnTables.HOSTNAME, State.DEFAULT, hostnames.size() + " hostname" + (hostnames.size() == 1 ? "" : "s") + ", none Live: purges are recorded, not sent.",
|
||||
"When the CDN serves a hostname, move it to Live (and leave Purge enabled on).")
|
||||
: new Item("Hostnames", CdnTables.HOSTNAME, State.OK, live + " of " + hostnames.size() + " hostnames Live.", null));
|
||||
}
|
||||
|
||||
////////////////////////
|
||||
// cache policies //
|
||||
////////////////////////
|
||||
long enabledPolicies = policies.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("enabled"))).count();
|
||||
items.add(policies.isEmpty()
|
||||
? new Item("Cache policies", CdnTables.CACHE_POLICY, State.OPTIONAL, "None: responses keep the headers your application sets.",
|
||||
"Run Create default policies for examples. Policies only change responses when the application calls CdnResponseHeaderPolicy.")
|
||||
: enabledPolicies == 0
|
||||
? new Item("Cache policies", CdnTables.CACHE_POLICY, State.DEFAULT, policies.size() + " polic" + (policies.size() == 1 ? "y" : "ies") + ", all turned off.",
|
||||
"Edit one and turn it on. It takes effect only where the application calls CdnResponseHeaderPolicy.")
|
||||
: new Item("Cache policies", CdnTables.CACHE_POLICY, State.OK, enabledPolicies + " polic" + (enabledPolicies == 1 ? "y" : "ies") + " turned on (applied where the application calls CdnResponseHeaderPolicy).", null));
|
||||
|
||||
////////////////////////////////////////
|
||||
// tag rules: every purgeable table //
|
||||
////////////////////////////////////////
|
||||
Set<String> covered = new HashSet<>();
|
||||
rules.stream().filter(r -> Boolean.TRUE.equals(r.getValueBoolean("enabled"))).forEach(r -> covered.add(r.getValueString("tableName")));
|
||||
List<String> uncovered = purgeableTables.stream().filter(t -> !covered.contains(t)).toList();
|
||||
if(purgeableTables.isEmpty())
|
||||
{
|
||||
items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.OPTIONAL, "The application lists no purgeable tables, so record changes purge nothing.", "List them in the qbit config (purgeableTables)."));
|
||||
}
|
||||
else if(uncovered.isEmpty())
|
||||
{
|
||||
items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.OK, "Every purgeable table has a rule (" + String.join(", ", purgeableTables) + ").", null));
|
||||
}
|
||||
else
|
||||
{
|
||||
items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.TODO, "No rule for " + String.join(", ", uncovered) + ": changes to them purge nothing.",
|
||||
"Run Create default rules (tags only), then add invalidation paths where your CDN purges by path."));
|
||||
}
|
||||
|
||||
///////////////////////
|
||||
// revalidation hooks //
|
||||
///////////////////////
|
||||
long enabledHooks = hooks.stream().filter(h -> Boolean.TRUE.equals(h.getValueBoolean("enabled"))).count();
|
||||
items.add(enabledHooks == 0
|
||||
? new Item("Revalidation hooks", CdnTables.REVALIDATION_HOOK, State.OPTIONAL, "None.", "Add one if a front end keeps its own cache (for example Next.js) that must refresh before the CDN is purged.")
|
||||
: new Item("Revalidation hooks", CdnTables.REVALIDATION_HOOK, State.OK, enabledHooks + " hook" + (enabledHooks == 1 ? "" : "s") + " turned on.", null));
|
||||
|
||||
////////////////////
|
||||
// origin checks //
|
||||
////////////////////
|
||||
items.add(checks == 0
|
||||
? new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OPTIONAL, "Never run.",
|
||||
"Run Check Origin Headers on Hostnames." + (scheduled ? " Hostnames that are not Off are also checked every night." : " No scheduler is configured, so nothing runs it automatically."))
|
||||
: new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OK, checks + " check" + (checks == 1 ? "" : "s") + " recorded.", null));
|
||||
return (items);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> all(String table) throws QException
|
||||
{
|
||||
return (new QueryAction().execute(new QueryInput(table)).getRecords());
|
||||
}
|
||||
}
|
||||
@@ -37,13 +37,12 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
/*******************************************************************************
|
||||
** The Admin -> CDN app: a home widget (the single owner of CDN data), four
|
||||
** sections of standard tables, and a status card for the app's Health page.
|
||||
** sections of standard tables.
|
||||
** Processes are never added to the nav.
|
||||
*******************************************************************************/
|
||||
public final class CdnWidgets
|
||||
{
|
||||
public static final String HOME_WIDGET = "cdnHome";
|
||||
public static final String HEALTH_CARD_WIDGET = "cdnHealthCard";
|
||||
public static final String HOME_WIDGET = "cdnHome";
|
||||
|
||||
|
||||
|
||||
@@ -63,9 +62,7 @@ public final class CdnWidgets
|
||||
{
|
||||
return (List.of(
|
||||
new QWidgetMetaData().withName(HOME_WIDGET).withLabel("CDN").withType(WidgetType.HTML.getType())
|
||||
.withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false).withPermissionRules(CdnTables.permissionRules()),
|
||||
new QWidgetMetaData().withName(HEALTH_CARD_WIDGET).withLabel("CDN").withType(WidgetType.HTML.getType())
|
||||
.withCodeReference(new QCodeReference(HealthCardRenderer.class)).withGridColumns(12).withIsCard(true).withPermissionRules(CdnTables.permissionRules())));
|
||||
.withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false).withPermissionRules(CdnTables.permissionRules())));
|
||||
}
|
||||
|
||||
|
||||
@@ -132,21 +129,4 @@ public final class CdnWidgets
|
||||
return (new RenderWidgetOutput(new RawHTML(null, CdnHomeView.render(CdnHomeData.load(), tablePathPrefix()))));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Health status card renderer.
|
||||
*******************************************************************************/
|
||||
public static class HealthCardRenderer extends AbstractWidgetRenderer
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Render.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
|
||||
{
|
||||
return (new RenderWidgetOutput(new RawHTML("CDN", CdnHomeView.renderHealthCard(CdnHomeData.load(), appPath()))));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,575 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.checks;
|
||||
|
||||
|
||||
import javax.net.ssl.SNIHostName;
|
||||
import javax.net.ssl.SSLParameters;
|
||||
import javax.net.ssl.SSLSocket;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Socket;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.headers.CdnResponseHeaderPolicy;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The origin header check (design spec section 3.2): requests a few sample paths
|
||||
** of a hostname from its origin and records whether each answers with the
|
||||
** Cache-Control the cache policies say it should. One cdn_origin_check row is
|
||||
** written per hostname and run: objects checked, correct, missing or wrong, and
|
||||
** unreadable (the request failed, or the origin answered 4xx or 5xx).
|
||||
**
|
||||
** The sample paths are "/", the path of every enabled PREFIX policy that applies
|
||||
** to the hostname, and the literal (placeholder-free) invalidation paths of the
|
||||
** enabled tag rules, up to MAX_PATHS. The origin is the provider's originHost when
|
||||
** it has one, otherwise the hostname itself. Requests are HTTPS GETs, never follow
|
||||
** redirects, and refuse hosts that resolve to private, loopback or link-local
|
||||
** addresses.
|
||||
**
|
||||
** A response is correct when it has a Cache-Control header and, when an enabled
|
||||
** policy matches the path, the header equals what the policy would send (compared
|
||||
** ignoring case and spaces). With no enabled policy for the path, any Cache-Control
|
||||
** header is correct: the check proves the origin sets one at all.
|
||||
*******************************************************************************/
|
||||
public final class CdnOriginCheck
|
||||
{
|
||||
public static final int MAX_PATHS = 20;
|
||||
|
||||
private static volatile Fetcher fetcher = new HttpFetcher();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One origin answer: status and lower-cased header names.
|
||||
*******************************************************************************/
|
||||
public record Response(int status, Map<String, String> headers)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** How the origin is asked (replaceable in tests).
|
||||
*******************************************************************************/
|
||||
public interface Fetcher
|
||||
{
|
||||
/*******************************************************************************
|
||||
** GET https://host/path; throws when the origin cannot be read at all.
|
||||
*******************************************************************************/
|
||||
Response fetch(String host, String path) throws IOException;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The outcome of checking one hostname.
|
||||
*******************************************************************************/
|
||||
public record Result(String scope, int checked, int correct, int wrong, int unreadable, List<String> problems)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnOriginCheck()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Replace how origins are asked; null restores the real HTTP fetcher.
|
||||
*******************************************************************************/
|
||||
public static void setFetcher(Fetcher newFetcher)
|
||||
{
|
||||
fetcher = newFetcher == null ? new HttpFetcher() : newFetcher;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The sample paths of a hostname.
|
||||
*******************************************************************************/
|
||||
public static List<String> samplePaths(String hostname) throws QException
|
||||
{
|
||||
Set<String> paths = new LinkedHashSet<>();
|
||||
paths.add("/");
|
||||
for(CdnResponseHeaderPolicy.Policy policy : CdnResponseHeaderPolicy.current())
|
||||
{
|
||||
if("PREFIX".equals(policy.matchType()) && policy.matchValue().startsWith("/") && (policy.hostname() == null || policy.hostname().equalsIgnoreCase(hostname)))
|
||||
{
|
||||
paths.add(policy.matchValue());
|
||||
}
|
||||
}
|
||||
for(QRecord rule : new QueryAction().execute(new QueryInput(CdnTables.TABLE_TAG_RULE)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)))).getRecords())
|
||||
{
|
||||
String text = rule.getValueString("invalidationPaths");
|
||||
for(String line : text == null ? new String[0] : text.split("\\R"))
|
||||
{
|
||||
String path = line.strip();
|
||||
if(path.startsWith("/") && !path.contains("{") && !path.contains("*"))
|
||||
{
|
||||
paths.add(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
return (new ArrayList<>(paths).subList(0, Math.min(MAX_PATHS, paths.size())));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check one hostname row (and record nothing; see record()).
|
||||
*******************************************************************************/
|
||||
public static Result check(QRecord hostname) throws QException
|
||||
{
|
||||
String name = hostname.getValueString("hostname");
|
||||
String origin = originOf(hostname, name);
|
||||
int correct = 0;
|
||||
int wrong = 0;
|
||||
int unread = 0;
|
||||
List<String> problems = new ArrayList<>();
|
||||
List<String> paths = samplePaths(name);
|
||||
for(String path : paths)
|
||||
{
|
||||
Response response;
|
||||
try
|
||||
{
|
||||
response = fetcher.fetch(origin, path);
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
unread++;
|
||||
problems.add(path + ": could not be read (" + e.getMessage() + ")");
|
||||
continue;
|
||||
}
|
||||
if(response.status() >= 400)
|
||||
{
|
||||
unread++;
|
||||
problems.add(path + ": the origin answered " + response.status());
|
||||
continue;
|
||||
}
|
||||
String actual = response.headers().get("cache-control");
|
||||
if(!StringUtils.hasContent(actual))
|
||||
{
|
||||
wrong++;
|
||||
problems.add(path + ": no Cache-Control header");
|
||||
continue;
|
||||
}
|
||||
CdnResponseHeaderPolicy.Decision decision = CdnResponseHeaderPolicy.decide(
|
||||
new CdnResponseHeaderPolicy.Request(name, path, null, null, false, false, List.of(), List.of()), null);
|
||||
if(decision.policyName() != null && !normalize(decision.cacheControl()).equals(normalize(actual)))
|
||||
{
|
||||
wrong++;
|
||||
problems.add(path + ": Cache-Control is \"" + actual + "\" but the policy " + decision.policyName() + " says \"" + decision.cacheControl() + "\"");
|
||||
}
|
||||
else
|
||||
{
|
||||
correct++;
|
||||
}
|
||||
}
|
||||
return (new Result(name, paths.size(), correct, wrong, unread, problems));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check a hostname and write its cdn_origin_check row; returns the result.
|
||||
*******************************************************************************/
|
||||
public static Result checkAndRecord(QRecord hostname) throws QException
|
||||
{
|
||||
Result result = check(hostname);
|
||||
new InsertAction().execute(new InsertInput(CdnTables.ORIGIN_CHECK).withRecord(new QRecord()
|
||||
.withValue("checkDate", CdnRuntime.clock().instant())
|
||||
.withValue("scope", result.scope())
|
||||
.withValue("objectsChecked", result.checked())
|
||||
.withValue("correct", result.correct())
|
||||
.withValue("missingOrWrong", result.wrong())
|
||||
.withValue("unreadable", result.unreadable())));
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The host to ask: the provider's origin host, else the hostname.
|
||||
*******************************************************************************/
|
||||
private static String originOf(QRecord hostname, String fallback) throws QException
|
||||
{
|
||||
java.io.Serializable providerId = hostname.getValue("cdnProviderId");
|
||||
if(providerId != null)
|
||||
{
|
||||
for(QRecord provider : new QueryAction().execute(new QueryInput(CdnTables.PROVIDER)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, providerId)))).getRecords())
|
||||
{
|
||||
String originHost = provider.getValueString("originHost");
|
||||
if(StringUtils.hasContent(originHost))
|
||||
{
|
||||
return (stripPort(originHost.strip()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return (fallback);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The host without a port: "[v6]:443" gives v6, "name:443" gives name, and a bare
|
||||
** IPv6 literal or a plain name is left alone.
|
||||
*******************************************************************************/
|
||||
static String stripPort(String hostAndPort)
|
||||
{
|
||||
if(hostAndPort.startsWith("[") && hostAndPort.contains("]"))
|
||||
{
|
||||
return (hostAndPort.substring(1, hostAndPort.indexOf(']')));
|
||||
}
|
||||
int colon = hostAndPort.indexOf(':');
|
||||
if(colon > 0 && colon == hostAndPort.lastIndexOf(':') && hostAndPort.substring(colon + 1).matches("\\d+"))
|
||||
{
|
||||
return (hostAndPort.substring(0, colon));
|
||||
}
|
||||
return (hostAndPort);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Lower case, no spaces.
|
||||
*******************************************************************************/
|
||||
private static String normalize(String cacheControl)
|
||||
{
|
||||
return (cacheControl == null ? "" : cacheControl.toLowerCase(Locale.ROOT).replace(" ", ""));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether an address is one an origin check may connect to: public unicast only.
|
||||
** IPv4 refuses loopback, private, link-local, carrier-grade NAT (100.64.0.0/10),
|
||||
** 0.0.0.0/8, benchmarking (198.18.0.0/15), documentation, multicast and reserved.
|
||||
** IPv6 is an allowlist: only global unicast 2000::/3 passes, minus Teredo
|
||||
** (2001::/32) and documentation (2001:db8::/32); 6to4 (2002::/16) passes only
|
||||
** when its embedded IPv4 is public. The forms that embed an IPv4 address (::/96,
|
||||
** v4-mapped, IPv4-translated, NAT64 64:ff9b::/96) are unwrapped and judged as
|
||||
** IPv4; the local-use NAT64 prefix 64:ff9b:1::/48 is refused.
|
||||
*******************************************************************************/
|
||||
public static boolean isPublic(InetAddress address)
|
||||
{
|
||||
byte[] b = address.getAddress();
|
||||
if(b.length == 4)
|
||||
{
|
||||
return (isPublicV4(b[0] & 0xff, b[1] & 0xff, b[2] & 0xff));
|
||||
}
|
||||
return (isPublicV6(b));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
static boolean isPublicV6(byte[] b)
|
||||
{
|
||||
if(zero(b, 0, 12) || (zero(b, 0, 10) && ff(b, 10)) || (zero(b, 0, 8) && ff(b, 8) && b[10] == 0 && b[11] == 0))
|
||||
{
|
||||
return (isPublicV4(b[12] & 0xff, b[13] & 0xff, b[14] & 0xff));
|
||||
}
|
||||
if((b[0] & 0xff) == 0x00 && (b[1] & 0xff) == 0x64 && (b[2] & 0xff) == 0xff && (b[3] & 0xff) == 0x9b)
|
||||
{
|
||||
return (zero(b, 4, 12) && isPublicV4(b[12] & 0xff, b[13] & 0xff, b[14] & 0xff));
|
||||
}
|
||||
if((b[0] & 0xe0) != 0x20)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if((b[0] & 0xff) == 0x20 && (b[1] & 0xff) == 0x01 && ((b[2] == 0 && b[3] == 0) || ((b[2] & 0xff) == 0x0d && (b[3] & 0xff) == 0xb8)))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if((b[0] & 0xff) == 0x20 && (b[1] & 0xff) == 0x02)
|
||||
{
|
||||
return (isPublicV4(b[2] & 0xff, b[3] & 0xff, b[4] & 0xff));
|
||||
}
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether bytes from (inclusive) to (exclusive) are all zero.
|
||||
*******************************************************************************/
|
||||
private static boolean zero(byte[] b, int from, int to)
|
||||
{
|
||||
for(int i = from; i < to; i++)
|
||||
{
|
||||
if(b[i] != 0)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether two bytes at an index are ff ff.
|
||||
*******************************************************************************/
|
||||
private static boolean ff(byte[] b, int at)
|
||||
{
|
||||
return ((b[at] & 0xff) == 0xff && (b[at + 1] & 0xff) == 0xff);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean isPublicV4(int a, int b, int c)
|
||||
{
|
||||
if(a == 0 || a == 10 || a == 127 || a >= 224)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 100 && b >= 64 && b <= 127)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 169 && b == 254)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 172 && b >= 16 && b <= 31)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 192 && (b == 168 || (b == 0 && (c == 0 || c == 2)) || (b == 88 && c == 99)))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 198 && (b == 18 || b == 19 || (b == 51 && c == 100)))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
return (!(a == 203 && b == 0 && c == 113));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The address to connect to: every address the name resolves to must be public,
|
||||
** and the first one is used, so the connection goes to what was checked.
|
||||
*******************************************************************************/
|
||||
static InetAddress pick(InetAddress[] resolved) throws IOException
|
||||
{
|
||||
if(resolved == null || resolved.length == 0)
|
||||
{
|
||||
throw (new IOException("the host did not resolve"));
|
||||
}
|
||||
for(InetAddress address : resolved)
|
||||
{
|
||||
if(!isPublic(address))
|
||||
{
|
||||
throw (new IOException("the host resolves to a non-public address"));
|
||||
}
|
||||
}
|
||||
return (resolved[0]);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read a status line and headers (and nothing else) from an HTTP/1.1 response,
|
||||
** within a total deadline and a size limit.
|
||||
*******************************************************************************/
|
||||
static Response readResponse(InputStream in, long deadlineNanos) throws IOException
|
||||
{
|
||||
String status = readLine(in, deadlineNanos);
|
||||
int total = status.length();
|
||||
String[] parts = status.split(" ", 3);
|
||||
if(parts.length < 2 || !parts[0].startsWith("HTTP/"))
|
||||
{
|
||||
throw (new IOException("not an HTTP response"));
|
||||
}
|
||||
int code;
|
||||
try
|
||||
{
|
||||
code = Integer.parseInt(parts[1]);
|
||||
}
|
||||
catch(NumberFormatException e)
|
||||
{
|
||||
throw (new IOException("bad status line"));
|
||||
}
|
||||
Map<String, String> headers = new LinkedHashMap<>();
|
||||
while(true)
|
||||
{
|
||||
String line = readLine(in, deadlineNanos);
|
||||
total += line.length();
|
||||
if(line.isEmpty())
|
||||
{
|
||||
return (new Response(code, headers));
|
||||
}
|
||||
if(total > 16_384)
|
||||
{
|
||||
throw (new IOException("the headers are too large"));
|
||||
}
|
||||
int colon = line.indexOf(':');
|
||||
if(colon > 0)
|
||||
{
|
||||
headers.merge(line.substring(0, colon).strip().toLowerCase(Locale.ROOT), line.substring(colon + 1).strip(), (x, y) -> x + ", " + y);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String readLine(InputStream in, long deadlineNanos) throws IOException
|
||||
{
|
||||
StringBuilder line = new StringBuilder();
|
||||
while(true)
|
||||
{
|
||||
if(System.nanoTime() > deadlineNanos)
|
||||
{
|
||||
throw (new IOException("the origin was too slow"));
|
||||
}
|
||||
int c = in.read();
|
||||
if(c < 0)
|
||||
{
|
||||
throw (new IOException("the connection closed early"));
|
||||
}
|
||||
if(c == '\n')
|
||||
{
|
||||
return (line.toString().stripTrailing());
|
||||
}
|
||||
line.append((char) c);
|
||||
if(line.length() > 8192)
|
||||
{
|
||||
throw (new IOException("a header line is too long"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The real fetcher: resolve once, refuse non-public addresses, connect to the
|
||||
** address that was checked (TLS with the name as SNI and for verification), send
|
||||
** one GET, read the status line and headers only, and close. No redirects, no body.
|
||||
*******************************************************************************/
|
||||
static final class HttpFetcher implements Fetcher
|
||||
{
|
||||
private static final ScheduledExecutorService WATCHDOG = Executors.newSingleThreadScheduledExecutor(runnable ->
|
||||
{
|
||||
Thread thread = new Thread(runnable, "cdn-origin-check-watchdog");
|
||||
thread.setDaemon(true);
|
||||
return (thread);
|
||||
});
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Response fetch(String host, String path) throws IOException
|
||||
{
|
||||
InetAddress address = pick(InetAddress.getAllByName(host));
|
||||
return (fetch(address, 443, host, path, Duration.ofSeconds(10)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch from one checked address within one hard deadline that covers connect,
|
||||
** the TLS handshake and the reads: at the deadline the socket is closed, which
|
||||
** fails whatever read or write is blocked.
|
||||
*******************************************************************************/
|
||||
Response fetch(InetAddress address, int port, String host, String path, Duration limit) throws IOException
|
||||
{
|
||||
long deadline = System.nanoTime() + limit.toNanos();
|
||||
try(Socket plain = new Socket())
|
||||
{
|
||||
ScheduledFuture<?> guard = WATCHDOG.schedule(() ->
|
||||
{
|
||||
try
|
||||
{
|
||||
plain.close();
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
// already closed
|
||||
}
|
||||
}, limit.toMillis(), TimeUnit.MILLISECONDS);
|
||||
try
|
||||
{
|
||||
plain.connect(new InetSocketAddress(address, port), (int) Math.min(5000, limit.toMillis()));
|
||||
plain.setSoTimeout(5000);
|
||||
SSLSocket socket = (SSLSocket) ((SSLSocketFactory) SSLSocketFactory.getDefault()).createSocket(plain, host, port, true);
|
||||
SSLParameters parameters = socket.getSSLParameters();
|
||||
parameters.setServerNames(List.of(new SNIHostName(host)));
|
||||
parameters.setEndpointIdentificationAlgorithm("HTTPS");
|
||||
socket.setSSLParameters(parameters);
|
||||
socket.startHandshake();
|
||||
socket.getOutputStream().write(("GET " + path + " HTTP/1.1\r\nHost: " + host + "\r\nUser-Agent: qbit-cdn-origin-check\r\nAccept: */*\r\nConnection: close\r\n\r\n")
|
||||
.getBytes(StandardCharsets.US_ASCII));
|
||||
socket.getOutputStream().flush();
|
||||
return (readResponse(new BufferedInputStream(socket.getInputStream()), deadline));
|
||||
}
|
||||
finally
|
||||
{
|
||||
guard.cancel(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.health;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.purge.CdnPurgeDispatcher;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnOperations;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCategory;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthDetails;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Central health checks reuse the read-only provider connection test.
|
||||
*******************************************************************************/
|
||||
public class CdnProviderHealthChecks implements HealthCheckSupplier
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Resolve the enabled provider rows each time the registry refreshes.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public List<HealthCheckMetaData> supply(QInstance qInstance) throws QException
|
||||
{
|
||||
List<HealthCheckMetaData> checks = new ArrayList<>();
|
||||
for(QRecord provider : new QueryAction().execute(new QueryInput(CdnTables.PROVIDER)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)))).getRecords())
|
||||
{
|
||||
Serializable id = provider.getValue("id");
|
||||
String name = "cdn.provider." + id.toString().toLowerCase(Locale.ROOT);
|
||||
checks.add(new HealthCheckMetaData().withName(name).withLabel(name).withComponent("cdn")
|
||||
.withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.CDN)
|
||||
.withIntervalSeconds(900).withTimeoutMs(20000).withCheckInstance(input -> run(id, input)));
|
||||
}
|
||||
return (checks);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reload configuration so disabled or deleted providers are not probed.
|
||||
** Never copy provider messages, addresses or exception text into outcomes.
|
||||
*******************************************************************************/
|
||||
private static HealthCheckOutcome run(Serializable id, HealthCheckInput input) throws QException
|
||||
{
|
||||
if(!Instant.now().isBefore(input.getDeadline()))
|
||||
{
|
||||
return (HealthCheckOutcome.down("CDN connection test timed out"));
|
||||
}
|
||||
QRecord provider = new GetAction().executeForRecord(new GetInput(CdnTables.PROVIDER).withPrimaryKey(id));
|
||||
if(provider == null || !Boolean.TRUE.equals(provider.getValueBoolean("enabled")))
|
||||
{
|
||||
return (HealthCheckOutcome.unknown("CDN provider is disabled or removed"));
|
||||
}
|
||||
CdnProviderConfig config = CdnPurgeDispatcher.configOf(provider);
|
||||
CdnProviderInterface adapter = CdnRuntime.provider(config.type());
|
||||
if(!(adapter instanceof CdnOperations operations))
|
||||
{
|
||||
return (HealthCheckOutcome.unknown("CDN provider does not support connection tests"));
|
||||
}
|
||||
return (probe(operations, config, input));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The existing SPI has no timeout parameter. Bound its wait by the remaining
|
||||
** deadline and interrupt the probe on timeout or runner cancellation.
|
||||
*******************************************************************************/
|
||||
private static HealthCheckOutcome probe(CdnOperations operations, CdnProviderConfig config, HealthCheckInput input)
|
||||
{
|
||||
long remaining = Duration.between(Instant.now(), input.getDeadline()).toNanos();
|
||||
if(remaining <= 0)
|
||||
{
|
||||
return (HealthCheckOutcome.down("CDN connection test timed out"));
|
||||
}
|
||||
FutureTask<CdnOperations.ConnectionTest> task = new FutureTask<>(() -> operations.connectionTest(config));
|
||||
Thread.startVirtualThread(task);
|
||||
try
|
||||
{
|
||||
CdnOperations.ConnectionTest test = task.get(remaining, TimeUnit.NANOSECONDS);
|
||||
HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed")
|
||||
: !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured")
|
||||
: HealthCheckOutcome.up("CDN connection test passed");
|
||||
// Only known provider states are safe; arbitrary status text may contain secrets.
|
||||
String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN";
|
||||
return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name())
|
||||
.withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured())));
|
||||
}
|
||||
catch(TimeoutException e)
|
||||
{
|
||||
return (HealthCheckOutcome.down("CDN connection test timed out"));
|
||||
}
|
||||
catch(InterruptedException e)
|
||||
{
|
||||
Thread.currentThread().interrupt();
|
||||
return (HealthCheckOutcome.down("CDN connection test interrupted"));
|
||||
}
|
||||
catch(ExecutionException e)
|
||||
{
|
||||
return (HealthCheckOutcome.down("CDN connection test failed"));
|
||||
}
|
||||
finally
|
||||
{
|
||||
task.cancel(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.model;
|
||||
|
||||
|
||||
import com.kingsrook.qbits.cdn.CdnQBitConfig;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnProcesses;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundle;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundleEntry;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Registers CDN help and binds its reusable record-action text to host tables.
|
||||
*******************************************************************************/
|
||||
public final class CdnHelp
|
||||
{
|
||||
public static final String RESOURCE = "help/qbit-cdn.yaml";
|
||||
public static final String APP_KEY = "app:cdnAdmin;slot:help";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnHelp()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Registration is metadata-only. Core's startup sync guards the absence of
|
||||
** helpContent, so hosts without the help tables can still install this qbit.
|
||||
** RC.7 supports suffix wildcards only; bind the dynamic purge names explicitly.
|
||||
*******************************************************************************/
|
||||
public static void register(QInstance qInstance, CdnQBitConfig config)
|
||||
{
|
||||
HelpBundles.register(qInstance, RESOURCE);
|
||||
var bundles = HelpBundles.of(qInstance).getBundles();
|
||||
HelpBundle bundle = bundles.get(bundles.size() - 1);
|
||||
HelpBundleEntry purgeHelp = bundle.getEntries().stream()
|
||||
.filter(entry -> entry.getKey().equals("app:cdnAdmin;slot:purgeRecords")).findFirst().orElseThrow();
|
||||
for(String table : config.getPurgeableTables())
|
||||
{
|
||||
bundle.getEntries().add(new HelpBundleEntry()
|
||||
.withKey("process:" + CdnProcesses.PURGE_PREFIX + table + ";slot:help")
|
||||
.withSummary(purgeHelp.getSummary()).withContent(purgeHelp.getContent()).withFormat(purgeHelp.getFormat()));
|
||||
}
|
||||
for(HelpBundleEntry entry : bundle.getEntries())
|
||||
{
|
||||
if(entry.getKey().startsWith("app:cdnAdmin;"))
|
||||
{
|
||||
entry.setKey(entry.getKey().replace("app:cdnAdmin;", "app:" + config.getAppName() + ";"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.model;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Signature format selected on each revalidation hook.
|
||||
*******************************************************************************/
|
||||
public enum CdnSignatureFormat implements PossibleValueEnum<String>
|
||||
{
|
||||
STANDARD_WEBHOOKS("STANDARD_WEBHOOKS", "Standard Webhooks"),
|
||||
LEGACY("LEGACY", "Legacy X-Revalidate-Signature");
|
||||
|
||||
private final String id;
|
||||
private final String label;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
CdnSignatureFormat(String id, String label)
|
||||
{
|
||||
this.id = id;
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for possibleValueId.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getPossibleValueId()
|
||||
{
|
||||
return (id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for possibleValueLabel.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (label);
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
@@ -54,11 +55,12 @@ public final class CdnTables
|
||||
public static final List<String> ALL = List.of(PROVIDER, HOSTNAME, CACHE_POLICY, TABLE_TAG_RULE, PURGE_REQUEST, REVALIDATION_HOOK,
|
||||
EDGE_STAT_DAILY, ORIGIN_CHECK, DRIFT_FINDING);
|
||||
|
||||
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
|
||||
public static final String PVS_STAGE = "cdn_hostname_stage";
|
||||
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
|
||||
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
|
||||
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
|
||||
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
|
||||
public static final String PVS_STAGE = "cdn_hostname_stage";
|
||||
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
|
||||
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
|
||||
public static final String PVS_SIGNATURE_FORMAT = "cdn_signature_format";
|
||||
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
|
||||
|
||||
|
||||
|
||||
@@ -95,6 +97,7 @@ public final class CdnTables
|
||||
QPossibleValueSource.newForEnum(PVS_PURGE_STATUS, CdnPurgeStatus.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_PURGE_SOURCE, CdnPurgeSource.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_POLICY_CLASS, CdnPolicyClass.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_SIGNATURE_FORMAT, CdnSignatureFormat.values()),
|
||||
QPossibleValueSource.newForTable(PROVIDER),
|
||||
QPossibleValueSource.newForTable(HOSTNAME)));
|
||||
}
|
||||
@@ -228,7 +231,8 @@ public final class CdnTables
|
||||
.withRecordLabelFormatAndFields("%s", "hostname")
|
||||
.withField(new QFieldMetaData("hostname", QFieldType.STRING).withIsRequired(true).withMaxLength(255))
|
||||
.withField(new QFieldMetaData("cdnProviderId", QFieldType.INTEGER).withIsRequired(true).withPossibleValueSourceName(PROVIDER))
|
||||
.withField(new QFieldMetaData("hostnameRole", QFieldType.STRING).withMaxLength(20))
|
||||
.withField(new QFieldMetaData("hostnameRole", QFieldType.STRING).withMaxLength(20)
|
||||
.withHelpContent(new QHelpContent("A label for your own reference (for example WEB, API or ASSETS). It is not used to change any behavior.")))
|
||||
.withField(new QFieldMetaData("stage", QFieldType.STRING).withIsRequired(true).withPossibleValueSourceName(PVS_STAGE).withDefaultValue(CdnHostnameStage.OFF.name()))
|
||||
.withField(new QFieldMetaData("purgeEnabled", QFieldType.BOOLEAN).withDefaultValue(true))
|
||||
.withField(new QFieldMetaData("notes", QFieldType.TEXT)));
|
||||
@@ -271,7 +275,8 @@ public final class CdnTables
|
||||
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withMaxLength(100))
|
||||
.withField(new QFieldMetaData("tagTemplates", QFieldType.TEXT))
|
||||
.withField(new QFieldMetaData("invalidationPaths", QFieldType.TEXT))
|
||||
.withField(new QFieldMetaData("onlyWhenFieldsChange", QFieldType.STRING).withMaxLength(1000))
|
||||
.withField(new QFieldMetaData("onlyWhenFieldsChange", QFieldType.STRING).withMaxLength(1000)
|
||||
.withHelpContent(new QHelpContent("Comma-separated, case-sensitive field names. Plain entries allow only those changes; !viewCount ignores viewCount changes. With only ignored entries, any other changed field counts. In mixed lists, ignored fields never count. Inserts and deletes are unaffected by this filter.")))
|
||||
.withField(new QFieldMetaData("publishedField", QFieldType.STRING).withMaxLength(100))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)));
|
||||
}
|
||||
@@ -318,6 +323,9 @@ public final class CdnTables
|
||||
.withField(new QFieldMetaData("url", QFieldType.STRING).withIsRequired(true).withMaxLength(500))
|
||||
.withField(new QFieldMetaData("fanOutDns", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData("secretRef", QFieldType.STRING).withMaxLength(100))
|
||||
.withField(new QFieldMetaData("signatureFormat", QFieldType.STRING).withPossibleValueSourceName(PVS_SIGNATURE_FORMAT).withDefaultValue(CdnSignatureFormat.STANDARD_WEBHOOKS.name()))
|
||||
.withField(new QFieldMetaData("allowPlainHttp", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData("internalAllowlist", QFieldType.TEXT))
|
||||
.withField(new QFieldMetaData("timeoutMs", QFieldType.INTEGER).withDefaultValue(5000))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.cdn.CdnQBitConfig;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.permissions.CdnPermissions;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import static com.kingsrook.qbits.cdn.processes.CdnProcesses.line;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The "Create defaults" table actions that give an empty CDN setup something to
|
||||
** start from. Each is a standard streamed ETL (Review lists the rows that will
|
||||
** be added, Result the rows that were), needs no selected records, needs
|
||||
** cdn.admin, and is insert-only: a row (or the whole kind of row) that exists is
|
||||
** never changed or duplicated.
|
||||
** <ul>
|
||||
** <li>Create default provider (cdn_provider): a "No CDN" provider, so changes
|
||||
** are recorded as would-be purges and nothing is sent, until a real CDN is
|
||||
** added;</li>
|
||||
** <li>Create default policies (cdn_cache_policy): two disabled examples
|
||||
** (immutable assets, never cache) to edit and turn on;</li>
|
||||
** <li>Create default rules (cdn_table_tag_rule): one tag rule per purgeable table
|
||||
** that has none (tags only; paths are the application's to add).</li>
|
||||
** </ul>
|
||||
** These are actions, not a database seed, because the qbit ships no DDL and a
|
||||
** host chooses its own column names; they work on whatever the backend is.
|
||||
*******************************************************************************/
|
||||
public final class CdnDefaultsProcesses
|
||||
{
|
||||
public static final String CREATE_PROVIDER = "cdnCreateDefaultProvider";
|
||||
public static final String CREATE_POLICIES = "cdnCreateDefaultPolicies";
|
||||
public static final String CREATE_RULES = "cdnCreateDefaultRules";
|
||||
|
||||
public static final String NONE_PROVIDER_NAME = "No CDN (record only)";
|
||||
public static final String ASSETS_POLICY_NAME = "Static assets (immutable)";
|
||||
public static final String NEVER_POLICY_NAME = "Never cache (default)";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnDefaultsProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The three actions.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes()
|
||||
{
|
||||
return (List.of(
|
||||
define(CREATE_PROVIDER, "Create default provider", CdnTables.PROVIDER, ProviderExtract.class,
|
||||
"This provider will be added. It records purges and sends nothing, until you add a real CDN."),
|
||||
define(CREATE_POLICIES, "Create default policies", CdnTables.CACHE_POLICY, PolicyExtract.class,
|
||||
"These policies will be added turned off, as examples to edit and turn on."),
|
||||
define(CREATE_RULES, "Create default rules", CdnTables.TABLE_TAG_RULE, RuleExtract.class,
|
||||
"These tag rules will be added, one for each purgeable table that has none.")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One table action.
|
||||
*******************************************************************************/
|
||||
private static QProcessMetaData define(String name, String label, String table, Class<? extends AbstractExtractStep> extract, String preview)
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, preview + " Existing rows are never changed.");
|
||||
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(extract, DefaultsTransform.class, DefaultsLoad.class, values);
|
||||
process.setName(name);
|
||||
process.setLabel(label);
|
||||
process.setIcon(new QIcon("playlist_add"));
|
||||
process.setTableName(table);
|
||||
process.setMinInputRecords(0);
|
||||
process.setPermissionRules(CdnTables.permissionRules());
|
||||
return (CdnProcesses.permission(process, CdnPermissions.ADMIN));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The rows the provider action would add: the No CDN provider, when the table
|
||||
** has no NONE provider at all.
|
||||
*******************************************************************************/
|
||||
public static List<Map<String, Serializable>> missingProviders() throws QException
|
||||
{
|
||||
for(QRecord row : all(CdnTables.PROVIDER))
|
||||
{
|
||||
if(CdnProviderType.NONE.name().equals(row.getValueString("providerType")))
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
}
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("name", NONE_PROVIDER_NAME);
|
||||
row.put("providerType", CdnProviderType.NONE.name());
|
||||
row.put("enabled", true);
|
||||
return (List.of(row));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The policies the action would add, by name: both disabled, so adding them
|
||||
** changes no response until an administrator turns one on.
|
||||
*******************************************************************************/
|
||||
public static List<Map<String, Serializable>> missingPolicies() throws QException
|
||||
{
|
||||
Set<String> existing = new HashSet<>();
|
||||
all(CdnTables.CACHE_POLICY).forEach(row -> existing.add(row.getValueString("name")));
|
||||
List<Map<String, Serializable>> rows = new ArrayList<>();
|
||||
if(!existing.contains(ASSETS_POLICY_NAME))
|
||||
{
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("name", ASSETS_POLICY_NAME);
|
||||
row.put("matchType", "GLOB");
|
||||
row.put("matchValue", "/assets/*");
|
||||
row.put("policyClass", "IMMUTABLE");
|
||||
row.put("browserMaxAge", 31536000);
|
||||
row.put("edgeMaxAge", 31536000);
|
||||
row.put("immutable", true);
|
||||
row.put("noStore", false);
|
||||
row.put("priority", 10);
|
||||
row.put("enabled", false);
|
||||
row.put("seeded", true);
|
||||
rows.add(row);
|
||||
}
|
||||
if(!existing.contains(NEVER_POLICY_NAME))
|
||||
{
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("name", NEVER_POLICY_NAME);
|
||||
row.put("matchType", "PREFIX");
|
||||
row.put("matchValue", "/");
|
||||
row.put("policyClass", "NEVER");
|
||||
row.put("immutable", false);
|
||||
row.put("noStore", true);
|
||||
row.put("priority", 1000);
|
||||
row.put("enabled", false);
|
||||
row.put("seeded", true);
|
||||
rows.add(row);
|
||||
}
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The rules the action would add: tags "table" and "table:id" for each purgeable
|
||||
** table without a rule (any rule, enabled or not).
|
||||
*******************************************************************************/
|
||||
public static List<Map<String, Serializable>> missingRules() throws QException
|
||||
{
|
||||
CdnQBitConfig config = CdnRuntime.getConfig();
|
||||
Set<String> have = new HashSet<>();
|
||||
all(CdnTables.TABLE_TAG_RULE).forEach(row -> have.add(row.getValueString("tableName")));
|
||||
List<Map<String, Serializable>> rows = new ArrayList<>();
|
||||
for(String table : config == null ? List.<String>of() : config.getPurgeableTables())
|
||||
{
|
||||
if(!have.contains(table))
|
||||
{
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("tableName", table);
|
||||
row.put("tagTemplates", table + ":{id}\n" + table);
|
||||
row.put("enabled", true);
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> all(String table) throws QException
|
||||
{
|
||||
return (new QueryAction().execute(new QueryInput(table)).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Extract the rows to add (computed; nothing is read from the selected records).
|
||||
*******************************************************************************/
|
||||
public abstract static class DefaultsExtract extends AbstractExtractStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** The table the rows go in.
|
||||
*******************************************************************************/
|
||||
protected abstract String table();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The rows that are missing.
|
||||
*******************************************************************************/
|
||||
protected abstract List<Map<String, Serializable>> missing() throws QException;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(Map<String, Serializable> values : missing())
|
||||
{
|
||||
QRecord record = new QRecord().withTableName(table());
|
||||
record.setValues(new LinkedHashMap<>(values));
|
||||
getRecordPipe().addRecord(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The provider rows.
|
||||
*******************************************************************************/
|
||||
public static class ProviderExtract extends DefaultsExtract
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected String table()
|
||||
{
|
||||
return (CdnTables.PROVIDER);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected List<Map<String, Serializable>> missing() throws QException
|
||||
{
|
||||
return (missingProviders());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The cache policy rows.
|
||||
*******************************************************************************/
|
||||
public static class PolicyExtract extends DefaultsExtract
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected String table()
|
||||
{
|
||||
return (CdnTables.CACHE_POLICY);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected List<Map<String, Serializable>> missing() throws QException
|
||||
{
|
||||
return (missingPolicies());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The tag rule rows.
|
||||
*******************************************************************************/
|
||||
public static class RuleExtract extends DefaultsExtract
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected String table()
|
||||
{
|
||||
return (CdnTables.TABLE_TAG_RULE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected List<Map<String, Serializable>> missing() throws QException
|
||||
{
|
||||
return (missingRules());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review: the rows that will be added.
|
||||
*******************************************************************************/
|
||||
public static class DefaultsTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine added = line(Status.OK, "default row will be added", "default rows will be added", "default row was added", "default rows were added");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
added.incrementCount();
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
added.addSelfToListIfAnyCount(lines);
|
||||
if(lines.isEmpty())
|
||||
{
|
||||
lines.add(new ProcessSummaryLine(Status.INFO, null, "Nothing to add: the defaults are already there."));
|
||||
}
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result: insert through the normal insert, so the user's own permissions apply.
|
||||
*******************************************************************************/
|
||||
public static class DefaultsLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine added = line(Status.OK, "default row will be added", "default rows will be added", "default row was added", "default rows were added");
|
||||
private final ProcessSummaryLine failed = line(Status.ERROR, "default row could not be added", "default rows could not be added", "default row could not be added", "default rows could not be added");
|
||||
private final List<String> problems = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord inserted : new InsertAction().execute(new InsertInput(input.getTableName()).withRecords(input.getRecords())).getRecords())
|
||||
{
|
||||
if(inserted.getErrors() == null || inserted.getErrors().isEmpty())
|
||||
{
|
||||
added.incrementCountAndAddPrimaryKey(inserted.getValue("id"));
|
||||
output.getRecords().add(inserted);
|
||||
}
|
||||
else
|
||||
{
|
||||
failed.incrementCount();
|
||||
inserted.getErrors().forEach(error -> problems.add(error.getMessage()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
added.addSelfToListIfAnyCount(lines);
|
||||
failed.setBulletsOfText(new ArrayList<>(problems));
|
||||
failed.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.processes;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.CdnQBitConfig;
|
||||
import com.kingsrook.qbits.cdn.checks.CdnOriginCheck;
|
||||
import com.kingsrook.qbits.cdn.model.CdnHostnameStage;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.permissions.CdnPermissions;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import static com.kingsrook.qbits.cdn.processes.CdnProcesses.line;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check Origin Headers: a record action on cdn_hostname (standard streamed ETL:
|
||||
** Review lists the hostnames, Result says how many sample objects had the right
|
||||
** Cache-Control) and a daily scheduled run over every hostname that is not Off.
|
||||
** The work is CdnOriginCheck; each run writes one cdn_origin_check row per hostname.
|
||||
*******************************************************************************/
|
||||
public final class CdnOriginCheckProcesses
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(CdnOriginCheckProcesses.class);
|
||||
|
||||
public static final String CHECK_ORIGIN = "cdnCheckOrigin";
|
||||
public static final String SCHEDULED_ORIGIN = "cdnScheduledCheckOrigin";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnOriginCheckProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The processes.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(CdnQBitConfig config)
|
||||
{
|
||||
List<QProcessMetaData> processes = new ArrayList<>();
|
||||
QProcessMetaData action = CdnProcesses.etl(CHECK_ORIGIN, "Check Origin Headers", CdnTables.HOSTNAME, CheckTransform.class, CheckLoad.class,
|
||||
"This is a preview of the hostnames whose origin will be asked for a few sample pages, to check their Cache-Control headers.");
|
||||
action.setIcon(new QIcon("fact_check"));
|
||||
processes.add(CdnProcesses.permission(action, CdnPermissions.ADMIN));
|
||||
|
||||
QProcessMetaData scheduled = new QProcessMetaData().withName(SCHEDULED_ORIGIN).withLabel("Check CDN Origin Headers").withIcon(new QIcon("schedule")).withIsHidden(true)
|
||||
.withPermissionRules(CdnTables.permissionRules())
|
||||
.withStep(new QBackendStepMetaData().withName("run").withCode(new QCodeReference(ScheduledStep.class)));
|
||||
if(config.getSchedulerName() != null)
|
||||
{
|
||||
scheduled.withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName()).withCronExpression("0 45 2 * * ?").withCronTimeZoneId("UTC"));
|
||||
}
|
||||
processes.add(CdnProcesses.permission(scheduled, CdnPermissions.ADMIN));
|
||||
return (processes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review: the hostnames that will be checked.
|
||||
*******************************************************************************/
|
||||
public static class CheckTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine hosts = line(Status.OK, "hostname will be checked", "hostnames will be checked", "hostname was checked", "hostnames were checked");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord hostname : input.getRecords())
|
||||
{
|
||||
hosts.incrementCountAndAddPrimaryKey(hostname.getValue("id"));
|
||||
output.getRecords().add(hostname);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
hosts.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result: check each hostname and record the row.
|
||||
*******************************************************************************/
|
||||
public static class CheckLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine clean = line(Status.OK, "hostname will be checked", "hostnames will be checked", "hostname has the right headers", "hostnames have the right headers");
|
||||
private final ProcessSummaryLine wrong = line(Status.WARNING, "page has a missing or wrong Cache-Control", "pages have a missing or wrong Cache-Control",
|
||||
"page has a missing or wrong Cache-Control", "pages have a missing or wrong Cache-Control");
|
||||
private final ProcessSummaryLine unreadable = line(Status.WARNING, "page could not be read", "pages could not be read", "page could not be read", "pages could not be read");
|
||||
private final List<String> problems = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord hostname : input.getRecords())
|
||||
{
|
||||
CdnOriginCheck.Result result = CdnOriginCheck.checkAndRecord(hostname);
|
||||
if(result.wrong() == 0 && result.unreadable() == 0)
|
||||
{
|
||||
clean.incrementCountAndAddPrimaryKey(hostname.getValue("id"));
|
||||
}
|
||||
wrong.incrementCount(result.wrong());
|
||||
unreadable.incrementCount(result.unreadable());
|
||||
result.problems().forEach(problem -> problems.add(result.scope() + problem));
|
||||
output.getRecords().add(hostname);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
clean.addSelfToListIfAnyCount(lines);
|
||||
wrong.setBulletsOfText(new ArrayList<>(problems));
|
||||
wrong.addSelfToListIfAnyCount(lines);
|
||||
unreadable.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled: every hostname that is not Off.
|
||||
*******************************************************************************/
|
||||
public static class ScheduledStep implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord hostname : new QueryAction().execute(new QueryInput(CdnTables.HOSTNAME)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("stage", QCriteriaOperator.NOT_EQUALS, CdnHostnameStage.OFF.name())))).getRecords())
|
||||
{
|
||||
try
|
||||
{
|
||||
CdnOriginCheck.checkAndRecord(hostname);
|
||||
}
|
||||
catch(RuntimeException | QException e)
|
||||
{
|
||||
LOG.warn("An origin check failed", e, logPair("hostname", hostname.getValueString("hostname")));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -167,6 +167,56 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Summary counts for requests that cannot be sent, shared by all actions.
|
||||
*******************************************************************************/
|
||||
private static class UnsentSummary
|
||||
{
|
||||
private final ProcessSummaryLine absent = line(Status.WARNING,
|
||||
"record has no enabled CDN provider", "records have no enabled CDN provider",
|
||||
"record had no enabled CDN provider (no purge request recorded)", "records had no enabled CDN provider (no purge requests recorded)");
|
||||
private final ProcessSummaryLine none = line(Status.WARNING,
|
||||
"No CDN provider configured: purges are recorded only, not sent.", "No CDN provider configured: purges are recorded only, not sent.",
|
||||
"purge request was recorded (not sent: no CDN provider)", "purge requests were recorded (not sent: no CDN provider)");
|
||||
private final ProcessSummaryLine off = line(Status.WARNING,
|
||||
"CDN hostnames are not live: purges are recorded only, not sent.", "CDN hostnames are not live: purges are recorded only, not sent.",
|
||||
"purge request was recorded (not sent: CDN hostnames are not live)", "purge requests were recorded (not sent: CDN hostnames are not live)");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count a skipped request; returns whether it cannot be sent.
|
||||
*******************************************************************************/
|
||||
boolean add(CdnPurgeStatus status)
|
||||
{
|
||||
if(status == CdnPurgeStatus.SKIPPED_NONE_PROVIDER)
|
||||
{
|
||||
none.incrementCount(1);
|
||||
return (true);
|
||||
}
|
||||
if(status == CdnPurgeStatus.SKIPPED_STAGE_OFF)
|
||||
{
|
||||
off.incrementCount(1);
|
||||
return (true);
|
||||
}
|
||||
return (false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Include only reasons encountered by this action.
|
||||
*******************************************************************************/
|
||||
void append(ArrayList<ProcessSummaryLineInterface> lines)
|
||||
{
|
||||
absent.addSelfToListIfAnyCount(lines);
|
||||
none.addSelfToListIfAnyCount(lines);
|
||||
off.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled: send due purges and poll in-progress ones.
|
||||
*******************************************************************************/
|
||||
@@ -189,6 +239,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class PurgeRecordsTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine records = line(Status.OK, "record will be purged", "records will be purged",
|
||||
"record was purged", "records were purged");
|
||||
private final ProcessSummaryLine noRule = line(Status.INFO, "record has no CDN tag rule (no change)", "records have no CDN tag rule (no change)",
|
||||
@@ -207,6 +258,7 @@ public final class CdnProcesses
|
||||
String table = input.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
|
||||
List<QRecord> rules = rules(table);
|
||||
TagRuleEngine engine = new TagRuleEngine();
|
||||
CdnPurgeStatus sending = CdnPurgeService.sendingStatus(null);
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
List<String> tags = new ArrayList<>();
|
||||
@@ -226,7 +278,10 @@ public final class CdnProcesses
|
||||
recordPaths.forEach(path -> paths.add(path.path()));
|
||||
record.setValue("cdnTags", String.join("\n", tags));
|
||||
record.setValue("cdnPaths", com.kingsrook.qbits.cdn.purge.PurgeText.paths(recordPaths));
|
||||
records.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(sending))
|
||||
{
|
||||
records.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -249,6 +304,7 @@ public final class CdnProcesses
|
||||
lines.add(pathLine);
|
||||
}
|
||||
noRule.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -256,10 +312,11 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Purge From CDN, execute: one purge request (sent at once, hubs included).
|
||||
** Purge From CDN, execute: record requests with hubs included.
|
||||
*******************************************************************************/
|
||||
public static class PurgeRecordsLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"purge request was recorded (sent within seconds)", "purge requests were recorded (sent within seconds)");
|
||||
|
||||
@@ -277,7 +334,17 @@ public final class CdnProcesses
|
||||
List<QRecord> rows = CdnPurgeService.enqueue(new CdnPurgeService.Request(CdnPurgeKind.TAG.name(),
|
||||
com.kingsrook.qbits.cdn.purge.PurgeText.tags(record.getValueString("cdnTags")),
|
||||
com.kingsrook.qbits.cdn.purge.PurgeText.paths(record.getValueString("cdnPaths")), true, CdnPurgeSource.RECORD_ACTION, table, record.getValue("id")));
|
||||
requested.incrementCount(rows.size());
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
unsent.absent.incrementCount(1);
|
||||
}
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
if(!unsent.add(CdnPurgeStatus.valueOf(row.getValueString("status"))))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -296,6 +363,7 @@ public final class CdnProcesses
|
||||
{
|
||||
transform.noRule.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -307,6 +375,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class RetryTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine retry = line(Status.OK, "purge request will be sent again", "purge requests will be sent again",
|
||||
"purge request was queued again", "purge requests were queued again");
|
||||
private final ProcessSummaryLine skip = line(Status.INFO, "purge request has not failed (no change)", "purge requests have not failed (no change)",
|
||||
@@ -325,7 +394,10 @@ public final class CdnProcesses
|
||||
String status = record.getValueString("status");
|
||||
if(CdnPurgeStatus.FAILED.name().equals(status) || CdnPurgeStatus.DEAD.name().equals(status) || CdnPurgeStatus.DEFERRED_BUDGET.name().equals(status))
|
||||
{
|
||||
retry.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(CdnPurgeService.sendingStatus(record.getValue("cdnProviderId"))))
|
||||
{
|
||||
retry.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
else
|
||||
@@ -346,6 +418,7 @@ public final class CdnProcesses
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
retry.addSelfToListIfAnyCount(lines);
|
||||
skip.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -353,10 +426,16 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Retry Purge, execute: back to PENDING (sent on the next dispatcher pass).
|
||||
** Retry Purge, execute: queue live providers or record why sending is skipped.
|
||||
*******************************************************************************/
|
||||
public static class RetryLoad extends AbstractLoadStep
|
||||
public static class RetryLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"purge request was queued again", "purge requests were queued again");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reset the rows.
|
||||
*******************************************************************************/
|
||||
@@ -366,7 +445,12 @@ public final class CdnProcesses
|
||||
List<QRecord> updates = new ArrayList<>();
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("status", CdnPurgeStatus.PENDING.name())
|
||||
CdnPurgeStatus status = CdnPurgeService.sendingStatus(record.getValue("cdnProviderId"));
|
||||
if(!unsent.add(status))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("status", status.name())
|
||||
.withValue("source", CdnPurgeSource.RETRY.name()).withValue("attempts", 0).withValue("nextAttemptDate", null)
|
||||
.withValue("errorMessage", null).withValue("completedDate", null).withValue("modifyDate", CdnRuntime.clock().instant()));
|
||||
}
|
||||
@@ -375,6 +459,24 @@ public final class CdnProcesses
|
||||
output.getRecords().addAll(new UpdateAction().execute(new UpdateInput(CdnTables.PURGE_REQUEST).withRecords(updates)).getRecords());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result lines from the requests recorded during execution.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
requested.addSelfToListIfAnyCount(lines);
|
||||
if(getTransformStep() instanceof RetryTransform transform)
|
||||
{
|
||||
transform.skip.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -384,6 +486,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class PurgeHostnameTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine providers = line(Status.WARNING,
|
||||
"CDN distribution will be purged completely (every hostname on it)", "CDN distributions will be purged completely (every hostname on them)",
|
||||
"CDN distribution purge was recorded", "CDN distribution purges were recorded");
|
||||
@@ -402,7 +505,10 @@ public final class CdnProcesses
|
||||
Serializable providerId = record.getValue("cdnProviderId");
|
||||
if(providerId != null && seen.add(providerId))
|
||||
{
|
||||
providers.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(CdnPurgeService.sendingStatus(providerId)))
|
||||
{
|
||||
providers.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -418,6 +524,7 @@ public final class CdnProcesses
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
providers.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -427,8 +534,14 @@ public final class CdnProcesses
|
||||
/*******************************************************************************
|
||||
** Purge Everything, execute.
|
||||
*******************************************************************************/
|
||||
public static class PurgeHostnameLoad extends AbstractLoadStep
|
||||
public static class PurgeHostnameLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"CDN distribution purge was recorded", "CDN distribution purges were recorded");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record an ALL purge for each provider.
|
||||
*******************************************************************************/
|
||||
@@ -437,11 +550,36 @@ public final class CdnProcesses
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
CdnPurgeService.enqueueFor(record.getValue("cdnProviderId"), new CdnPurgeService.Request(CdnPurgeKind.ALL.name(), List.of(), List.of(), true,
|
||||
List<QRecord> rows = CdnPurgeService.enqueueFor(record.getValue("cdnProviderId"), new CdnPurgeService.Request(CdnPurgeKind.ALL.name(), List.of(), List.of(), true,
|
||||
CdnPurgeSource.MANUAL, CdnTables.HOSTNAME, record.getValue("id")));
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
unsent.absent.incrementCount(1);
|
||||
}
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
if(!unsent.add(CdnPurgeStatus.valueOf(row.getValueString("status"))))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result lines from the requests recorded during execution.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
requested.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -525,7 +525,7 @@ public final class CdnProviderProcesses
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled: daily drift check (and connection test) for every operable
|
||||
** Scheduled: daily drift check for every operable
|
||||
** provider.
|
||||
*******************************************************************************/
|
||||
public static class ScheduledDriftStep implements BackendStep
|
||||
@@ -538,7 +538,6 @@ public final class CdnProviderProcesses
|
||||
{
|
||||
for(QRecord provider : operableProviders())
|
||||
{
|
||||
testAndRecord(provider);
|
||||
driftAndRecord(provider);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,6 +120,36 @@ public final class CdnPurgeService
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sending eligibility for one provider, or all enabled providers when null.
|
||||
** A pending status means at least one provider can send; otherwise the
|
||||
** status gives the same reason used when recording a new request.
|
||||
*******************************************************************************/
|
||||
public static CdnPurgeStatus sendingStatus(Serializable providerId) throws QException
|
||||
{
|
||||
QQueryFilter filter = new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true));
|
||||
if(providerId != null)
|
||||
{
|
||||
filter.addCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, providerId));
|
||||
}
|
||||
CdnPurgeStatus status = CdnPurgeStatus.SKIPPED_NONE_PROVIDER;
|
||||
for(QRecord provider : new QueryAction().execute(new QueryInput(CdnTables.PROVIDER).withFilter(filter)).getRecords())
|
||||
{
|
||||
CdnPurgeStatus candidate = initialStatus(provider);
|
||||
if(candidate == CdnPurgeStatus.PENDING)
|
||||
{
|
||||
return (candidate);
|
||||
}
|
||||
if(candidate == CdnPurgeStatus.SKIPPED_STAGE_OFF)
|
||||
{
|
||||
status = candidate;
|
||||
}
|
||||
}
|
||||
return (status);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Status of a new request for a provider.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -21,16 +21,18 @@ import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.cdn.model.CdnSignatureFormat;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
@@ -41,17 +43,20 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpClient;
|
||||
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpPolicy;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSigner;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Calls every enabled cdn_revalidation_hook before the CDN is purged, so the
|
||||
** edge never refetches a stale page (design spec section 4.5 step 3).
|
||||
**
|
||||
** Request: POST {"tags":[...]} with header
|
||||
** "X-Revalidate-Signature: sha256=<hex HMAC-SHA256 of the body>", keyed by
|
||||
** the secret in the environment variable the hook's secretRef names. With
|
||||
** fanOutDns, the hook's host is resolved and every address is called (one
|
||||
** per pod behind a headless Service). Every call must answer 2xx.
|
||||
** Request: POST {"tags":[...]} signed with core Standard Webhooks headers.
|
||||
** LEGACY keeps X-Revalidate-Signature with a hex HMAC of the body. The
|
||||
** secretRef names an environment variable. SafeHttpClient checks and pins
|
||||
** every address; fan-out preserves the original Host and TLS server name.
|
||||
*******************************************************************************/
|
||||
public class RevalidationHookRunner
|
||||
{
|
||||
@@ -59,9 +64,8 @@ public class RevalidationHookRunner
|
||||
|
||||
private static final ObjectMapper JSON = new ObjectMapper();
|
||||
|
||||
private final HttpClient http;
|
||||
private final Function<String, String> secrets;
|
||||
private final Resolver resolver;
|
||||
private final Function<String, String> secrets;
|
||||
private final Resolver resolver;
|
||||
|
||||
|
||||
|
||||
@@ -83,7 +87,7 @@ public class RevalidationHookRunner
|
||||
*******************************************************************************/
|
||||
public RevalidationHookRunner()
|
||||
{
|
||||
this(HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(), System::getenv,
|
||||
this(System::getenv,
|
||||
host -> java.util.Arrays.stream(InetAddress.getAllByName(host)).map(InetAddress::getHostAddress).toList());
|
||||
}
|
||||
|
||||
@@ -92,9 +96,8 @@ public class RevalidationHookRunner
|
||||
/*******************************************************************************
|
||||
** Constructor with collaborators.
|
||||
*******************************************************************************/
|
||||
public RevalidationHookRunner(HttpClient http, Function<String, String> secrets, Resolver resolver)
|
||||
public RevalidationHookRunner(Function<String, String> secrets, Resolver resolver)
|
||||
{
|
||||
this.http = http;
|
||||
this.secrets = secrets;
|
||||
this.resolver = resolver;
|
||||
}
|
||||
@@ -133,42 +136,63 @@ public class RevalidationHookRunner
|
||||
byte[] body = JSON.writeValueAsBytes(java.util.Map.of("tags", tags));
|
||||
String secretRef = hook.getValueString("secretRef");
|
||||
String secret = StringUtils.hasContent(secretRef) ? secrets.apply(secretRef) : null;
|
||||
String formatValue = hook.getValueString("signatureFormat");
|
||||
CdnSignatureFormat format = formatValue == null ? CdnSignatureFormat.LEGACY : CdnSignatureFormat.valueOf(formatValue);
|
||||
WebhookSecret signingSecret = secret != null && format == CdnSignatureFormat.STANDARD_WEBHOOKS ? WebhookSecret.parse(secret) : null;
|
||||
String messageId = UUID.randomUUID().toString();
|
||||
URI uri = URI.create(hook.getValueString("url"));
|
||||
List<URI> targets = new ArrayList<>();
|
||||
if(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns"))))
|
||||
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
|
||||
if(timeout != null && timeout <= 0)
|
||||
{
|
||||
for(String address : resolver.resolve(uri.getHost()))
|
||||
{
|
||||
String host = address.contains(":") ? "[" + address + "]" : address;
|
||||
targets.add(new URI(uri.getScheme(), null, host, uri.getPort(), uri.getPath(), uri.getQuery(), null));
|
||||
}
|
||||
throw (new QException("Revalidation hook timeoutMs must be positive"));
|
||||
}
|
||||
else
|
||||
String allowlist = hook.getValueString("internalAllowlist");
|
||||
SafeHttpPolicy policy = SafeHttpPolicy.defaults()
|
||||
.withAllowPlainHttp(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("allowPlainHttp"))))
|
||||
.withInternalAllowlist(allowlist == null ? List.of() : Arrays.asList(allowlist.split("[,\\s]+")))
|
||||
.withTotalTimeout(Duration.ofMillis(timeout == null ? 5000 : timeout));
|
||||
List<String> resolved = resolver.resolve(uri.getHost());
|
||||
InetAddress[] addresses = new InetAddress[resolved.size()];
|
||||
for(int i = 0; i < resolved.size(); i++)
|
||||
{
|
||||
targets.add(uri);
|
||||
addresses[i] = InetAddress.getByName(resolved.get(i));
|
||||
}
|
||||
if(targets.isEmpty())
|
||||
if(addresses.length == 0)
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " resolved to no addresses"));
|
||||
}
|
||||
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
|
||||
for(URI target : targets)
|
||||
try(SafeHttpClient validator = new SafeHttpClient(policy, host -> addresses))
|
||||
{
|
||||
HttpRequest.Builder request = HttpRequest.newBuilder(target)
|
||||
.timeout(Duration.ofMillis(timeout == null ? 5000 : timeout))
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofByteArray(body));
|
||||
validator.checkUrl(uri.toString());
|
||||
}
|
||||
boolean fanOut = Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns")));
|
||||
int calls = fanOut ? addresses.length : 1;
|
||||
for(int i = 0; i < calls; i++)
|
||||
{
|
||||
InetAddress[] targetAddresses = fanOut ? new InetAddress[] {addresses[i]} : addresses;
|
||||
Map<String, String> headers = new LinkedHashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
if(secret != null)
|
||||
{
|
||||
request.header(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
|
||||
if(format == CdnSignatureFormat.LEGACY)
|
||||
{
|
||||
headers.put(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
|
||||
}
|
||||
else
|
||||
{
|
||||
headers.putAll(WebhookSigner.headers(messageId, Instant.now(), body, List.of(signingSecret)));
|
||||
}
|
||||
}
|
||||
HttpResponse<String> response = http.send(request.build(), HttpResponse.BodyHandlers.ofString());
|
||||
if(response.statusCode() / 100 != 2)
|
||||
try(SafeHttpClient client = new SafeHttpClient(policy, host -> targetAddresses))
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " answered " + response.statusCode() + " from " + target.getHost()));
|
||||
int status = client.post(uri.toString(), headers, body).statusCode();
|
||||
if(status / 100 != 2)
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " answered " + status + " from " + uri.getHost()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return (targets.size());
|
||||
return (calls);
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
@@ -188,7 +212,7 @@ public class RevalidationHookRunner
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Hex HMAC-SHA256.
|
||||
** Legacy compatibility: hex HMAC-SHA256 over the body with raw UTF-8 key bytes.
|
||||
*******************************************************************************/
|
||||
public static String sign(String secret, byte[] body) throws Exception
|
||||
{
|
||||
|
||||
@@ -55,8 +55,8 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
** expires by TTL instead of being invalidated;
|
||||
** - publishedField: when set, only records that are (or were) published
|
||||
** count, so draft edits purge nothing;
|
||||
** - onlyWhenFieldsChange: comma list; an update that changes none of them
|
||||
** purges nothing.
|
||||
** - onlyWhenFieldsChange: comma list of watched fields; !field ignores a
|
||||
** field. With only ignored fields, any other changed field counts.
|
||||
**
|
||||
** A dotted placeholder "{blog.slug}" reads field "slug" of the record that
|
||||
** field "blogId" points to (the xId convention). The joined table is the one
|
||||
@@ -159,7 +159,27 @@ public class TagRuleEngine
|
||||
String watched = rule.getValueString("onlyWhenFieldsChange");
|
||||
if(current != null && old != null && StringUtils.hasContent(watched))
|
||||
{
|
||||
return (Arrays.stream(watched.split(",")).map(String::strip).filter(StringUtils::hasContent)
|
||||
Set<String> included = new LinkedHashSet<>();
|
||||
Set<String> ignored = new LinkedHashSet<>();
|
||||
for(String entry : watched.split(","))
|
||||
{
|
||||
String field = entry.strip();
|
||||
if(field.startsWith("!"))
|
||||
{
|
||||
ignored.add(field.substring(1).strip());
|
||||
}
|
||||
else if(StringUtils.hasContent(field))
|
||||
{
|
||||
included.add(field);
|
||||
}
|
||||
}
|
||||
if(included.isEmpty() && !ignored.isEmpty())
|
||||
{
|
||||
included.addAll(current.getValues().keySet());
|
||||
included.addAll(old.getValues().keySet());
|
||||
}
|
||||
included.removeAll(ignored);
|
||||
return (included.stream()
|
||||
.anyMatch(field -> !Objects.equals(current.getValue(field), old.getValue(field))));
|
||||
}
|
||||
return (true);
|
||||
|
||||
@@ -0,0 +1,487 @@
|
||||
bundle: qbit-cdn
|
||||
version: ${project.version}
|
||||
layer: QBIT
|
||||
entries:
|
||||
- key: app:cdnAdmin;slot:help
|
||||
summary: Manage your CDN connection, cache rules, purges and edge health.
|
||||
content: |
|
||||
A CDN keeps copies of your pages close to visitors. Use this app to configure how your application caches responses and clears old copies when records change.
|
||||
|
||||
The **Setup** card shows what you still need to configure. Open **Connection** for providers and hostnames, **Settings** for cache policies and tag rules, **Activity** for purge requests and revalidation hooks, and **Performance** for statistics, origin checks and drift findings.
|
||||
|
||||
Start with **Create default provider** and **Create default rules**. Add your real provider and hostnames when you are ready. Purges require an enabled provider and a Live or Locked hostname with purging enabled. With No CDN, requests are recorded but not sent; with no enabled provider, no request is recorded.
|
||||
|
||||
You can use **Purge From CDN** on tables your application makes purgeable, or **{{label:cdnPurgeHostname}}** on a hostname. Your access depends on the CDN view, purge, purge-all and admin permissions granted by your application.
|
||||
- key: table:cdn_provider;slot:help
|
||||
summary: The CDN service in front of the site, with the account details used to talk to it and its latest health.
|
||||
content: |
|
||||
A provider row says which CDN you use (or that you use none), which distribution or zone to clear, and which server settings hold the credentials. While its type is **No CDN**, purge requests are recorded but nothing is sent.
|
||||
|
||||
Credentials are never stored here. **{{label:cdn_provider.secretRef}}** names the prefix of server settings that hold them. After changing the type or distribution, run **{{label:cdnTestConnection}}**; it records the result and fills in the DNS target and origin.
|
||||
|
||||
Other actions: **{{label:cdnImportEdgeStats}}** and **{{label:cdnCheckDrift}}**. When a scheduler is configured, daily jobs run both automatically. Editing needs the CDN admin permission.
|
||||
|
||||
**How rows get created:** you add one with New (pick the type, the distribution or zone id, and name the environment variable that holds the API key; the key itself is never stored), or run Create default provider, which adds "No CDN (record only)": matching changes to configured purgeable tables are then recorded as a would-be purge and nothing is sent. Test Connection, Import Edge Stats and Check Config Drift are actions on a provider.
|
||||
- key: table:cdn_provider;field:name
|
||||
summary: A short name for this CDN connection, shown on the CDN home screen.
|
||||
- key: table:cdn_provider;field:providerType
|
||||
summary: Which CDN service this is. Choose No CDN until a real one is connected.
|
||||
content: |
|
||||
See the list for the values. Only **Amazon CloudFront** and **Souin (self-hosted)** can send purges today; the others are listed for future use. With **No CDN** nothing is sent and purge requests are marked "Not sent (no CDN)".
|
||||
- key: table:cdn_provider;field:enabled
|
||||
summary: Turn off to stop purging and checking this provider without deleting it.
|
||||
content: |
|
||||
Disabled providers get no new purge requests and are skipped by the daily jobs.
|
||||
- key: table:cdn_provider;field:accountRef
|
||||
summary: The account the CDN belongs to, for your own reference, such as the AWS account number.
|
||||
- key: table:cdn_provider;field:zoneOrDistributionId
|
||||
summary: 'The CDN''s own id for what to clear: the CloudFront distribution id, or the Souin API address.'
|
||||
content: |
|
||||
Required for purges and checks to work. Purges to the wrong id clear the wrong cache, so copy it exactly from the CDN console.
|
||||
- key: table:cdn_provider;field:secretRef
|
||||
summary: The name prefix of the server settings that hold the CDN credentials. Not the credentials themselves.
|
||||
content: |
|
||||
For CloudFront, a prefix such as CDN_AWS means the server must have CDN_AWS_ACCESS_KEY_ID and CDN_AWS_SECRET_ACCESS_KEY set. Credentials are set on the server by an administrator and never typed here.
|
||||
- key: table:cdn_provider;field:settingsJson
|
||||
summary: Extra provider settings as JSON, such as the flush window and the invalidation budget.
|
||||
content: |
|
||||
Use JSON for provider settings such as the flush window or path budget. Values are strings. Missing settings use the application defaults. Ask your administrator which settings your provider supports before changing them.
|
||||
- key: table:cdn_provider;field:dnsTarget
|
||||
summary: The address your DNS records should point at to use this CDN. Filled in by Test Connection.
|
||||
- key: table:cdn_provider;field:originHost
|
||||
summary: The server the CDN fetches pages from, with its port. Filled in by Test Connection.
|
||||
- key: table:cdn_provider;field:credentialRotatedDate
|
||||
summary: When the CDN credentials were last replaced. Enter it by hand after you rotate them.
|
||||
content: |
|
||||
Enter this date after you rotate the credentials. The home screen shows their age and warns from 75 days; choose a rotation policy appropriate for your application.
|
||||
- key: table:cdn_provider;field:maskedKeyId
|
||||
summary: A masked access key identifier, if supplied by your application.
|
||||
content: |
|
||||
This version does not populate this field. Never enter a secret here.
|
||||
- key: table:cdn_provider;field:lastHealthStatus
|
||||
summary: Result of the last connection test, OK or FAILED. Filled in by Test Connection and the daily check.
|
||||
- key: table:cdn_provider;field:lastHealthMessage
|
||||
summary: What the last connection test found, or why it failed. Filled in automatically.
|
||||
- key: table:cdn_provider;field:lastHealthCheckDate
|
||||
summary: When the connection was last tested, filled in automatically.
|
||||
- key: table:cdn_provider;field:lastSuccessfulTestDate
|
||||
summary: When a connection test last passed, filled in automatically.
|
||||
- key: table:cdn_hostname;slot:help
|
||||
summary: A web address served through the CDN, and how far along it is in being switched over.
|
||||
content: |
|
||||
Add one row per hostname, such as the site, the API and the media host. **{{label:cdn_hostname.stage}}** lets you roll the CDN out gradually: purges are only sent when at least one hostname of the provider is **Live** or **Locked to CDN** and has purging on.
|
||||
|
||||
Use **{{label:cdnPurgeHostname}}** to clear a provider's whole cache. Editing needs the CDN admin permission.
|
||||
|
||||
**How rows get created:** you add one with New for each hostname the CDN serves, at stage Off, then move it to Headers only and Live as the rollout proceeds. Only Live and Locked hostnames with purging enabled are sent purges. Check Origin Headers is an action on a hostname, and hostnames that are not Off are checked nightly when your application configures a scheduler. The role field is a label for your own reference and changes nothing.
|
||||
- key: table:cdn_hostname;field:hostname
|
||||
summary: Enter the hostname without a scheme or path, for example www.example.com.
|
||||
- key: table:cdn_hostname;field:cdnProviderId
|
||||
summary: The CDN provider that serves this hostname.
|
||||
- key: table:cdn_hostname;field:hostnameRole
|
||||
summary: What the hostname is used for, such as SITE, API or MEDIA. A label for your own reference.
|
||||
- key: table:cdn_hostname;field:stage
|
||||
summary: How far this hostname is switched over to the CDN.
|
||||
content: |
|
||||
- **Off** (default: {{default:cdn_hostname.stage}}): not using the CDN yet.
|
||||
- **Headers only**: a preparation step. Purges are not sent yet.
|
||||
- **Live**: in use, and purges are sent.
|
||||
- **Locked to CDN**: the final step, locked to the CDN. Purges are sent.
|
||||
|
||||
If no hostname of the provider is Live or Locked, new purge requests are marked "Not sent (hostnames not live)".
|
||||
- key: table:cdn_hostname;field:purgeEnabled
|
||||
summary: Whether purges count for this hostname. Turn off to stop purging for it.
|
||||
content: |
|
||||
A purge is only sent when at least one hostname of the provider is Live or Locked and has this on.
|
||||
- key: table:cdn_hostname;field:notes
|
||||
summary: Free-form notes about this hostname.
|
||||
- key: table:cdn_cache_policy;slot:help
|
||||
summary: Rules that decide how long browsers and the CDN may keep a page or file before asking again.
|
||||
content: |
|
||||
Each rule matches requests by address, kind of route or table, and gives the cache lifetimes to send. The rule with the lowest **{{label:cdn_cache_policy.priority}}** number that matches wins. A rule tied to a **Hostname** applies only there.
|
||||
|
||||
Responses to signed-in visitors, and responses that set a cookie, are always sent as private and never cached, whatever the rules say. A rule with **Never Store** also sends private, no-store.
|
||||
|
||||
Changes can take up to a minute to be noticed. Editing needs the CDN admin permission.
|
||||
|
||||
**How rows get created:** you add one with New, or run Create default policies, which adds two turned-off examples (immutable static assets, never cache) to edit and turn on. A policy changes a response only where the application calls the CDN header policy integration when it answers; the qbit does not rewrite responses on its own. A response that sets a cookie or answers a signed-in user is always private, whatever the policies say.
|
||||
- key: table:cdn_cache_policy;field:name
|
||||
summary: A short name that says what the rule is for, such as Static assets.
|
||||
- key: table:cdn_cache_policy;field:cdnHostnameId
|
||||
summary: Limits the rule to one hostname. Leave empty to apply it to all hostnames.
|
||||
- key: table:cdn_cache_policy;field:matchType
|
||||
summary: How the rule picks the requests it applies to.
|
||||
content: |
|
||||
How the rule picks requests: PREFIX, GLOB, ROUTE_CLASS or TABLE.
|
||||
|
||||
- **PREFIX**: the address starts with the value, for example /blogs/.
|
||||
- **GLOB**: the address matches a pattern where * stands for anything.
|
||||
- **ROUTE_CLASS**: the kind of route the site assigns to the page.
|
||||
- **TABLE**: the response is for records of the named table.
|
||||
Any other value never matches.
|
||||
- key: table:cdn_cache_policy;field:matchValue
|
||||
summary: What to match against, in the form the match type needs, such as /blogs/ for a prefix.
|
||||
- key: table:cdn_cache_policy;field:policyClass
|
||||
summary: 'What kind of content the rule is for: {{values:cdn_policy_class}}.'
|
||||
content: |
|
||||
- **{{label:cdn_cache_policy.immutable}}**: files that never change at the same address.
|
||||
- **Leaf**: a single page or record.
|
||||
- **Hub**: a list page that changes whenever its items do.
|
||||
- **Never cache**: private or live content.
|
||||
It describes the rule; the lifetimes and flags below decide what is actually sent.
|
||||
- key: table:cdn_cache_policy;field:browserMaxAge
|
||||
summary: Seconds a visitor's browser may reuse the page without asking. Empty counts as 0.
|
||||
- key: table:cdn_cache_policy;field:edgeMaxAge
|
||||
summary: Seconds the CDN may keep its copy, sent as s-maxage. Leave empty to not set one.
|
||||
- key: table:cdn_cache_policy;field:staleWhileRevalidate
|
||||
summary: Seconds an out-of-date copy may still be shown while a fresh one is fetched in the background.
|
||||
- key: table:cdn_cache_policy;field:staleIfError
|
||||
summary: Seconds an out-of-date copy may be shown if the site is failing to answer.
|
||||
- key: table:cdn_cache_policy;field:immutable
|
||||
summary: Tells browsers the content at this address never changes, so they skip rechecking.
|
||||
content: |
|
||||
Only use for files whose address changes when their content does.
|
||||
- key: table:cdn_cache_policy;field:noStore
|
||||
summary: Never cache anything that matches this rule. It is sent as private, no-store.
|
||||
content: |
|
||||
The lifetimes on the rule are ignored when this is on.
|
||||
- key: table:cdn_cache_policy;field:priority
|
||||
summary: 'Which rule wins when several match: the lowest number goes first. Default {{default:cdn_cache_policy.priority}}.'
|
||||
- key: table:cdn_cache_policy;field:enabled
|
||||
summary: Turn off to ignore this rule without deleting it.
|
||||
- key: table:cdn_cache_policy;field:seeded
|
||||
summary: Marks example policies created by Create default policies; you cannot edit this flag.
|
||||
- key: table:cdn_table_tag_rule;slot:help
|
||||
summary: Says which cache tags and page addresses to purge when a record of a table is changed.
|
||||
content: |
|
||||
When a record in the table changes, the rule works out what to clear. **{{label:cdn_table_tag_rule.tagTemplates}}** are labels attached to cached copies. **{{label:cdn_table_tag_rule.invalidationPaths}}** are paths on the site. Both are filled from the record's own values, for example {slug}, or a related record's values, for example {blog.slug}.
|
||||
|
||||
One line per tag or address. A line is skipped when one of its values is empty. Put hub: in front of an address to mark it as a list page. List pages are only cleared by a manual **Purge From CDN**, not by automatic purges, because path-based providers bill for each path.
|
||||
|
||||
Editing needs the CDN admin permission.
|
||||
|
||||
**How rows get created:** you add one with New for each purgeable table the application lists, or run Create default rules, which adds a tags-only rule for every purgeable table that has none (tags "table:id" and "table"). Add invalidation paths where your CDN purges by path, and a published field so draft edits purge nothing.
|
||||
- key: table:cdn_table_tag_rule;field:tableName
|
||||
summary: The table whose changes this rule reacts to. Only tables that are set up as purgeable take effect.
|
||||
- key: table:cdn_table_tag_rule;field:tagTemplates
|
||||
summary: Cache tags to clear, one per line, filled in from the record, for example post:{id}.
|
||||
content: |
|
||||
A tag is a lowercase word, a colon, then printable characters without commas, at most 64 characters in all. Tags that do not fit that form are dropped. Providers that purge by path ignore tags.
|
||||
- key: table:cdn_table_tag_rule;field:invalidationPaths
|
||||
summary: Page addresses to clear, one per line; patterns may use record fields and a trailing wildcard.
|
||||
content: |
|
||||
Page addresses to clear, one per line, such as /blogs/{blog.slug}/{slug}*.
|
||||
|
||||
Each address must start with /. A trailing * means everything below it. Start a line with hub: for list pages such as /blogs. When there are many paths, the system merges them into wildcards to stay within the CDN's limits.
|
||||
- key: table:cdn_table_tag_rule;field:onlyWhenFieldsChange
|
||||
summary: Choose which changed fields trigger an automatic purge; leave empty for any change.
|
||||
content: |
|
||||
Enter comma-separated, case-sensitive field names. Plain names allow only those changes. Prefix a name with ! to ignore it, such as !viewCount. With only ignored names, any other changed field counts; in mixed lists ignored fields never count. This filter applies to updates only. The published-field filter still applies to inserts and deletes.
|
||||
- key: table:cdn_table_tag_rule;field:publishedField
|
||||
summary: The yes/no field that marks a record as live, such as isPublished. Unpublished records are ignored.
|
||||
content: |
|
||||
A change counts when the record is published now or was published before the change. So unpublishing a page still clears it, but editing a draft does nothing. Empty means all changes count.
|
||||
- key: table:cdn_table_tag_rule;field:enabled
|
||||
summary: Turn off to stop this rule without deleting it.
|
||||
- key: table:cdn_purge_request;slot:help
|
||||
summary: 'A record of one clear-the-cache request: what was cleared, why, and whether the CDN accepted it.'
|
||||
content: |
|
||||
Use **{{label:cdn_purge_request.status}}** and **{{label:cdn_purge_request.errorMessage}}** to follow a purge. Automatic changes are gathered within the configured flush window. Manual actions bypass that window, but still need a dispatcher run. Enabled revalidation hooks run before the CDN request.
|
||||
|
||||
Failed requests may retry with backoff. Non-retryable errors or the configured attempt limit make a request Dead. Use **{{label:cdnRetryPurge}}** for Failed, Dead or Deferred requests after resolving the cause.
|
||||
|
||||
**How rows get created:** matching changes to configured purgeable tables, **Purge From CDN**, or **{{label:cdnPurgeHostname}}** record requests for enabled providers. No CDN records a skipped request. Without an enabled provider, no new row is created. You cannot edit these rows by hand.
|
||||
- key: table:cdn_purge_request;field:cdnProviderId
|
||||
summary: The CDN provider this request is for.
|
||||
- key: table:cdn_purge_request;field:kind
|
||||
summary: Whether the request clears tags, paths or everything (TAG, PATHS or ALL). Set automatically.
|
||||
content: |
|
||||
Providers that cannot purge by tag, such as CloudFront, are sent paths instead.
|
||||
- key: table:cdn_purge_request;field:tags
|
||||
summary: The cache tags to clear, one per line.
|
||||
- key: table:cdn_purge_request;field:paths
|
||||
summary: 'The page addresses to clear, one per line. Lines starting with hub: are list pages.'
|
||||
content: |
|
||||
Once the request is sent this shows the paths actually used, after merging into wildcards.
|
||||
- key: table:cdn_purge_request;field:includeHubs
|
||||
summary: Whether list pages are included. Yes for actions you run, no for automatic purges.
|
||||
- key: table:cdn_purge_request;field:billablePaths
|
||||
summary: How many paths this request counts against the monthly invalidation allowance.
|
||||
content: |
|
||||
Set when the request is planned for a provider that bills by path, such as CloudFront. A whole-site purge counts as one.
|
||||
- key: table:cdn_purge_request;field:source
|
||||
summary: What caused this request.
|
||||
content: |
|
||||
- **Record change**: someone edited or published a record.
|
||||
- **Record action**: someone ran **Purge From CDN** on a record.
|
||||
- **Manual**: someone ran **{{label:cdnPurgeHostname}}**.
|
||||
- **Deploy**: sent as part of a deployment.
|
||||
- **Retry**: queued again with **{{label:cdnRetryPurge}}**.
|
||||
- key: table:cdn_purge_request;field:sourceTable
|
||||
summary: The table of the record that caused this request, if there was one.
|
||||
- key: table:cdn_purge_request;field:sourceRecordId
|
||||
summary: The id of the record that caused this request, when it was a single record.
|
||||
- key: table:cdn_purge_request;field:requestedBy
|
||||
summary: Who caused the request. Empty for automatic changes made without a signed-in user.
|
||||
- key: table:cdn_purge_request;field:status
|
||||
summary: 'Where the request is: Pending, Sent, Succeeded, Failed, Dead and others.'
|
||||
content: |
|
||||
- **Pending**: waiting for the flush window or the next run.
|
||||
- **Merged**: combined into another request, see **Merged Into**.
|
||||
- **Sent** and **In progress**: the CDN accepted it and is clearing copies.
|
||||
- **Succeeded**: the CDN finished.
|
||||
- **Failed (will retry)**: will be tried again after a pause.
|
||||
- **Dead**: gave up after repeated failures; use **{{label:cdnRetryPurge}}**.
|
||||
- **Deferred (budget)**: held because the monthly path budget would be exceeded; retry it later or raise the budget.
|
||||
- **Not sent (hostnames not live)**, **Not sent (no CDN)** and **Not sent (hub pages only)**: recorded but deliberately not sent.
|
||||
- key: table:cdn_purge_request;field:mergedIntoId
|
||||
summary: For a merged request, the main request it was combined into.
|
||||
- key: table:cdn_purge_request;field:attempts
|
||||
summary: How many times sending was tried.
|
||||
- key: table:cdn_purge_request;field:nextAttemptDate
|
||||
summary: When a failed request will next be tried. Empty when nothing is scheduled.
|
||||
- key: table:cdn_purge_request;field:providerRequestId
|
||||
summary: The CDN's own id for the request, such as a CloudFront invalidation id. Useful when asking the CDN about it.
|
||||
- key: table:cdn_purge_request;field:errorMessage
|
||||
summary: Why the last attempt failed, as reported by the CDN or the revalidation hooks.
|
||||
- key: table:cdn_purge_request;field:sentDate
|
||||
summary: When the request was sent to the CDN.
|
||||
- key: table:cdn_purge_request;field:completedDate
|
||||
summary: 'When the request reached a final state: finished, dead or deliberately not sent.'
|
||||
- key: table:cdn_revalidation_hook;slot:help
|
||||
summary: A call to the website's own servers, made before a purge, so they drop their copies too.
|
||||
content: |
|
||||
If your front end keeps its own cached pages, you can refresh them before clearing the CDN. Before each purge is sent to the CDN, every enabled hook is called with the list of cache tags that changed. If a hook fails, the purge is not sent and is retried later.
|
||||
|
||||
With **{{label:cdn_revalidation_hook.fanOutDns}}** on, the hook is called once for each address behind the host name, so every server hears about it. If a secret is named, the call is signed. The address is checked for safety first: internal addresses are blocked unless configured. Editing needs the CDN admin permission.
|
||||
|
||||
**How rows get created:** you add one with New: the URL, the signature format and the environment variable that holds the signing secret (never the secret). It is optional; add one only when a front end keeps its own cache.
|
||||
- key: table:cdn_revalidation_hook;field:name
|
||||
summary: A short name for the receiver, such as Website frontend pods.
|
||||
- key: table:cdn_revalidation_hook;field:url
|
||||
summary: The address that is called with a POST. It must answer with a success status.
|
||||
content: |
|
||||
The address is resolved and checked before the call. Redirects are not followed, and replies are read only up to 64 KiB.
|
||||
- key: table:cdn_revalidation_hook;field:fanOutDns
|
||||
summary: Call every server address behind the host name, one by one, instead of just one.
|
||||
content: |
|
||||
Use it when the name leads to several servers, such as a Kubernetes headless service, and each keeps its own cache.
|
||||
- key: table:cdn_revalidation_hook;field:secretRef
|
||||
summary: The name of the server setting holding the signing key. Not the key itself. Empty sends the call unsigned.
|
||||
- key: table:cdn_revalidation_hook;field:timeoutMs
|
||||
summary: How long to wait for each call to finish, in milliseconds. Must be above 0.
|
||||
content: |
|
||||
How long to wait for each call to finish, in milliseconds. Default {{default:cdn_revalidation_hook.timeoutMs}}. Must be above 0.
|
||||
- key: table:cdn_revalidation_hook;field:enabled
|
||||
summary: Turn off to skip this hook without deleting it.
|
||||
- key: table:cdn_edge_stat_daily;slot:help
|
||||
summary: 'Daily traffic figures for the CDN: requests, data served, cache hit rate and error rates.'
|
||||
content: |
|
||||
One row per provider and day, filled in automatically by **{{label:cdnImportEdgeStats}}** and by a daily job when a scheduler is configured. Each import covers the last 3 complete days and updates rows that already exist. Days are UTC days. You cannot edit these rows.
|
||||
|
||||
Figures come from the CDN's own monitoring. The cache hit rate is only available if the CDN's additional metrics are switched on; otherwise it stays empty.
|
||||
|
||||
**How rows get created:** Import Edge Stats on a provider writes it, and, when a scheduler is configured, a daily run does the same for every provider that supports it. A No CDN provider has no statistics.
|
||||
- key: table:cdn_edge_stat_daily;field:cdnProviderId
|
||||
summary: The CDN provider these figures came from.
|
||||
- key: table:cdn_edge_stat_daily;field:scope
|
||||
summary: What the figures cover. Currently the whole distribution.
|
||||
- key: table:cdn_edge_stat_daily;field:statDate
|
||||
summary: The UTC day the figures are for.
|
||||
- key: table:cdn_edge_stat_daily;field:requests
|
||||
summary: How many requests the CDN answered that day.
|
||||
- key: table:cdn_edge_stat_daily;field:bytesDownloaded
|
||||
summary: How much data the CDN sent to visitors that day, in bytes.
|
||||
- key: table:cdn_edge_stat_daily;field:cacheHitRate
|
||||
summary: The percentage of requests answered from the CDN's copy. Higher is better. Empty when not reported.
|
||||
- key: table:cdn_edge_stat_daily;field:error4xxRate
|
||||
summary: The percentage of requests that got a client error such as 404.
|
||||
- key: table:cdn_edge_stat_daily;field:error5xxRate
|
||||
summary: The percentage of requests that got a server error. This should stay near zero.
|
||||
- key: table:cdn_edge_stat_daily;field:importedDate
|
||||
summary: When these figures were last fetched from the CDN.
|
||||
- key: table:cdn_origin_check;slot:help
|
||||
summary: Results of asking a hostname's origin for sample pages and comparing their Cache-Control headers.
|
||||
content: |
|
||||
Use these results to find origin responses with missing or incorrect cache headers.
|
||||
|
||||
|
||||
**How rows get created:** Check Origin Headers on a hostname writes one row per hostname, and, when a scheduler is configured, a nightly run does the same for every hostname that is not Off. It requests the home page, the paths of prefix cache policies and the literal paths of tag rules over HTTPS and counts the right, missing or wrong, and unreadable ones.
|
||||
- key: table:cdn_origin_check;field:checkDate
|
||||
summary: When the check ran.
|
||||
- key: table:cdn_origin_check;field:scope
|
||||
summary: The hostname whose origin responses were checked.
|
||||
- key: table:cdn_origin_check;field:objectsChecked
|
||||
summary: How many sample paths were checked at the origin.
|
||||
- key: table:cdn_origin_check;field:correct
|
||||
summary: How many had the correct cache header.
|
||||
- key: table:cdn_origin_check;field:missingOrWrong
|
||||
summary: How many had a missing or wrong cache header. These should be fixed.
|
||||
- key: table:cdn_origin_check;field:unreadable
|
||||
summary: How many could not be read, so their header is unknown.
|
||||
- key: table:cdn_drift_finding;slot:help
|
||||
summary: A difference between how the CDN is actually set up and how it is supposed to be.
|
||||
content: |
|
||||
**{{label:cdnCheckDrift}}** (and a daily job when a scheduler is configured) reads the live CDN settings and compares them with the expected setup defined by the infrastructure code. Each difference becomes a finding with what was expected and what was found.
|
||||
|
||||
A finding stays open until a later check no longer sees it, then it is marked resolved. You cannot edit findings; fix the CDN or the expected setup instead.
|
||||
|
||||
**How rows get created:** Check Config Drift on a provider writes it, and, when a scheduler is configured, a nightly run does the same. It needs the expected configuration in the environment variable the provider names (CDN_EXPECTED_CONFIG by default); a finding is resolved when it no longer appears.
|
||||
- key: table:cdn_drift_finding;field:cdnProviderId
|
||||
summary: The CDN provider that was checked.
|
||||
- key: table:cdn_drift_finding;field:checkedDate
|
||||
summary: When the difference was last seen.
|
||||
- key: table:cdn_drift_finding;field:area
|
||||
summary: What differs, such as aliases, origin.domain, a behavior path or an origin header.
|
||||
- key: table:cdn_drift_finding;field:expected
|
||||
summary: What the expected setup says this should be.
|
||||
- key: table:cdn_drift_finding;field:actual
|
||||
summary: What the CDN actually has. (missing) means it is not set at all.
|
||||
- key: table:cdn_drift_finding;field:severity
|
||||
summary: How serious the difference is. Currently always WARNING.
|
||||
- key: table:cdn_drift_finding;field:resolvedDate
|
||||
summary: When a check stopped finding this difference. Empty while the finding is open.
|
||||
- key: process:cdnRetryPurge;slot:help
|
||||
summary: Queues failed, dead or deferred purge requests to be sent again.
|
||||
content: |
|
||||
Select purge requests and run it. Only requests that are **Failed**, **Dead** or **Deferred (budget)** are retried; anything else is skipped and reported as unchanged. Review the selected requests before continuing. A retried request has its attempt count reset and its source set to **Retry**. If sending is available, it becomes **Pending** for the next dispatcher run; otherwise its status explains why nothing was sent.
|
||||
|
||||
A deferred request is held back by the monthly path budget, so retrying it only helps if the budget has room or you raised it in the provider's settings.
|
||||
|
||||
Needs the CDN purge permission.
|
||||
- key: process:cdnPurgeHostname;slot:help
|
||||
summary: Clears the entire CDN cache for the provider this hostname belongs to.
|
||||
content: |
|
||||
This purges everything the CDN holds, for every hostname on that provider, not only the one you selected. Visitors get fresh pages, but the first requests are slower while the cache refills. For CloudFront it counts as one path of the monthly allowance.
|
||||
|
||||
If you select several hostnames of the same provider, one purge is recorded for it. The request is eligible for the next dispatcher run and is tracked under **CDN Purge Requests**. Use it sparingly; to clear one record, use **Purge From CDN** on that record instead.
|
||||
|
||||
Needs the CDN purge-all permission.
|
||||
|
||||
Review the affected providers before continuing. If no enabled real provider has a Live or Locked hostname with purging enabled, nothing is sent.
|
||||
- key: process:cdnDispatchPurges;slot:help
|
||||
summary: Send pending CDN purges and check requests already in progress.
|
||||
content: |
|
||||
Your application schedules this job when a scheduler is configured. It gathers requests within the flush window, calls revalidation hooks, sends purges and retries eligible failures. Check **{{label:cdn_purge_request}}** for the outcome.
|
||||
- key: process:cdnTestConnection;slot:help
|
||||
summary: Checks that the CDN can be reached with the saved credentials and records the result on the provider.
|
||||
content: |
|
||||
Select a provider and run it. For CloudFront it checks the credentials, the distribution, access to the invalidation list, and whether the origin verification header is set. The result is saved under **Last Health Status** and **Last Health Message**, and a pass saves the reported DNS target and origin when available.
|
||||
|
||||
Providers that cannot be tested, such as No CDN, are skipped. It reads from the CDN and changes nothing there. Needs the CDN admin permission.
|
||||
- key: process:cdnImportEdgeStats;slot:help
|
||||
summary: Fetches the last 3 complete days of traffic figures from the CDN and saves them.
|
||||
content: |
|
||||
Select a provider and run it. It reads daily requests, data served, cache hit rate and error rates from the CDN's monitoring and adds or updates one row per day in **CDN Edge Stats**. With a scheduler configured, a daily job does this automatically, so you only need it to refresh now.
|
||||
|
||||
Only CloudFront supports it; other providers are skipped. Needs the CDN admin permission.
|
||||
- key: process:cdnCheckDrift;slot:help
|
||||
summary: Compares the live CDN setup with the expected setup and records any differences.
|
||||
content: |
|
||||
Select a provider and run it. It reads the CDN's current configuration and compares it with the expected setup that the infrastructure code provides to the server. New differences are added to **CDN Drift Findings**; findings that no longer apply are marked resolved.
|
||||
|
||||
If the expected setup is not provided to the server, the check fails with a message saying so. Only CloudFront supports it. With a scheduler configured, a daily job runs the same check. Needs the CDN admin permission.
|
||||
- key: process:cdnScheduledImportEdgeStats;slot:help
|
||||
summary: Import recent CDN traffic statistics on the configured schedule.
|
||||
content: |
|
||||
When your application configures a scheduler, this job imports the last three complete UTC days for enabled providers that support statistics. Use **{{label:cdnImportEdgeStats}}** to refresh on demand.
|
||||
- key: process:cdnScheduledCheckDrift;slot:help
|
||||
summary: Check enabled CDN providers for configuration drift on the configured schedule.
|
||||
content: |
|
||||
When your application configures a scheduler, this job updates drift findings for supported providers. Connection health is checked separately by central System Health. Use **{{label:cdnCheckDrift}}** to check drift on demand.
|
||||
- key: widget:cdnHome;slot:label
|
||||
summary: 'The CDN at a glance: provider, hostnames, traffic, purge activity, invalidation budget, origin checks and
|
||||
drift.'
|
||||
content: |
|
||||
Each card says plainly when there is nothing yet, for example when no CDN is connected or no statistics have been imported. Traffic figures cover the last 7 and 30 days; purge failures cover 24 hours and 7 days.
|
||||
- key: possibleValueSource:cdn_provider_type;slot:help
|
||||
summary: The CDN services a provider can be.
|
||||
content: |
|
||||
- **Amazon CloudFront**: supported. Purges by path, with connection test, traffic import and drift check.
|
||||
- **Souin (self-hosted)**: supported. Purges by cache tag through its own API.
|
||||
- **No CDN**: nothing is sent; purge requests are only recorded.
|
||||
- **Cloudflare**, **Fastly**, **Bunny.net** and **Varnish (self-hosted)**: listed, but not yet usable for purging.
|
||||
- key: possibleValueSource:cdn_hostname_stage;slot:help
|
||||
summary: How far a hostname is along in being switched over to the CDN.
|
||||
content: |
|
||||
- **Off**: not using the CDN. If every hostname is Off, purges are not sent.
|
||||
- **Headers only**: preparation step; purges are not sent yet.
|
||||
- **Live**: in use; purges are sent.
|
||||
- **Locked to CDN**: the final step; purges are sent.
|
||||
- key: possibleValueSource:cdn_purge_status;slot:help
|
||||
summary: The stage of a purge request.
|
||||
content: |
|
||||
- **Pending**: waiting to be sent.
|
||||
- **Merged**: combined into another request.
|
||||
- **Sent**: handed to the CDN.
|
||||
- **In progress**: the CDN is still clearing copies.
|
||||
- **Succeeded**: finished.
|
||||
- **Failed (will retry)**: failed, will be tried again.
|
||||
- **Dead**: failed too many times; use **{{label:cdnRetryPurge}}**.
|
||||
- **Deferred (budget)**: held back by the monthly path budget.
|
||||
- **Not sent (hostnames not live)**: no hostname is Live yet.
|
||||
- **Not sent (no CDN)**: the provider is No CDN.
|
||||
- **Not sent (hub pages only)**: only list pages were affected, and automatic purges skip those.
|
||||
- key: possibleValueSource:cdn_purge_source;slot:help
|
||||
summary: What caused a purge request.
|
||||
content: |
|
||||
- **Record change**: a record was edited or published.
|
||||
- **Record action**: someone ran Purge From CDN on a record.
|
||||
- **Manual**: someone ran Purge Everything.
|
||||
- **Deploy**: part of a deployment.
|
||||
- **Retry**: queued again with Retry Purge.
|
||||
- key: possibleValueSource:cdn_policy_class;slot:help
|
||||
summary: The kind of content a cache rule is for.
|
||||
content: |
|
||||
- **Immutable**: files whose address changes when the content does, safe to keep a long time.
|
||||
- **Leaf**: a single page or record.
|
||||
- **Hub**: a list page that changes when any of its items do.
|
||||
- **Never cache**: private or live content that must not be kept.
|
||||
- key: table:cdn_revalidation_hook;field:signatureFormat
|
||||
summary: The signing format expected by the revalidation endpoint.
|
||||
content: |
|
||||
New hooks default to Standard Webhooks. Choose Legacy only for an endpoint expecting the older signature format. Signing uses **{{label:cdn_revalidation_hook.secretRef}}**.
|
||||
- key: table:cdn_revalidation_hook;field:allowPlainHttp
|
||||
summary: Allow this hook to call an HTTP endpoint without TLS; off by default.
|
||||
content: |
|
||||
Leave this off for HTTPS endpoints. Enabling it does not remove the internal-address checks.
|
||||
- key: table:cdn_revalidation_hook;field:internalAllowlist
|
||||
summary: Explicit exceptions to the hook's internal-address restrictions.
|
||||
content: |
|
||||
Separate IP addresses, CIDR blocks, exact host names or host suffixes beginning with a dot with commas or whitespace. Leave empty when the endpoint does not require access to an internal destination. Cloud metadata and unspecified addresses remain blocked.
|
||||
- key: process:cdnCheckOrigin;slot:help
|
||||
summary: Check sample origin responses for the expected cache headers.
|
||||
content: |
|
||||
Select hostnames and review the preview before continuing. You can then inspect **{{label:cdn_origin_check}}** for counts of correct, missing or wrong, and unreadable responses. The check uses HTTPS and samples the home page, prefix policy paths and literal tag-rule paths. It reads responses and does not change the origin.
|
||||
- key: process:cdnScheduledCheckOrigin;slot:help
|
||||
summary: Check origin cache headers for hostnames that are not Off.
|
||||
content: |
|
||||
When your application configures a scheduler, this job checks hostnames that are not Off. You can run **{{label:cdnCheckOrigin}}** on demand and inspect **{{label:cdn_origin_check}}** for results.
|
||||
- key: process:cdnCreateDefaultProvider;slot:help
|
||||
summary: Add a No CDN provider so you can record purges before connecting a real CDN.
|
||||
content: |
|
||||
Review the proposed provider before continuing. You get **No CDN (record only)** if no No CDN provider exists. Existing rows are never changed. You need CDN admin permission.
|
||||
- key: process:cdnCreateDefaultPolicies;slot:help
|
||||
summary: Add two disabled cache policy examples for you to review and configure.
|
||||
content: |
|
||||
Review the static-assets and never-cache examples before continuing. Only missing examples are added, by name; existing rows are never changed. Edit their matches and lifetimes before enabling them. Policies affect responses only where your application integrates the CDN header policy. You need CDN admin permission.
|
||||
- key: process:cdnCreateDefaultRules;slot:help
|
||||
summary: Add a tag rule for each configured purgeable table that has no rule.
|
||||
content: |
|
||||
Review the proposed rules before continuing. Existing rules, including disabled ones, are left alone. New rules contain tags but no invalidation paths. Add paths if your provider purges by path, and configure a published field if draft changes should be ignored. You need CDN admin permission.
|
||||
- key: app:cdnAdmin;slot:purgeRecords
|
||||
summary: Clear cached copies associated with the records you select.
|
||||
content: |
|
||||
Use **Purge From CDN** when cached content needs refreshing. Review the selected records and planned tags and paths before continuing. This action does not change your records. Manual purges include hub paths and bypass automatic change filters. Follow requests in **{{label:cdn_purge_request}}**. Sending still requires an enabled real provider, a Live or Locked hostname with purging enabled, and a dispatcher run. You need CDN purge permission.
|
||||
- key: possibleValueSource:cdn_signature_format;slot:help
|
||||
summary: Choose the signing format your revalidation endpoint expects.
|
||||
content: |
|
||||
Use Standard Webhooks for new endpoints, or Legacy for a receiver expecting the older signature. You must configure a signing secret to send signed calls.
|
||||
- key: possibleValueSource:cdn_provider;slot:help
|
||||
summary: Choose the CDN provider this record belongs to.
|
||||
content: |
|
||||
Choose a connection from **{{label:cdn_provider}}**. Configure that provider before using it for purges or checks.
|
||||
- key: possibleValueSource:cdn_hostname;slot:help
|
||||
summary: Choose the hostname this cache policy applies to.
|
||||
content: |
|
||||
Choose a hostname from **{{label:cdn_hostname}}**, or leave the policy hostname empty to apply it across hostnames.
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn;
|
||||
|
||||
|
||||
import java.net.URLClassLoader;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
|
||||
import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundleLayer;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
|
||||
import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule;
|
||||
import com.kingsrook.qqq.backend.core.testing.HelpBundleAssertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Coverage for shipped help and names bound from the host configuration.
|
||||
*******************************************************************************/
|
||||
class CdnHelpBundleTest
|
||||
{
|
||||
@TempDir
|
||||
Path temporary;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The resource covers all fixed metadata, without host objects or exclusions.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void shippedBundleIsComplete() throws Exception
|
||||
{
|
||||
QInstance instance = new QInstance();
|
||||
instance.addBackend(new QBackendMetaData().withName("memory").withBackendType(MemoryBackendModule.class));
|
||||
new CdnQBitProducer().withQBitConfig(new CdnQBitConfig().withBackendName("memory")).produce(instance);
|
||||
// Core adds bulk CRUD processes during enrichment; those belong to core.
|
||||
Set<String> ownedProcesses = Set.copyOf(instance.getProcesses().keySet());
|
||||
Set<String> ownedWidgets = Set.copyOf(instance.getWidgets().keySet());
|
||||
Set<String> ownedSources = Set.copyOf(instance.getPossibleValueSources().keySet());
|
||||
new QInstanceEnricher(instance).enrich();
|
||||
HelpBundleAssertions.forInstance(instance).bundles("help/qbit-cdn.yaml")
|
||||
.owning(target -> target.name().startsWith("cdn")
|
||||
&& (!target.kind().equals("process") || ownedProcesses.contains(target.name()))).assertComplete();
|
||||
assertThat(instance.getTable("helpContent")).isNull();
|
||||
var bundle = HelpBundles.of(instance).getBundles().get(0);
|
||||
assertThat(bundle.getId()).isEqualTo("qbit-cdn");
|
||||
assertThat(bundle.getVersion()).as("the help bundle carries the artifact version").isEqualTo(System.getProperty("qbit.artifactVersion"));
|
||||
assertThat(bundle.getLayer()).isEqualTo(HelpBundleLayer.QBIT);
|
||||
assertThat(bundle.getEntries()).extracting(entry -> entry.getKey()).doesNotHaveDuplicates();
|
||||
// Core's gate does not require widgets or possible-value sources yet.
|
||||
ownedWidgets.forEach(name -> assertThat(bundle.getEntries()).as(name)
|
||||
.anyMatch(entry -> entry.getKey().equals("widget:" + name + ";slot:label")));
|
||||
ownedSources.forEach(name -> assertThat(bundle.getEntries()).as(name)
|
||||
.anyMatch(entry -> entry.getKey().equals("possibleValueSource:" + name + ";slot:help")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check the registered bundle, including host-named actions and a renamed app,
|
||||
** through the core gate using a temporary classpath resource.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void configuredBundleIsComplete() throws Exception
|
||||
{
|
||||
QInstance instance = CdnTestSupport.setUp();
|
||||
var config = CdnRuntime.getConfig().withAppName("edgeConsole");
|
||||
QInstance renamed = new QInstance();
|
||||
instance.getBackends().values().forEach(renamed::addBackend);
|
||||
renamed.addTable(instance.getTable("post"));
|
||||
new CdnQBitProducer().withQBitConfig(config).produce(renamed);
|
||||
Set<String> ownedProcesses = Set.copyOf(renamed.getProcesses().keySet());
|
||||
new QInstanceEnricher(renamed).enrich();
|
||||
var bundle = HelpBundles.of(renamed).getBundles().get(0);
|
||||
assertThat(bundle.getEntries()).anyMatch(entry -> entry.getKey().equals("process:cdnPurge_post;slot:help")
|
||||
&& entry.getContent().contains("Review the selected records"));
|
||||
assertThat(bundle.getEntries()).anyMatch(entry -> entry.getKey().equals("app:edgeConsole;slot:help"));
|
||||
assertThat(bundle.getEntries()).noneMatch(entry -> entry.getKey().startsWith("app:cdnAdmin;"));
|
||||
ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
|
||||
mapper.writeValue(temporary.resolve("configured.yaml").toFile(), Map.of("bundle", bundle.getId(),
|
||||
"version", bundle.getVersion(), "layer", bundle.getLayer(), "entries", bundle.getEntries()));
|
||||
ClassLoader original = Thread.currentThread().getContextClassLoader();
|
||||
try(URLClassLoader loader = new URLClassLoader(new java.net.URL[] { temporary.toUri().toURL() }, original))
|
||||
{
|
||||
Thread.currentThread().setContextClassLoader(loader);
|
||||
HelpBundleAssertions.forInstance(renamed).bundles("configured.yaml")
|
||||
.owning(target -> (target.name().startsWith("cdn") || target.name().equals("edgeConsole"))
|
||||
&& (!target.kind().equals("process") || ownedProcesses.contains(target.name()))).assertComplete();
|
||||
mapper.writeValue(temporary.resolve("host.yaml").toFile(), Map.of("bundle", "host", "version", "1",
|
||||
"layer", "HOST", "entries", java.util.List.of(Map.of("key", "table:cdn_provider;slot:help", "summary", "Your host instructions."))));
|
||||
HelpBundles.register(renamed, "host.yaml");
|
||||
assertThat(HelpBundles.of(renamed).resolveWinningEntries()
|
||||
.get(new HelpBundles.EntryKey("table:cdn_provider;slot:help", "ALL_SCREENS")).entry().getSummary())
|
||||
.isEqualTo("Your host instructions.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
Thread.currentThread().setContextClassLoader(original);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -52,6 +52,7 @@ class CdnQBitProducerTest
|
||||
assertThat(instance.getProcess(CdnProcesses.PURGE_HOSTNAME)).isNotNull();
|
||||
assertThat(instance.getProcess(CdnProcesses.DISPATCH_PURGES).getIsHidden()).isTrue();
|
||||
assertThat(instance.getWidget(CdnWidgets.HOME_WIDGET)).isNotNull();
|
||||
assertThat(instance.getWidget("cdnHealthCard")).isNull();
|
||||
assertThat(instance.getApp("cdnAdmin").getSections()).extracting(section -> section.getLabel())
|
||||
.containsExactly("Connection", "Settings", "Activity", "Performance");
|
||||
assertThat(instance.getRecordChangeListeners()).isNotEmpty();
|
||||
|
||||
@@ -34,7 +34,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The CDN home and Health card: honest empty states, then real data.
|
||||
** The CDN home: honest empty states, then real data.
|
||||
*******************************************************************************/
|
||||
class CdnHomeTest
|
||||
{
|
||||
@@ -58,10 +58,9 @@ class CdnHomeTest
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
String html = render(CdnWidgets.HOME_WIDGET);
|
||||
assertThat(html).contains("No CDN connected yet.").contains("No edge statistics yet.").contains("No purges recorded yet.")
|
||||
.contains("No path-billed provider connected yet.").contains("No origin header check has run yet.").contains("Nothing to compare yet.")
|
||||
assertThat(html).contains("No CDN provider configured: purges are recorded only.").contains("No edge statistics yet.").contains("No purges recorded yet.")
|
||||
.contains("No path-billed provider connected yet.").contains("No origin check yet.").contains("Nothing to compare yet.")
|
||||
.contains("minmax(min(100%,22rem),1fr)");
|
||||
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN connected yet.").contains("href=\"/cdnAdmin\"");
|
||||
}
|
||||
|
||||
|
||||
@@ -91,7 +90,6 @@ class CdnHomeTest
|
||||
CdnTestSupport.insert(CdnTables.DRIFT_FINDING, new QRecord().withValue("cdnProviderId", provider).withValue("area", "BEHAVIOR").withValue("checkedDate", CdnTestSupport.NOW));
|
||||
|
||||
CdnHomeData data = CdnHomeData.load();
|
||||
assertEquals("ATTENTION", data.overallState());
|
||||
assertEquals(1, data.dead24h());
|
||||
assertEquals(4, data.budgets().get(0).billed());
|
||||
assertEquals(age, data.providers().get(0).credentialAgeDays());
|
||||
@@ -99,8 +97,7 @@ class CdnHomeTest
|
||||
String html = CdnHomeView.render(data, "/cdnAdmin");
|
||||
assertThat(html).contains("d1.cloudfront.net").contains(age + " days (rotate every 90) - rotation due").contains("1,500").contains("91.5%")
|
||||
.contains("<polyline").contains("396 of 396").contains("BEHAVIOR").contains("1 open").contains("/cdnAdmin/cdn_purge_request/")
|
||||
.contains("www.example.com");
|
||||
assertThat(CdnHomeView.renderHealthCard(data, "/cdnAdmin")).contains("Needs attention");
|
||||
.contains("www.example.com").doesNotContain("recorded only (not sent)", "nothing has been checked");
|
||||
}
|
||||
|
||||
|
||||
@@ -119,4 +116,30 @@ class CdnHomeTest
|
||||
assertEquals("50.0%", CdnHomeView.percent(new BigDecimal("0.5")));
|
||||
assertEquals("91.5%", CdnHomeView.percent(new BigDecimal("91.5")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** NONE and disabled providers do not imply a connection or a sent purge.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void recordedOnlyHome() throws Exception
|
||||
{
|
||||
for(boolean none : List.of(true, false))
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
Integer provider = CdnTestSupport.liveProvider(none ? CdnProviderType.NONE : CdnProviderType.CLOUDFRONT, null);
|
||||
if(!none)
|
||||
{
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.PROVIDER)
|
||||
.withRecord(new QRecord().withValue("id", provider).withValue("enabled", false)));
|
||||
}
|
||||
CdnTestSupport.insert(CdnTables.PURGE_REQUEST, new QRecord().withValue("cdnProviderId", provider).withValue("status", "SKIPPED_NONE_PROVIDER"));
|
||||
assertThat(render(CdnWidgets.HOME_WIDGET)).contains("No CDN provider configured: purges are recorded only.")
|
||||
.contains("The CloudFront distribution is created in the cutover window.").contains("recorded only (not sent)")
|
||||
.contains("No origin check yet.");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.checks;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The safety of the origin check's own HTTP code: which addresses it refuses
|
||||
** (every literal is parsed without DNS), that it connects to the address it
|
||||
** checked, and that it reads only a bounded status line and headers.
|
||||
*******************************************************************************/
|
||||
class CdnOriginCheckSafetyTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean isPublic(String literal) throws Exception
|
||||
{
|
||||
return (CdnOriginCheck.isPublic(InetAddress.getByName(literal)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Public unicast is allowed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void publicAddressesAreAllowed() throws Exception
|
||||
{
|
||||
for(String literal : List.of("93.184.216.34", "8.8.8.8", "100.63.255.255", "100.128.0.1", "172.15.0.1", "172.32.0.1", "198.17.0.1", "198.20.0.1", "2606:4700:4700::1111", "2001:4860:4860::8888"))
|
||||
{
|
||||
assertThat(isPublic(literal)).as(literal).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Everything non-public is refused, including cloud metadata addresses and the
|
||||
** IPv6 forms that carry an IPv4 address.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void nonPublicAddressesAreRefused() throws Exception
|
||||
{
|
||||
for(String literal : List.of("127.0.0.1", "10.1.2.3", "172.16.0.1", "172.31.255.255", "192.168.1.1", "169.254.169.254", "100.64.0.1", "100.100.100.200", "100.127.255.255",
|
||||
"0.0.0.0", "0.1.2.3", "198.18.0.1", "198.19.255.255", "224.0.0.1", "255.255.255.255", "192.0.2.1", "203.0.113.9",
|
||||
"::1", "::", "fe80::1", "fc00::1", "fd00:ec2::254", "fdff::1", "ff02::1",
|
||||
"::ffff:127.0.0.1", "::ffff:169.254.169.254", "::ffff:10.0.0.1", "::127.0.0.1", "64:ff9b::7f00:1", "64:ff9b::a9fe:a9fe", "2002:7f00:1::1", "2002:a9fe:a9fe::1"))
|
||||
{
|
||||
assertThat(isPublic(literal)).as(literal).isFalse();
|
||||
}
|
||||
assertThat(isPublic("::ffff:8.8.8.8")).as("a mapped public address is judged by its IPv4 part").isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** IPv6 is an allowlist, built from raw bytes so the IPv6 path is really used (Java
|
||||
** turns ::ffff:a.b.c.d literals into IPv4 addresses).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ipv6IsAnAllowlistOfGlobalUnicast() throws Exception
|
||||
{
|
||||
for(String hex : List.of("00000000000000000000ffff7f000001", "00000000000000000000ffffa9fea9fe", "0000000000000000ffff00007f000001", "0064ff9b0000000000000000a9fea9fe",
|
||||
"0064ff9b0001000000000000" + "7f000001", "20010db8000000000000000000000001", "01000000000000000000000000000001", "20010000000000000000000000000001",
|
||||
"2002 7f00 0001 0000 0000 0000 0000 0001".replace(" ", ""), "fc000000000000000000000000000001", "fe800000000000000000000000000001", "ff020000000000000000000000000001",
|
||||
"00000000000000000000000000000001", "00000000000000000000000000000000", "40000000000000000000000000000001"))
|
||||
{
|
||||
assertThat(CdnOriginCheck.isPublicV6(bytes(hex))).as(hex).isFalse();
|
||||
}
|
||||
for(String hex : List.of("2606470047000000000000000000" + "1111", "20014860486000000000000000008888", "0064ff9b000000000000000008080808", "00000000000000000000ffff08080808",
|
||||
"2002080808080000000000000000" + "0001"))
|
||||
{
|
||||
assertThat(CdnOriginCheck.isPublicV6(bytes(hex))).as(hex).isTrue();
|
||||
}
|
||||
assertThat(CdnOriginCheck.isPublic(java.net.Inet6Address.getByAddress(bytes("20014860486000000000000000008888")))).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An origin that drips one byte at a time (here, never finishing the TLS handshake)
|
||||
** is cut off at the hard deadline, not after hours.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void aDrippingOriginIsCutOffAtTheDeadline() throws Exception
|
||||
{
|
||||
try(java.net.ServerSocket server = new java.net.ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")))
|
||||
{
|
||||
Thread dripper = new Thread(() ->
|
||||
{
|
||||
try(java.net.Socket client = server.accept())
|
||||
{
|
||||
for(int i = 0; i < 60; i++)
|
||||
{
|
||||
client.getOutputStream().write(0x16);
|
||||
client.getOutputStream().flush();
|
||||
Thread.sleep(300);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
// the client gave up
|
||||
}
|
||||
});
|
||||
dripper.setDaemon(true);
|
||||
dripper.start();
|
||||
long start = System.nanoTime();
|
||||
assertThatThrownBy(() -> new CdnOriginCheck.HttpFetcher().fetch(InetAddress.getByName("127.0.0.1"), server.getLocalPort(), "localhost", "/", java.time.Duration.ofSeconds(2)))
|
||||
.isInstanceOf(IOException.class);
|
||||
long seconds = (System.nanoTime() - start) / 1_000_000_000L;
|
||||
assertThat(seconds).as("gave up at about the 2 second deadline").isLessThan(6);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A port is stripped from name:port and [v6]:port, never from a bare IPv6 literal.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void stripsPortsOnlyWhenThereIsOne()
|
||||
{
|
||||
assertThat(CdnOriginCheck.stripPort("origin.example.com:443")).isEqualTo("origin.example.com");
|
||||
assertThat(CdnOriginCheck.stripPort("origin.example.com")).isEqualTo("origin.example.com");
|
||||
assertThat(CdnOriginCheck.stripPort("[2606:4700::1111]:443")).isEqualTo("2606:4700::1111");
|
||||
assertThat(CdnOriginCheck.stripPort("2606:4700::1111")).isEqualTo("2606:4700::1111");
|
||||
assertThat(CdnOriginCheck.stripPort("2606:4700:4700::1")).isEqualTo("2606:4700:4700::1");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Hex to 16 bytes.
|
||||
*******************************************************************************/
|
||||
private static byte[] bytes(String hex)
|
||||
{
|
||||
assertThat(hex).hasSize(32);
|
||||
byte[] out = new byte[16];
|
||||
for(int i = 0; i < 16; i++)
|
||||
{
|
||||
out[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16);
|
||||
}
|
||||
return (out);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One bad address in a name's answer refuses the whole name, and the address
|
||||
** used is one that was checked (the first).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void pickRefusesAnyBadAddressAndReturnsACheckedOne() throws Exception
|
||||
{
|
||||
InetAddress good = InetAddress.getByName("93.184.216.34");
|
||||
assertThat(CdnOriginCheck.pick(new InetAddress[] {good, InetAddress.getByName("8.8.8.8")})).isSameAs(good);
|
||||
assertThatThrownBy(() -> CdnOriginCheck.pick(new InetAddress[] {good, InetAddress.getByName("169.254.169.254")})).isInstanceOf(IOException.class).hasMessageContaining("non-public");
|
||||
assertThatThrownBy(() -> CdnOriginCheck.pick(new InetAddress[0])).isInstanceOf(IOException.class);
|
||||
assertThatThrownBy(() -> new CdnOriginCheck.HttpFetcher().fetch("localhost", "/")).isInstanceOf(IOException.class).hasMessageContaining("non-public");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only the status line and headers are read, names are lower-cased, repeats are
|
||||
** joined, and the body is never touched.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void readsOnlyStatusAndHeaders() throws Exception
|
||||
{
|
||||
InputStream in = new ByteArrayInputStream("HTTP/1.1 200 OK\r\nCache-Control: public, max-age=60\r\nVary: a\r\nVary: b\r\n\r\nBODY".getBytes(StandardCharsets.US_ASCII));
|
||||
CdnOriginCheck.Response response = CdnOriginCheck.readResponse(in, System.nanoTime() + 1_000_000_000L);
|
||||
assertThat(response.status()).isEqualTo(200);
|
||||
assertThat(response.headers()).containsEntry("cache-control", "public, max-age=60").containsEntry("vary", "a, b");
|
||||
assertThat(in.read()).as("the body is left unread").isEqualTo('B');
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Malformed, oversized, truncated and too-slow responses are refused.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesBadResponses() throws Exception
|
||||
{
|
||||
long deadline = System.nanoTime() + 1_000_000_000L;
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("garbage\r\n\r\n"), deadline)).isInstanceOf(IOException.class);
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 abc OK\r\n\r\n"), deadline)).isInstanceOf(IOException.class);
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\nA: b\r\n"), deadline)).hasMessageContaining("closed early");
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\nA: " + "x".repeat(9000) + "\r\n\r\n"), deadline)).hasMessageContaining("too long");
|
||||
StringBuilder many = new StringBuilder("HTTP/1.1 200 OK\r\n");
|
||||
for(int i = 0; i < 400; i++)
|
||||
{
|
||||
many.append("H").append(i).append(": ").append("v".repeat(60)).append("\r\n");
|
||||
}
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream(many.append("\r\n").toString()), deadline)).hasMessageContaining("too large");
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\n\r\n"), System.nanoTime() - 1)).hasMessageContaining("too slow");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static InputStream stream(String text)
|
||||
{
|
||||
return (new ByteArrayInputStream(text.getBytes(StandardCharsets.US_ASCII)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.health;
|
||||
|
||||
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnOperations;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCategory;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckInput;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckKind;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData;
|
||||
import com.kingsrook.qqq.backend.core.health.HealthStatus;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QSession;
|
||||
import com.kingsrook.qqq.backend.core.model.session.QUser;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
import static org.mockito.Mockito.doReturn;
|
||||
import static org.mockito.Mockito.doThrow;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.spy;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Provider checks reuse the probe, honor deadlines and expose only safe facts.
|
||||
*******************************************************************************/
|
||||
class CdnProviderHealthChecksTest
|
||||
{
|
||||
private static final String SECRET = "S3NT1NEL-secret";
|
||||
private QInstance instance;
|
||||
private Integer providerId;
|
||||
private CdnTestSupport.FakePathProvider adapter;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A memory-backed instance and the existing fake adapter, with no network.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
instance = CdnTestSupport.setUp();
|
||||
providerId = CdnTestSupport.liveProvider(CdnProviderType.CLOUDFRONT, null);
|
||||
adapter = spy(new CdnTestSupport.FakePathProvider());
|
||||
CdnRuntime.registerProvider(CdnProviderType.CLOUDFRONT, () -> adapter);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Restore process-wide test state.
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
CdnRuntime.setEnvironment(null);
|
||||
CdnRuntime.setClock(null);
|
||||
Thread.interrupted();
|
||||
QContext.clear();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The producer registers a dynamic dependency supplier with the spec defaults.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void registrationAndRefresh() throws Exception
|
||||
{
|
||||
var registry = HealthChecksMetaData.ofOrWithNew(instance);
|
||||
assertThat(registry.getSuppliers()).singleElement().satisfies(supplier ->
|
||||
{
|
||||
assertThat(supplier.getNamePrefix()).isEqualTo("cdn.provider");
|
||||
assertThat(supplier.getComponent()).isEqualTo("cdn");
|
||||
});
|
||||
CdnTestSupport.insert(CdnTables.PROVIDER, new QRecord().withValue("name", "disabled").withValue("providerType", "CLOUDFRONT").withValue("enabled", false));
|
||||
// Match the runner's automated context rather than relying on CDN UI permissions.
|
||||
QContext.init(instance, new QSession().withUser(new QUser().withIdReference("health_runner").withFullName("health_runner"))
|
||||
.withSecurityKeyValues(Map.of()).withPermissions("health.read", "health.write"));
|
||||
assertThat(registry.resolveAll(instance)).singleElement().satisfies(check ->
|
||||
{
|
||||
assertThat(check.getName()).isEqualTo("cdn.provider." + providerId).matches("[a-z0-9]+([._-][a-z0-9]+)*");
|
||||
assertThat(check.getComponent()).isEqualTo("cdn");
|
||||
assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY);
|
||||
assertThat(check.getCategory()).isEqualTo(HealthCategory.CDN);
|
||||
assertThat(check.getIntervalSeconds()).isEqualTo(900);
|
||||
assertThat(check.getTimeoutMs()).isEqualTo(20000);
|
||||
assertThat(check.getIncludeInReadiness()).isFalse();
|
||||
});
|
||||
assertThat(check().getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UP);
|
||||
update(new QRecord().withValue("enabled", false));
|
||||
assertThat(registry.resolveAll(instance)).isEmpty();
|
||||
update(new QRecord().withValue("enabled", true));
|
||||
assertThat(registry.resolveAll(instance)).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Map success, missing origin protection and failure without writing lastHealth.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void connectionStates() throws Exception
|
||||
{
|
||||
var check = check().getCheckInstance();
|
||||
HealthCheckOutcome up = check.run(input());
|
||||
assertThat(up.getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(up.getDetails().getValues()).containsExactlyInAnyOrderEntriesOf(Map.of(
|
||||
"providerType", "CLOUDFRONT", "status", "Deployed", "originHeaderConfigured", true));
|
||||
doReturn(new CdnOperations.ConnectionTest(true, null, null, null, false, "InProgress", "unused")).when(adapter).connectionTest(any());
|
||||
HealthCheckOutcome degraded = check.run(input());
|
||||
assertThat(degraded.getStatus()).isEqualTo(HealthStatus.DEGRADED);
|
||||
assertThat(degraded.getDetails().getValues()).containsEntry("originHeaderConfigured", false).containsEntry("status", "InProgress");
|
||||
doReturn(new CdnOperations.ConnectionTest(false, null, null, null, false, null, "unused")).when(adapter).connectionTest(any());
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
QRecord row = new GetAction().executeForRecord(new GetInput(CdnTables.PROVIDER).withPrimaryKey(providerId));
|
||||
assertThat(row.getValue("lastHealthCheckDate")).isNull();
|
||||
assertThat(row.getValue("lastHealthStatus")).isNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A previously resolved check reloads config and skips disabled/deleted rows.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void reloadsProviderAndHandlesUnsupported() throws Exception
|
||||
{
|
||||
var check = check().getCheckInstance();
|
||||
update(new QRecord().withValue("providerType", "NONE"));
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
update(new QRecord().withValue("enabled", false));
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
new DeleteAction().execute(new DeleteInput(CdnTables.PROVIDER).withPrimaryKeys(List.of(providerId)));
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
verify(adapter, never()).connectionTest(any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poison credentials, every free-form probe field and thrown error messages.
|
||||
** HealthRunner.sanitize is package-private, so assert the raw boundary here.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void noSecretsInOutcomes() throws Exception
|
||||
{
|
||||
update(new QRecord().withValue("secretRef", "TEST_CDN").withValue("name", SECRET)
|
||||
.withValue("zoneOrDistributionId", SECRET).withValue("settingsJson", "{\"originVerifyHeader\":\"" + SECRET + "\"}"));
|
||||
CdnRuntime.setEnvironment(name -> SECRET);
|
||||
for(HealthStatus expected : List.of(HealthStatus.UP, HealthStatus.DEGRADED, HealthStatus.DOWN))
|
||||
{
|
||||
doAnswer(invocation ->
|
||||
{
|
||||
CdnProviderConfig config = invocation.getArgument(0);
|
||||
assertThat(CdnRuntime.environment(config.secretRef() + "_ACCESS_KEY_ID")).isEqualTo(SECRET);
|
||||
assertThat(CdnRuntime.environment(config.secretRef() + "_SECRET_ACCESS_KEY")).isEqualTo(SECRET);
|
||||
return (new CdnOperations.ConnectionTest(expected != HealthStatus.DOWN, SECRET, SECRET, 443,
|
||||
expected == HealthStatus.UP, SECRET, SECRET));
|
||||
}).when(adapter).connectionTest(any());
|
||||
HealthCheckOutcome outcome = check().getCheckInstance().run(input());
|
||||
assertThat(outcome.getStatus()).isEqualTo(expected);
|
||||
assertClean(outcome);
|
||||
}
|
||||
doThrow(new IllegalStateException(SECRET)).when(adapter).connectionTest(any());
|
||||
HealthCheckOutcome failed = check().getCheckInstance().run(input());
|
||||
assertThat(failed.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertClean(failed);
|
||||
update(new QRecord().withValue("providerType", "NONE"));
|
||||
assertClean(check().getCheckInstance().run(input()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An expired deadline never starts the existing probe.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void expiredDeadline() throws Exception
|
||||
{
|
||||
HealthCheckOutcome outcome = check().getCheckInstance().run(input().withDeadline(Instant.now().minusSeconds(1)));
|
||||
assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(outcome.getMessage()).contains("timed out");
|
||||
verify(adapter, never()).connectionTest(any());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A slow adapter is cancelled at the deadline, without a blocking executor close.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void deadlineCancelsProbe() throws Exception
|
||||
{
|
||||
CountDownLatch cancelled = new CountDownLatch(1);
|
||||
doAnswer(invocation ->
|
||||
{
|
||||
try
|
||||
{
|
||||
new CountDownLatch(1).await();
|
||||
return (null);
|
||||
}
|
||||
finally
|
||||
{
|
||||
cancelled.countDown();
|
||||
}
|
||||
}).when(adapter).connectionTest(any());
|
||||
HealthCheckOutcome outcome = assertTimeoutPreemptively(Duration.ofSeconds(3), () ->
|
||||
{
|
||||
QContext.init(instance, new QSession());
|
||||
try
|
||||
{
|
||||
return (check().getCheckInstance().run(input().withDeadline(Instant.now().plusMillis(200))));
|
||||
}
|
||||
finally
|
||||
{
|
||||
QContext.clear();
|
||||
}
|
||||
});
|
||||
assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(outcome.getMessage()).contains("timed out");
|
||||
assertThat(cancelled.await(2, TimeUnit.SECONDS)).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Runner cancellation preserves the caller's interrupt status.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void interruptedProbe() throws Exception
|
||||
{
|
||||
doAnswer(invocation ->
|
||||
{
|
||||
new CountDownLatch(1).await();
|
||||
return (null);
|
||||
}).when(adapter).connectionTest(any());
|
||||
var check = check().getCheckInstance();
|
||||
Thread.currentThread().interrupt();
|
||||
try
|
||||
{
|
||||
HealthCheckOutcome outcome = check.run(input());
|
||||
assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN);
|
||||
assertThat(outcome.getMessage()).contains("interrupted");
|
||||
assertThat(Thread.currentThread().isInterrupted()).isTrue();
|
||||
}
|
||||
finally
|
||||
{
|
||||
Thread.interrupted();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Resolve through the actual registration, including QCodeReference loading.
|
||||
*******************************************************************************/
|
||||
private HealthCheckMetaData check() throws Exception
|
||||
{
|
||||
return (HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance).getFirst());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fresh deadline for each call.
|
||||
*******************************************************************************/
|
||||
private HealthCheckInput input()
|
||||
{
|
||||
return (new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Modify the real memory row to exercise fresh reads.
|
||||
*******************************************************************************/
|
||||
private void update(QRecord values) throws Exception
|
||||
{
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.PROVIDER).withRecord(values.withValue("id", providerId)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No provider-controlled text can reach the central health boundary.
|
||||
*******************************************************************************/
|
||||
private static void assertClean(HealthCheckOutcome outcome)
|
||||
{
|
||||
assertThat(outcome.getMessage()).doesNotContain(SECRET, "\n", "\r");
|
||||
assertThat(outcome.getDetails().toJson()).doesNotContain(SECRET, "TEST_CDN", "origin.example.com", "d1.cloudfront.net");
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,16 @@ class CdnProcessesTest
|
||||
RunProcessOutput review = new RunProcessAction().execute(input);
|
||||
List<ProcessSummaryLineInterface> reviewLines = (List<ProcessSummaryLineInterface>) review.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
|
||||
for(ProcessSummaryLineInterface reviewLine : reviewLines)
|
||||
{
|
||||
ProcessSummaryLine summary = (ProcessSummaryLine) reviewLine;
|
||||
summary.pickMessage(false);
|
||||
if(summary.getMessage().contains("purges are recorded only"))
|
||||
{
|
||||
assertEquals(com.kingsrook.qqq.backend.core.model.actions.processes.Status.WARNING, summary.getStatus());
|
||||
}
|
||||
}
|
||||
|
||||
RunProcessInput go = new RunProcessInput();
|
||||
go.setProcessName(process);
|
||||
go.setProcessUUID(review.getProcessUUID());
|
||||
@@ -137,6 +147,7 @@ class CdnProcessesTest
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "SUCCEEDED"));
|
||||
List<List<String>> lines = run(CdnProcesses.RETRY_PURGE, rows.get(0).getValue("id") + "," + rows.get(1).getValue("id"));
|
||||
assertThat(lines.get(0)).containsExactly("1 purge request will be sent again", "1 purge request has not failed (no change)");
|
||||
assertThat(lines.get(1)).contains("1 purge request was queued again");
|
||||
QRecord retried = purges().get(0);
|
||||
assertEquals("PENDING", retried.getValueString("status"));
|
||||
assertEquals(0, retried.getValueInteger("attempts"));
|
||||
@@ -155,9 +166,99 @@ class CdnProcessesTest
|
||||
CdnTestSupport.insert(CdnTables.PROVIDER, new QRecord().withValue("name", "other").withValue("providerType", "CLOUDFRONT").withValue("enabled", true));
|
||||
List<List<String>> lines = run(CdnProcesses.PURGE_HOSTNAME, "1");
|
||||
assertThat(lines.get(0)).containsExactly("1 CDN distribution will be purged completely (every hostname on it)");
|
||||
assertThat(lines.get(1)).containsExactly("1 CDN distribution purge was recorded");
|
||||
List<QRecord> rows = purges();
|
||||
assertThat(rows).hasSize(1);
|
||||
assertEquals("ALL", rows.get(0).getValueString("kind"));
|
||||
assertEquals(1, rows.get(0).getValueInteger("cdnProviderId"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every action reports NONE and OFF honestly, including plural results.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void recordedOnlySummaries() throws Exception
|
||||
{
|
||||
for(boolean none : List.of(true, false))
|
||||
{
|
||||
for(String process : List.of(CdnProcesses.PURGE_PREFIX + "post", CdnProcesses.PURGE_HOSTNAME, CdnProcesses.RETRY_PURGE))
|
||||
{
|
||||
setUp();
|
||||
String table = none ? CdnTables.PROVIDER : CdnTables.HOSTNAME;
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(table)
|
||||
.withRecord(new QRecord().withValue("id", 1).withValue(none ? "providerType" : "stage", none ? "NONE" : "OFF")));
|
||||
String ids = "1";
|
||||
if(process.equals(CdnProcesses.PURGE_PREFIX + "post"))
|
||||
{
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "one"), new QRecord().withValue("slug", "two"));
|
||||
ids = "1,2";
|
||||
}
|
||||
else if(process.equals(CdnProcesses.RETRY_PURGE))
|
||||
{
|
||||
CdnTestSupport.insert(CdnTables.PURGE_REQUEST,
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "FAILED"),
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "DEAD"));
|
||||
ids = "1,2";
|
||||
}
|
||||
List<List<String>> lines = run(process, ids);
|
||||
String warning = none ? "No CDN provider configured" : "CDN hostnames are not live";
|
||||
assertThat(lines.get(0)).anyMatch(line -> line.contains(warning + ": purges are recorded only, not sent."));
|
||||
assertThat(lines.get(0)).noneMatch(line -> line.contains("will be purged") || line.contains("will be sent again"));
|
||||
String reason = none ? "no CDN provider" : "CDN hostnames are not live";
|
||||
assertThat(lines.get(1)).containsExactly(process.equals(CdnProcesses.PURGE_HOSTNAME)
|
||||
? "1 purge request was recorded (not sent: " + reason + ")"
|
||||
: "2 purge requests were recorded (not sent: " + reason + ")");
|
||||
assertThat(purges()).allMatch(row -> (none ? "SKIPPED_NONE_PROVIDER" : "SKIPPED_STAGE_OFF").equals(row.getValueString("status")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Eligibility considers enabled providers and their purge-enabled hosts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void sendingEligibility() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_NONE_PROVIDER,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
CdnTestSupport.liveProvider(CdnProviderType.NONE, null);
|
||||
Integer live = CdnTestSupport.liveProvider(CdnProviderType.CLOUDFRONT, null);
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.PENDING,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.HOSTNAME)
|
||||
.withRecord(new QRecord().withValue("id", 2).withValue("purgeEnabled", false)));
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_STAGE_OFF,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(live));
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.PROVIDER)
|
||||
.withRecord(new QRecord().withValue("id", live).withValue("enabled", false)));
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_NONE_PROVIDER,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without an enabled provider, do not claim a request row was recorded.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void absentProvider() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
CdnTestSupport.postRule();
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "one"));
|
||||
List<List<String>> lines = run(CdnProcesses.PURGE_PREFIX + "post", "1");
|
||||
assertThat(lines.get(0)).contains("1 No CDN provider configured: purges are recorded only, not sent.");
|
||||
assertThat(lines.get(1)).containsExactly("1 record had no enabled CDN provider (no purge request recorded)");
|
||||
assertThat(purges()).isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -199,9 +199,14 @@ class CdnProviderProcessesTest
|
||||
@Test
|
||||
void scheduledSteps() throws Exception
|
||||
{
|
||||
FakePathProvider adapter = org.mockito.Mockito.spy(new FakePathProvider());
|
||||
CdnRuntime.registerProvider(CdnProviderType.CLOUDFRONT, () -> adapter);
|
||||
FakePathProvider.drift = List.of(new CdnOperations.DriftItem("origin.header", "present", "missing"));
|
||||
new CdnProviderProcesses.ScheduledStatsStep().run(null, null);
|
||||
new CdnProviderProcesses.ScheduledDriftStep().run(null, null);
|
||||
assertThat(new QueryAction().execute(new QueryInput(CdnTables.EDGE_STAT_DAILY)).getRecords()).hasSize(2);
|
||||
assertEquals("OK", row().getValueString("lastHealthStatus"));
|
||||
assertThat(new QueryAction().execute(new QueryInput(CdnTables.DRIFT_FINDING)).getRecords()).hasSize(1);
|
||||
assertNull(row().getValueString("lastHealthStatus"));
|
||||
org.mockito.Mockito.verify(adapter, org.mockito.Mockito.never()).connectionTest(org.mockito.ArgumentMatchers.any());
|
||||
}
|
||||
}
|
||||
@@ -17,23 +17,25 @@
|
||||
package com.kingsrook.qbits.cdn.purge;
|
||||
|
||||
|
||||
import java.net.http.HttpClient;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport.FakePathProvider;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport.FakeTagProvider;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnException;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnPurgeKind;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -46,7 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
*******************************************************************************/
|
||||
class PurgePipelineTest
|
||||
{
|
||||
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of()));
|
||||
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(name -> null, host -> List.of()));
|
||||
|
||||
|
||||
|
||||
@@ -139,6 +141,32 @@ class PurgePipelineTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record-change listener skips counter updates but records title changes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ignoredCounterChangeRecordsNoPurge() throws Exception
|
||||
{
|
||||
QContext.getQInstance().getTable("post").withField(new QFieldMetaData("viewCount", QFieldType.INTEGER));
|
||||
CdnTestSupport.liveProvider(CdnProviderType.CLOUDFRONT, null);
|
||||
QRecord rule = new QueryAction().execute(new QueryInput(CdnTables.TABLE_TAG_RULE)).getRecords().get(0);
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.TABLE_TAG_RULE).withRecord(new QRecord().withValue("id", rule.getValue("id"))
|
||||
.withValue("onlyWhenFieldsChange", "!viewCount")));
|
||||
QRecord post = CdnTestSupport.insert("post", new QRecord().withValue("slug", "counter-post").withValue("title", "Old")
|
||||
.withValue("blogId", 1).withValue("isPublished", true).withValue("viewCount", 10)).get(0);
|
||||
assertThat(purges()).hasSize(1);
|
||||
|
||||
new UpdateAction().execute(new UpdateInput("post").withRecord(new QRecord().withValue("id", post.getValue("id")).withValue("viewCount", 11)));
|
||||
assertThat(purges()).as("counter-only update adds no purge request").hasSize(1);
|
||||
|
||||
new UpdateAction().execute(new UpdateInput("post").withRecord(new QRecord().withValue("id", post.getValue("id")).withValue("title", "New")));
|
||||
assertThat(purges()).hasSize(2);
|
||||
assertThat(purges().get(1).getValueString("paths")).contains("/blog/quick-bits/counter-post*");
|
||||
assertThat(purges().get(1).getValueString("tags")).contains("post:" + post.getValue("id"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Retryable failures back off and end DEAD; permanent ones go DEAD at once.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -18,15 +18,19 @@ package com.kingsrook.qbits.cdn.purge;
|
||||
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.http.HttpClient;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerificationException;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerifier;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -46,6 +50,7 @@ class RevalidationHookRunnerTest
|
||||
private final List<String> signatures = new CopyOnWriteArrayList<>();
|
||||
private final AtomicInteger status = new AtomicInteger(200);
|
||||
private HttpServer server;
|
||||
private final List<Map<String, String>> headers = new CopyOnWriteArrayList<>();
|
||||
|
||||
|
||||
|
||||
@@ -59,6 +64,9 @@ class RevalidationHookRunnerTest
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/api/revalidate", exchange ->
|
||||
{
|
||||
Map<String, String> received = new LinkedHashMap<>();
|
||||
exchange.getRequestHeaders().forEach((key, values) -> received.put(key.toLowerCase(java.util.Locale.ROOT), values.get(0)));
|
||||
headers.add(received);
|
||||
bodies.add(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
|
||||
String signature = exchange.getRequestHeaders().getFirst(RevalidationHookRunner.SIGNATURE_HEADER);
|
||||
signatures.add(signature == null ? "none" : signature);
|
||||
@@ -88,13 +96,21 @@ class RevalidationHookRunnerTest
|
||||
void signsAndFansOut() throws Exception
|
||||
{
|
||||
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true).withValue("fanOutDns", true)
|
||||
.withValue("secretRef", "SECRET").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> "SECRET".equals(name) ? "s3cret" : null,
|
||||
.withValue("secretRef", "SECRET").withValue("allowPlainHttp", true).withValue("internalAllowlist", "frontend.invalid").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "SECRET".equals(name) ? "test-secret-at-least-16-bytes" : null,
|
||||
host -> List.of("127.0.0.1", "127.0.0.1"));
|
||||
assertEquals(2, runner.run(List.of("post:1")));
|
||||
assertThat(bodies).hasSize(2).allMatch("{\"tags\":[\"post:1\"]}"::equals);
|
||||
String expected = "sha256=" + RevalidationHookRunner.sign("s3cret", bodies.get(0).getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(signatures).allMatch(expected::equals);
|
||||
WebhookVerifier verifier = new WebhookVerifier(List.of(WebhookSecret.parse("test-secret-at-least-16-bytes")));
|
||||
for(int i = 0; i < bodies.size(); i++)
|
||||
{
|
||||
verifier.verify(headers.get(i), bodies.get(i).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
assertThat(signatures).containsExactly("none", "none");
|
||||
assertThat(headers.get(0).get("webhook-id")).isEqualTo(headers.get(1).get("webhook-id"));
|
||||
assertThat(headers.get(0).get("host")).startsWith("frontend.invalid:");
|
||||
assertThatThrownBy(() -> verifier.verify(headers.get(0), "tampered".getBytes(StandardCharsets.UTF_8)))
|
||||
.isInstanceOf(WebhookVerificationException.class);
|
||||
assertEquals(0, runner.run(List.of()));
|
||||
}
|
||||
|
||||
@@ -108,12 +124,94 @@ class RevalidationHookRunnerTest
|
||||
{
|
||||
status.set(500);
|
||||
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true)
|
||||
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of());
|
||||
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1").withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.run(List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("500");
|
||||
assertThat(signatures).containsExactly("none");
|
||||
|
||||
QRecord fanOut = new QRecord().withValue("name", "empty").withValue("fanOutDns", true).withValue("url", "http://x.invalid/api/revalidate");
|
||||
assertThatThrownBy(() -> runner.call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
|
||||
assertThatThrownBy(() -> new RevalidationHookRunner(name -> null, host -> List.of()).call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Legacy mode preserves fixed signatures, including whitespace and whsec_ keys.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void legacyGoldenVectors() throws Exception
|
||||
{
|
||||
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
|
||||
assertThat(signatures.get(0)).isEqualTo("sha256=540457336a45ef962b165f8732f71ebf327ffa67c0cb1047c446df0231fa0db2");
|
||||
new RevalidationHookRunner(name -> " whsec_test-key-with-spaces ", host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
|
||||
assertThat(signatures.get(1)).isEqualTo("sha256=a4dd022bffe6f799633102c415fdc18e22f078bfc98357f909b5df58d5318b05");
|
||||
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
|
||||
.call(hook(), List.of("post:1"));
|
||||
assertThat(signatures.get(2)).as("a null format (existing rows) keeps the legacy signature").isEqualTo(signatures.get(0));
|
||||
assertThat(bodies).containsExactly("{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}");
|
||||
assertThat(headers).allSatisfy(value -> assertThat(value).doesNotContainKeys("webhook-id", "webhook-timestamp", "webhook-signature"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An explicit standard format accepts encoded secrets.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void explicitStandardFormat() throws Exception
|
||||
{
|
||||
WebhookSecret secret = WebhookSecret.generate();
|
||||
new RevalidationHookRunner(name -> secret.reveal(), host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"));
|
||||
new WebhookVerifier(List.of(secret)).verify(headers.get(0), bodies.get(0).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalid format and short standard secrets fail before sending.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void invalidSigningConfiguration()
|
||||
{
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "short", host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("at least 16 bytes");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "UNKNOWN"), List.of("post:1"))).isInstanceOf(QException.class);
|
||||
assertThat(bodies).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Safe defaults reject HTTP, private addresses and mixed fan-out before sending.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void safeHttpPolicy()
|
||||
{
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("allowPlainHttp", null), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Only https");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("internalAllowlist", null), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
|
||||
RevalidationHookRunner mixed = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1", "169.254.169.254"));
|
||||
assertThatThrownBy(() -> mixed.call(hook().withValue("fanOutDns", true), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("timeoutMs", 0), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("positive");
|
||||
assertThat(bodies).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A local test hook explicitly opts into HTTP and loopback.
|
||||
*******************************************************************************/
|
||||
private QRecord hook()
|
||||
{
|
||||
return (new QRecord().withValue("name", "test").withValue("secretRef", "SECRET")
|
||||
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1")
|
||||
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
}
|
||||
}
|
||||
@@ -135,6 +135,67 @@ class TagRuleEngineTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ignore-only rules compare actual values after merging partial updates.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ignoresCounterOnlyUpdates() throws Exception
|
||||
{
|
||||
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", " !viewCount, !lastIndexedAt ").withValue("tagTemplates", "post:{id}");
|
||||
QRecord old = new QRecord().withValue("id", 1).withValue("title", "Old").withValue("viewCount", 10).withValue("lastIndexedAt", "yesterday");
|
||||
TagRuleEngine engine = new TagRuleEngine();
|
||||
assertTrue(engine.evaluate(rule, List.of(new QRecord(old)), List.of(old)).isEmpty());
|
||||
assertTrue(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("viewCount", 11).withValue("lastIndexedAt", "today")), List.of(old)).isEmpty());
|
||||
assertTrue(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", "Old")), List.of(old)).isEmpty());
|
||||
assertThat(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", "New").withValue("viewCount", 11)), List.of(old)).tags()).containsExactly("post:1");
|
||||
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", null)), List.of(old)).isEmpty());
|
||||
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("newField", "value")), List.of(old)).isEmpty());
|
||||
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("ViewCount", 11)), List.of(old)).isEmpty());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain lists retain allow-list behavior; ignored entries always win.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void appliesAllowListAndMixedRules()
|
||||
{
|
||||
QRecord old = new QRecord().withValue("title", "Old").withValue("viewCount", 10);
|
||||
QRecord counterChange = new QRecord(old).withValue("viewCount", 11);
|
||||
QRecord titleChange = new QRecord(old).withValue("title", "New");
|
||||
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", " title, ,slug ");
|
||||
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
|
||||
assertTrue(TagRuleEngine.counts(rule, titleChange, old));
|
||||
assertFalse(TagRuleEngine.counts(rule, new QRecord(old), old));
|
||||
rule.setValue("onlyWhenFieldsChange", "title, viewCount, !viewCount");
|
||||
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
|
||||
assertTrue(TagRuleEngine.counts(rule, titleChange, old));
|
||||
assertFalse(TagRuleEngine.counts(rule, new QRecord(old).withValue("slug", "new"), old));
|
||||
rule.setValue("onlyWhenFieldsChange", "viewCount,!viewCount");
|
||||
assertFalse(TagRuleEngine.counts(rule, titleChange, old));
|
||||
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Changed-field filters never suppress inserts or deletes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void insertsAndDeletesIgnoreFieldFilters() throws Exception
|
||||
{
|
||||
QRecord post = new QRecord().withValue("id", 1).withValue("viewCount", 10);
|
||||
for(String fields : List.of("!viewCount", "title", "title,!viewCount"))
|
||||
{
|
||||
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", fields).withValue("tagTemplates", "post:{id}");
|
||||
assertThat(new TagRuleEngine().evaluate(rule, List.of(post), List.of()).tags()).containsExactly("post:1");
|
||||
assertThat(new TagRuleEngine().evaluate(rule, List.of(), List.of(post)).tags()).containsExactly("post:1");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stored text round-trips; invalid lines are dropped.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.tables;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.app.CdnSetupStatus;
|
||||
import com.kingsrook.qbits.cdn.checks.CdnOriginCheck;
|
||||
import com.kingsrook.qbits.cdn.headers.CdnResponseHeaderPolicy;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The "Create default ..." table actions (insert-only, change nothing that is
|
||||
** there) and the origin header check (the writer of cdn_origin_check, which had
|
||||
** none): what it counts, what it records, and the action and nightly entry points.
|
||||
*******************************************************************************/
|
||||
class CdnDefaultsAndOriginCheckTest
|
||||
{
|
||||
private final Map<String, CdnOriginCheck.Response> origin = new LinkedHashMap<>();
|
||||
private final List<String> asked = new java.util.ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
CdnOriginCheck.setFetcher((host, path) ->
|
||||
{
|
||||
asked.add(host + path);
|
||||
CdnOriginCheck.Response response = origin.get(path);
|
||||
if(response == null)
|
||||
{
|
||||
throw (new IOException("connection refused"));
|
||||
}
|
||||
return (response);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
CdnOriginCheck.setFetcher(null);
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run a table action through Review and then Result; returns the result lines.
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<String> run(String process, String recordIds) throws Exception
|
||||
{
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName(process);
|
||||
if(recordIds != null)
|
||||
{
|
||||
input.addValue("recordsParam", "recordIds");
|
||||
input.addValue("recordIds", recordIds);
|
||||
}
|
||||
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
|
||||
RunProcessOutput review = new RunProcessAction().execute(input);
|
||||
RunProcessInput go = new RunProcessInput();
|
||||
go.setProcessName(process);
|
||||
go.setProcessUUID(review.getProcessUUID());
|
||||
go.setStartAfterStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW);
|
||||
go.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
|
||||
RunProcessOutput result = new RunProcessAction().execute(go);
|
||||
List<ProcessSummaryLineInterface> lines = (List<ProcessSummaryLineInterface>) result.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
|
||||
return (lines == null ? List.of() : lines.stream().map(line ->
|
||||
{
|
||||
ProcessSummaryLine summary = (ProcessSummaryLine) line;
|
||||
summary.pickMessage(true);
|
||||
return (summary.getCount() + " " + summary.getMessage());
|
||||
}).toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> rows(String table) throws Exception
|
||||
{
|
||||
return (new QueryAction().execute(new QueryInput(table)).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create default provider adds No CDN once; and with it a record change is
|
||||
** recorded as a would-be purge (with no provider it is not recorded at all).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void defaultProviderMakesPurgesRecorded() throws Exception
|
||||
{
|
||||
CdnTestSupport.postRule();
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "a").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true));
|
||||
assertThat(rows(CdnTables.PURGE_REQUEST)).as("no provider: nothing is recorded").isEmpty();
|
||||
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_PROVIDER, null)).containsExactly("1 default row was added");
|
||||
assertThat(rows(CdnTables.PROVIDER)).singleElement().satisfies(p ->
|
||||
{
|
||||
assertThat(p.getValueString("name")).isEqualTo(CdnDefaultsProcesses.NONE_PROVIDER_NAME);
|
||||
assertThat(p.getValueString("providerType")).isEqualTo("NONE");
|
||||
assertThat(p.getValueBoolean("enabled")).isTrue();
|
||||
});
|
||||
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "b").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true));
|
||||
assertThat(rows(CdnTables.PURGE_REQUEST)).singleElement().satisfies(r -> assertThat(r.getValueString("status")).isEqualTo("SKIPPED_NONE_PROVIDER"));
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// insert-only: again changes nothing, and an edited row is left alone //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.PROVIDER).withRecord(new QRecord().withValue("id", rows(CdnTables.PROVIDER).get(0).getValue("id")).withValue("name", "Ours")));
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_PROVIDER, null)).containsExactly("null Nothing to add: the defaults are already there.");
|
||||
assertThat(rows(CdnTables.PROVIDER)).singleElement().satisfies(p -> assertThat(p.getValueString("name")).isEqualTo("Ours"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create default policies adds two turned-off policies (so nothing changes until
|
||||
** one is turned on), by name; an edited policy survives a second run; turning one
|
||||
** on makes the header decision change.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void defaultPoliciesAreOffUntilTurnedOn() throws Exception
|
||||
{
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_POLICIES, null)).containsExactly("2 default rows were added");
|
||||
List<QRecord> policies = rows(CdnTables.CACHE_POLICY);
|
||||
assertThat(policies).hasSize(2).allSatisfy(p -> assertThat(p.getValueBoolean("enabled")).isFalse());
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
assertThat(CdnResponseHeaderPolicy.current()).as("turned-off policies change no response").isEmpty();
|
||||
|
||||
QRecord assets = policies.stream().filter(p -> CdnDefaultsProcesses.ASSETS_POLICY_NAME.equals(p.getValueString("name"))).findFirst().orElseThrow();
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.CACHE_POLICY).withRecord(new QRecord().withValue("id", assets.getValue("id")).withValue("enabled", true).withValue("browserMaxAge", 600)));
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
CdnResponseHeaderPolicy.Decision decision = CdnResponseHeaderPolicy.decide(new CdnResponseHeaderPolicy.Request("www.example.com", "/assets/app.js", null, null, false, false, List.of(), List.of()), null);
|
||||
assertThat(decision.policyName()).isEqualTo(CdnDefaultsProcesses.ASSETS_POLICY_NAME);
|
||||
assertThat(decision.cacheControl()).startsWith("public, max-age=600").endsWith("immutable");
|
||||
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_POLICIES, null)).containsExactly("null Nothing to add: the defaults are already there.");
|
||||
assertThat(rows(CdnTables.CACHE_POLICY)).hasSize(2).anySatisfy(p -> assertThat(p.getValueInteger("browserMaxAge")).isEqualTo(600));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create default rules adds a tags-only rule for each purgeable table without
|
||||
** one, and the rule makes a change record tags.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void defaultRulesCoverPurgeableTables() throws Exception
|
||||
{
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_RULES, null)).containsExactly("1 default row was added");
|
||||
assertThat(rows(CdnTables.TABLE_TAG_RULE)).singleElement().satisfies(r ->
|
||||
{
|
||||
assertThat(r.getValueString("tableName")).isEqualTo("post");
|
||||
assertThat(r.getValueString("tagTemplates")).isEqualTo("post:{id}\npost");
|
||||
assertThat(r.getValueBoolean("enabled")).isTrue();
|
||||
});
|
||||
CdnTestSupport.insert(CdnTables.PROVIDER, new QRecord().withValue("name", "none").withValue("providerType", "NONE").withValue("enabled", true));
|
||||
QRecord post = CdnTestSupport.insert("post", new QRecord().withValue("slug", "a").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true)).get(0);
|
||||
assertThat(rows(CdnTables.PURGE_REQUEST)).singleElement().satisfies(r -> assertThat(r.getValueString("tags")).contains("post:" + post.getValue("id")));
|
||||
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_RULES, null)).containsExactly("null Nothing to add: the defaults are already there.");
|
||||
assertThat(rows(CdnTables.TABLE_TAG_RULE)).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The origin check: a good page, a page with no Cache-Control, a page whose
|
||||
** Cache-Control differs from its policy, a 404 and an unreachable page.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void originCheckCountsRightWrongAndUnreadable() throws Exception
|
||||
{
|
||||
Integer provider = CdnTestSupport.liveProvider(com.kingsrook.qbits.cdn.spi.CdnProviderType.CLOUDFRONT, null);
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.PROVIDER).withRecord(new QRecord().withValue("id", provider).withValue("originHost", "origin.example.com:443")));
|
||||
CdnTestSupport.insert(CdnTables.CACHE_POLICY,
|
||||
new QRecord().withValue("name", "blog").withValue("matchType", "PREFIX").withValue("matchValue", "/blog").withValue("policyClass", "LEAF").withValue("browserMaxAge", 60).withValue("enabled", true),
|
||||
new QRecord().withValue("name", "news").withValue("matchType", "PREFIX").withValue("matchValue", "/news").withValue("policyClass", "LEAF").withValue("browserMaxAge", 60).withValue("enabled", true));
|
||||
CdnTestSupport.insert(CdnTables.TABLE_TAG_RULE, new QRecord().withValue("tableName", "post").withValue("enabled", true)
|
||||
.withValue("invalidationPaths", "/about\n/blog/{slug}*\nhub:/blog"));
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
|
||||
origin.put("/", new CdnOriginCheck.Response(200, Map.of("cache-control", "max-age=5")));
|
||||
origin.put("/blog", new CdnOriginCheck.Response(200, Map.of("cache-control", "Public, max-age=60")));
|
||||
origin.put("/news", new CdnOriginCheck.Response(200, Map.of("cache-control", "public, max-age=999")));
|
||||
origin.put("/about", new CdnOriginCheck.Response(200, Map.of()));
|
||||
CdnTestSupport.insert(CdnTables.CACHE_POLICY, new QRecord().withValue("name", "gone").withValue("matchType", "PREFIX").withValue("matchValue", "/gone").withValue("policyClass", "LEAF").withValue("enabled", true),
|
||||
new QRecord().withValue("name", "down").withValue("matchType", "PREFIX").withValue("matchValue", "/down").withValue("policyClass", "LEAF").withValue("enabled", true));
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
origin.put("/gone", new CdnOriginCheck.Response(404, Map.of()));
|
||||
|
||||
QRecord hostname = rows(CdnTables.HOSTNAME).get(0);
|
||||
CdnOriginCheck.Result result = CdnOriginCheck.checkAndRecord(hostname);
|
||||
|
||||
assertThat(asked).as("the provider's origin host (without a port) is asked, not the public name").allMatch(url -> url.startsWith("origin.example.com/"));
|
||||
assertThat(result.checked()).isEqualTo(6);
|
||||
assertThat(result.correct()).as("/ (any Cache-Control, no policy) and /blog (equals its policy, ignoring case)").isEqualTo(2);
|
||||
assertThat(result.wrong()).as("/about has no header; /news differs from its policy").isEqualTo(2);
|
||||
assertThat(result.unreadable()).as("/gone answered 404; /down is unreachable").isEqualTo(2);
|
||||
assertThat(result.problems()).anyMatch(p -> p.startsWith("/about: no Cache-Control")).anyMatch(p -> p.startsWith("/news") && p.contains("public, max-age=60"))
|
||||
.anyMatch(p -> p.startsWith("/gone: the origin answered 404")).anyMatch(p -> p.startsWith("/down: could not be read"));
|
||||
|
||||
assertThat(rows(CdnTables.ORIGIN_CHECK)).singleElement().satisfies(row ->
|
||||
{
|
||||
assertThat(row.getValueString("scope")).isEqualTo("www.example.com");
|
||||
assertThat(row.getValueInteger("objectsChecked")).isEqualTo(6);
|
||||
assertThat(row.getValueInteger("correct")).isEqualTo(2);
|
||||
assertThat(row.getValueInteger("missingOrWrong")).isEqualTo(2);
|
||||
assertThat(row.getValueInteger("unreadable")).isEqualTo(2);
|
||||
assertThat(row.getValue("checkDate")).isNotNull();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sample paths are bounded and only literal paths are used.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void samplePathsAreLiteralAndBounded() throws Exception
|
||||
{
|
||||
StringBuilder paths = new StringBuilder("/blog/{slug}*");
|
||||
for(int i = 0; i < 40; i++)
|
||||
{
|
||||
paths.append("\n/page-").append(i);
|
||||
}
|
||||
CdnTestSupport.insert(CdnTables.TABLE_TAG_RULE, new QRecord().withValue("tableName", "post").withValue("enabled", true).withValue("invalidationPaths", paths.toString()));
|
||||
List<String> sample = CdnOriginCheck.samplePaths("www.example.com");
|
||||
assertThat(sample).hasSize(CdnOriginCheck.MAX_PATHS).startsWith("/", "/page-0").noneMatch(p -> p.contains("{") || p.contains("*"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record action writes a row per selected hostname; the nightly entry point
|
||||
** skips hostnames that are Off.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void actionAndNightlyRun() throws Exception
|
||||
{
|
||||
CdnTestSupport.liveProvider(com.kingsrook.qbits.cdn.spi.CdnProviderType.CLOUDFRONT, null);
|
||||
CdnTestSupport.insert(CdnTables.HOSTNAME, new QRecord().withValue("hostname", "off.example.com").withValue("cdnProviderId", 1).withValue("stage", "OFF"));
|
||||
origin.put("/", new CdnOriginCheck.Response(200, Map.of("cache-control", "max-age=5")));
|
||||
|
||||
List<String> lines = run(CdnOriginCheckProcesses.CHECK_ORIGIN, "1");
|
||||
assertThat(lines).containsExactly("1 hostname has the right headers");
|
||||
assertThat(rows(CdnTables.ORIGIN_CHECK)).hasSize(1);
|
||||
|
||||
new CdnOriginCheckProcesses.ScheduledStep().run(new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput(), new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput());
|
||||
assertThat(rows(CdnTables.ORIGIN_CHECK)).extracting(r -> r.getValueString("scope")).containsExactly("www.example.com", "www.example.com");
|
||||
|
||||
CdnRuntime.setClock(null);
|
||||
assertThat(CdnSetupStatus.load()).anySatisfy(item ->
|
||||
{
|
||||
assertThat(item.tableName()).isEqualTo(CdnTables.ORIGIN_CHECK);
|
||||
assertThat(item.state()).isEqualTo(CdnSetupStatus.State.OK);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.tables;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.app.CdnSetupStatus;
|
||||
import com.kingsrook.qbits.cdn.app.CdnSetupStatus.Item;
|
||||
import com.kingsrook.qbits.cdn.app.CdnSetupStatus.State;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Setup card of the CDN home: empty, on the No CDN default, and fully set up.
|
||||
*******************************************************************************/
|
||||
class CdnSetupStatusTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Item item(List<Item> items, String label)
|
||||
{
|
||||
return (items.stream().filter(i -> i.label().equals(label)).findFirst().orElseThrow());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Nothing set up: the next steps name the Create default actions, and the home shows them.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void emptySetupPointsAtTheDefaults() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
List<Item> items = CdnSetupStatus.load();
|
||||
assertThat(item(items, "Providers").state()).isEqualTo(State.TODO);
|
||||
assertThat(item(items, "Providers").nextStep()).contains("Create default provider");
|
||||
assertThat(item(items, "Tag rules").state()).isEqualTo(State.TODO);
|
||||
assertThat(item(items, "Tag rules").nextStep()).contains("Create default rules");
|
||||
assertThat(item(items, "Cache policies").nextStep()).contains("Create default policies");
|
||||
assertThat(items).extracting(Item::tableName).contains(CdnTables.PROVIDER, CdnTables.HOSTNAME, CdnTables.CACHE_POLICY, CdnTables.TABLE_TAG_RULE,
|
||||
CdnTables.REVALIDATION_HOOK, CdnTables.ORIGIN_CHECK);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A real CDN, a Live hostname, a policy that is on, a rule for each table and a hook.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void aCompleteSetupIsOk()
|
||||
{
|
||||
List<QRecord> providers = List.of(new QRecord().withValue("providerType", "CLOUDFRONT").withValue("enabled", true).withValue("lastHealthStatus", "OK"));
|
||||
List<QRecord> hostnames = List.of(new QRecord().withValue("stage", "LIVE").withValue("purgeEnabled", true));
|
||||
List<QRecord> policies = List.of(new QRecord().withValue("enabled", true));
|
||||
List<QRecord> rules = List.of(new QRecord().withValue("tableName", "post").withValue("enabled", true));
|
||||
List<QRecord> hooks = List.of(new QRecord().withValue("enabled", true));
|
||||
List<Item> items = CdnSetupStatus.build(providers, hostnames, policies, rules, hooks, 1, List.of("post"), true);
|
||||
assertThat(items).extracting(Item::state).containsOnly(State.OK);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The softer states: No CDN only, hostnames not Live, a failing provider, a table without a rule.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void softerStatesAreNamed()
|
||||
{
|
||||
QRecord none = new QRecord().withValue("providerType", "NONE").withValue("enabled", true);
|
||||
List<Item> items = CdnSetupStatus.build(List.of(none), List.of(new QRecord().withValue("stage", "OFF")), List.of(new QRecord().withValue("enabled", false)),
|
||||
List.of(), List.of(), 0, List.of("post", "page"), false);
|
||||
assertThat(item(items, "Providers").state()).isEqualTo(State.DEFAULT);
|
||||
assertThat(item(items, "Hostnames").state()).isEqualTo(State.DEFAULT);
|
||||
assertThat(item(items, "Cache policies").state()).isEqualTo(State.DEFAULT);
|
||||
assertThat(item(items, "Tag rules").detail()).contains("post, page");
|
||||
assertThat(item(items, "Origin checks").nextStep()).contains("No scheduler is configured");
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
// like the runtime: Live needs purge enabled, and a null enabled counts as off //
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
List<Item> unpurgeable = CdnSetupStatus.build(List.of(new QRecord().withValue("providerType", "CLOUDFRONT")), List.of(new QRecord().withValue("stage", "LIVE").withValue("purgeEnabled", false)),
|
||||
List.of(), List.of(), List.of(), 0, List.of(), true);
|
||||
assertThat(item(unpurgeable, "Providers").state()).isEqualTo(State.TODO);
|
||||
assertThat(item(unpurgeable, "Hostnames").state()).isEqualTo(State.DEFAULT);
|
||||
assertThat(item(unpurgeable, "Origin checks").nextStep()).contains("every night");
|
||||
|
||||
QRecord failing = new QRecord().withValue("providerType", "FASTLY").withValue("enabled", true).withValue("lastHealthStatus", "FAILED");
|
||||
assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), 0, List.of(), false), "Providers").state()).isEqualTo(State.TODO);
|
||||
assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), 0, List.of(), false), "Hostnames").state()).isEqualTo(State.TODO);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.tables;
|
||||
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnHelp;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.bundles.HelpBundles;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Guardrail (owner requirement 2026-10-14: "all tables should be populated and
|
||||
** the qbit should be fully using them"). For every cdn_* table:
|
||||
** - runtime code (not just its model, metadata or home screen) reads or writes it;
|
||||
** - it is listed here with why it starts empty and, where there is one, the table
|
||||
** action that sets it up (this qbit ships no DDL and a host chooses its column
|
||||
** names, so defaults are actions, not a database seed);
|
||||
** - it has bundle help that says what it is for and how its rows get created
|
||||
** (shown on the table screen and when it is empty).
|
||||
** A new table fails this test until it is wired and explained.
|
||||
*******************************************************************************/
|
||||
class CdnTablesGuardrailTest
|
||||
{
|
||||
private static final Path MAIN = Path.of("src", "main", "java", "com", "kingsrook", "qbits", "cdn");
|
||||
|
||||
/*******************************************************************************
|
||||
** Why each table starts empty, and where its rows come from.
|
||||
*******************************************************************************/
|
||||
static final Map<String, String> STARTS_EMPTY = Map.ofEntries(
|
||||
Map.entry(CdnTables.PROVIDER, "host connection config: the Create default provider action adds No CDN; a real CDN needs the host's account and key variable"),
|
||||
Map.entry(CdnTables.HOSTNAME, "host-specific: the hostnames the host serves"),
|
||||
Map.entry(CdnTables.CACHE_POLICY, "the Create default policies action adds two turned-off examples; turning one on is the host's decision"),
|
||||
Map.entry(CdnTables.TABLE_TAG_RULE, "the Create default rules action adds a tags-only rule per purgeable table; paths are the host's"),
|
||||
Map.entry(CdnTables.PURGE_REQUEST, "runtime: a record change or a purge action"),
|
||||
Map.entry(CdnTables.REVALIDATION_HOOK, "host-specific and optional: a front end with its own cache"),
|
||||
Map.entry(CdnTables.EDGE_STAT_DAILY, "runtime: Import Edge Stats and the daily import"),
|
||||
Map.entry(CdnTables.ORIGIN_CHECK, "runtime: Check Origin Headers and the nightly check"),
|
||||
Map.entry(CdnTables.DRIFT_FINDING, "runtime: Check Config Drift and the nightly check"));
|
||||
|
||||
/*******************************************************************************
|
||||
** The table action that sets a table up, by table.
|
||||
*******************************************************************************/
|
||||
static final Map<String, String> SETUP_ACTIONS = Map.of(
|
||||
CdnTables.PROVIDER, CdnDefaultsProcesses.CREATE_PROVIDER,
|
||||
CdnTables.CACHE_POLICY, CdnDefaultsProcesses.CREATE_POLICIES,
|
||||
CdnTables.TABLE_TAG_RULE, CdnDefaultsProcesses.CREATE_RULES);
|
||||
|
||||
private QInstance qInstance;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
qInstance = CdnTestSupport.setUp();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table is used by runtime code.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableIsUsedAtRuntime() throws Exception
|
||||
{
|
||||
List<String> runtimeSource = new ArrayList<>();
|
||||
try(Stream<Path> files = Files.walk(MAIN))
|
||||
{
|
||||
for(Path file : files.filter(p -> p.toString().endsWith(".java")).filter(p -> !p.startsWith(MAIN.resolve("model")) && !p.startsWith(MAIN.resolve("app"))).toList())
|
||||
{
|
||||
runtimeSource.add(Files.readString(file, StandardCharsets.UTF_8));
|
||||
}
|
||||
}
|
||||
for(String table : cdnTables())
|
||||
{
|
||||
String constant = constantFor(table);
|
||||
Pattern use = Pattern.compile("\\bCdnTables\\." + constant + "\\b");
|
||||
assertThat(runtimeSource.stream().anyMatch(source -> use.matcher(source).find()))
|
||||
.as(table + " (CdnTables." + constant + ") must be read or written by runtime code, not only declared").isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table is explained, and the setup actions it names exist.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableIsExplained()
|
||||
{
|
||||
assertThat(cdnTables()).containsExactlyInAnyOrderElementsOf(STARTS_EMPTY.keySet());
|
||||
assertThat(CdnTables.ALL).containsExactlyInAnyOrderElementsOf(STARTS_EMPTY.keySet());
|
||||
SETUP_ACTIONS.forEach((table, action) ->
|
||||
{
|
||||
assertThat(qInstance.getProcess(action)).as(action).isNotNull();
|
||||
assertThat(qInstance.getProcess(action).getTableName()).isEqualTo(table);
|
||||
assertThat(qInstance.getProcess(action).getMinInputRecords()).as(action + " needs no selected records").isEqualTo(0);
|
||||
});
|
||||
assertThat(qInstance.getProcess(CdnOriginCheckProcesses.CHECK_ORIGIN)).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table and the app have bundle help that says how rows get created, and
|
||||
** the help of a table with a setup action names it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableHasEmptyStateHelp() throws Exception
|
||||
{
|
||||
var entries = HelpBundles.of(qInstance).getBundles().get(0).getEntries();
|
||||
for(String table : cdnTables())
|
||||
{
|
||||
var entry = entries.stream().filter(e -> e.getKey().equals("table:" + table + ";slot:help")).findFirst().orElseThrow();
|
||||
assertThat(entry.getContent()).as(table + " help says how rows get created").contains("How rows get created");
|
||||
if(SETUP_ACTIONS.containsKey(table))
|
||||
{
|
||||
assertThat(entry.getContent()).as(table + " help names its setup action").contains(qInstance.getProcess(SETUP_ACTIONS.get(table)).getLabel());
|
||||
}
|
||||
}
|
||||
assertThat(entries).anyMatch(e -> e.getKey().equals(CdnHelp.APP_KEY));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<String> cdnTables()
|
||||
{
|
||||
List<String> tables = qInstance.getTables().values().stream().map(QTableMetaData::getName).filter(n -> n.startsWith("cdn_")).sorted().toList();
|
||||
assertThat(tables).hasSize(9);
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The CdnTables constant whose value is the table name.
|
||||
*******************************************************************************/
|
||||
private static String constantFor(String table) throws Exception
|
||||
{
|
||||
for(Field field : CdnTables.class.getFields())
|
||||
{
|
||||
if(field.getType() == String.class && table.equals(field.get(null)) && !field.getName().startsWith("PVS_"))
|
||||
{
|
||||
return (field.getName());
|
||||
}
|
||||
}
|
||||
throw (new IllegalStateException("no constant for " + table));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user