Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bf0952f5ed
|
||
|
|
d03c791ca3 | ||
|
|
5a0d3e824a
|
||
|
|
422b34988d
|
||
|
|
3980e05719
|
||
|
|
8b9a14df59
|
||
|
|
b4d08dc065 | ||
|
|
49c0db82f3
|
||
|
|
675b9fb32c | ||
|
|
440f69fc5c
|
||
|
|
37fb879f51 | ||
|
|
ecceaf31ee
|
||
|
|
4546a17d6b | ||
|
|
0092574b6e
|
||
|
|
fb4a536494 | ||
|
|
967c52f8bc
|
||
|
|
253f862ecd
|
||
|
|
5801ff2ebd | ||
|
|
7510392166
|
||
|
|
59fc98ccb2 | ||
|
|
1e2af2acf1
|
||
|
|
2c41409fb2
|
||
|
|
b1d7d9543f | ||
|
|
15439d2fba |
No files matched your search
+57
-11
@@ -1,14 +1,60 @@
|
||||
version: 2.1
|
||||
setup: true
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.9
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release\/.*)/
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only: [develop]
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only: [/release\/.*/]
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [/^v\d+\.\d+\.\d+$/]
|
||||
@@ -1,25 +0,0 @@
|
||||
# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
|
||||
name: qrun-release
|
||||
on:
|
||||
push: {}
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
jobs:
|
||||
release:
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
docker-runner: container-build
|
||||
browsers: true
|
||||
browser-runner: browser-build
|
||||
postpublish:
|
||||
if: ${{ needs.release.outputs.capture == 'true' }}
|
||||
needs: release
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
@@ -1,23 +0,0 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -0,0 +1,25 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release-no-hotfix
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
hotfix: true
|
||||
asciidoc: false
|
||||
@@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository.
|
||||
## Dependencies
|
||||
|
||||
- Java 21.
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.4` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.5` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
|
||||
- Tests: see pom.xml.
|
||||
|
||||
## Release
|
||||
|
||||
@@ -4,7 +4,20 @@ All notable changes to this project are documented here. The format follows Keep
|
||||
|
||||
## [0.1.1] - Unreleased
|
||||
|
||||
### Added
|
||||
- Origin header check: Check Origin Headers on Hostnames (and a nightly run over hostnames that are not Off) asks the origin for sample pages and writes `cdn_origin_check` rows. The table had no writer before.
|
||||
- Guided empty states: help bundle `help/qbit-cdn.yaml` (app and all nine tables, with "How rows get created") applied by `CdnHelp`, and a Setup card on the CDN home.
|
||||
- "Create default provider" (No CDN), "Create default policies" (two turned-off examples) and "Create default rules" (a tags-only rule per purgeable table) table actions; all insert-only.
|
||||
- `hostnameRole` is documented as a label that changes nothing; cache policies are documented as applying only where the application calls `CdnResponseHeaderPolicy`.
|
||||
|
||||
### Changed
|
||||
- Require released QQQ BOM 4.1.0-RC.5; revalidation hooks default to core Standard Webhooks signing. Per-hook `signatureFormat=LEGACY` preserves the prior header and signature bytes.
|
||||
- Use core `SafeHttpClient` for revalidation requests: HTTPS/public targets by default, explicit `allowPlainHttp`/`internalAllowlist` exceptions, bounded responses and checked DNS pinning. Fan-out preserves Host/TLS identity. Host schema migration and receiver migration are described in README and `docs/revalidation-hook-migration.md`.
|
||||
|
||||
|
||||
### Fixed
|
||||
- Tag rules accept ignored fields in `onlyWhenFieldsChange`, such as `!viewCount`, so counter-only updates do not purge rendered pages. Plain allow-lists still work, and ignored fields take precedence in mixed lists. No schema change is needed.
|
||||
- Purge actions warn when providers are NONE or hostnames are OFF, and report requests as recorded but not sent. The home explains recorded-only purges and that this version does not run origin header checks.
|
||||
- Tag rule `{prefix.field}` placeholders now resolve the joined table from the `<prefix>Id` field's possible-value source, so prefixed tables (`cms_blog`) work; a missing joined table is logged instead of silently skipped.
|
||||
|
||||
## [0.1.0] - Unreleased
|
||||
|
||||
@@ -31,7 +31,25 @@ Design: QRun-IO/qqq `docs/specs/2026-10-12-qbit-cdn-design.md`.
|
||||
NONE. See `docs/cloudfront-expected-config.md`.
|
||||
- **Permissions:** `cdn.view`, `cdn.purge`, `cdn.purgeAll`, `cdn.admin`.
|
||||
|
||||
## Purge-path placeholders
|
||||
## Setup
|
||||
|
||||
This qbit ships no DDL and seeds nothing from a changelog (a host chooses its own column names). Open the CDN app: the Setup card lists each part with the next step. On a fresh install run **Create default provider** (No CDN: purges are recorded, nothing is sent), **Create default rules** (tags for every purgeable table), and optionally **Create default policies** (two turned-off examples). Add your CDN and hostnames when you have them. Cache policies change a response only where the application calls `CdnResponseHeaderPolicy.decide`. **Check Origin Headers** on a hostname (and the nightly run) asks the origin, the provider's `originHost` when set, for the home page, prefix policy paths and literal tag-rule paths, and records how many have the right Cache-Control.
|
||||
|
||||
## Tag rules
|
||||
|
||||
`cdn_table_tag_rule.onlyWhenFieldsChange` is a comma-separated list of QQQ field
|
||||
names. Plain entries (for example `title,slug`) purge only when at least one
|
||||
listed field changes. Prefix a field with `!` to ignore it: `!viewCount` prevents
|
||||
a view-count-only update from purging, while a title change still purges.
|
||||
With `!viewCount, !lastIndexedAt`, any change outside those fields counts.
|
||||
Mixed lists keep the allow-list and exclude ignored fields, even if also listed
|
||||
plainly: `title,viewCount,!viewCount` counts only title changes.
|
||||
|
||||
Names are trimmed and case-sensitive. Partial updates retain omitted old values;
|
||||
only unequal old and new values count. This filter does not restrict inserts or
|
||||
deletes. The separate `publishedField` filter still applies to all changes.
|
||||
|
||||
### Purge-path placeholders
|
||||
|
||||
`cdn_table_tag_rule` templates (`tagTemplates`, `invalidationPaths`) can use `{field}` for the changed record's own fields and `{prefix.field}` for a joined record, e.g. `/blog/{blog.slug}/{slug}*`.
|
||||
|
||||
@@ -58,6 +76,36 @@ CloudFront credentials: `<secretRef>_ACCESS_KEY_ID` / `<secretRef>_SECRET_ACCESS
|
||||
Drift: the OpenTofu `expected_config` JSON in `CDN_EXPECTED_CONFIG` (or the
|
||||
variable named by the provider setting `expectedConfigEnv`).
|
||||
|
||||
## Revalidation hooks
|
||||
|
||||
Requires QQQ **4.1.0-RC.5** (the default `qqq-snapshot` profile imports the released BOM;
|
||||
its historical property name does not select a core SNAPSHOT).
|
||||
|
||||
New `cdn_revalidation_hook` rows default `signatureFormat` to `STANDARD_WEBHOOKS`;
|
||||
existing rows with a null format keep the `LEGACY` signature. Signed requests use core `WebhookSigner` and
|
||||
`WebhookSecret`: `webhook-id`, `webhook-timestamp`, and `webhook-signature`.
|
||||
Rows on `LEGACY` (or null) keep sending
|
||||
`X-Revalidate-Signature: sha256=<lowercase hex HMAC-SHA256 of the body>`.
|
||||
Legacy keys remain the exact UTF-8 environment value, with no trimming or decoding.
|
||||
Standard keys must contain at least 16 bytes after core parsing (trimmed raw text,
|
||||
or base64-decoded `whsec_` text). `secretRef` names the environment variable;
|
||||
as before, no resolved secret means no signature headers.
|
||||
|
||||
Core `SafeHttpClient` now enforces HTTPS and rejects internal addresses by default.
|
||||
For an internal HTTP pod endpoint explicitly set `allowPlainHttp=true` and
|
||||
`internalAllowlist` to its exact hostname, a `.suffix`, IP, or CIDR (comma/whitespace
|
||||
separated). Metadata and unspecified addresses remain blocked. Every DNS address
|
||||
is checked before fan-out; the original hostname is retained for Host/TLS, and
|
||||
connections use the checked addresses without resolving again. Redirects fail,
|
||||
responses are bounded to 64 KiB, and `timeoutMs` is a positive per-request total
|
||||
limit (default 5000). These transport rules apply to both signature formats.
|
||||
|
||||
Host apps must migrate `cdn_revalidation_hook` to add `signatureFormat` (string),
|
||||
`allowPlainHttp` (boolean), and `internalAllowlist` (text). Backfill `LEGACY` for
|
||||
old receivers; configure internal network exceptions explicitly before dispatch.
|
||||
See [receiver migration](docs/revalidation-hook-migration.md) for the inspected
|
||||
KoF22 receiver and exact changes required.
|
||||
|
||||
## Build
|
||||
|
||||
```sh
|
||||
@@ -67,3 +115,18 @@ variable named by the provider setting `expectedConfigEnv`).
|
||||
## License
|
||||
|
||||
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
||||
|
||||
### Before a CDN is live
|
||||
|
||||
With no enabled CDN provider (including providers of type `NONE`), purges are
|
||||
recorded only, not sent. A real provider also needs a `LIVE` or `LOCKED` hostname
|
||||
with purging enabled. Purge From CDN, Purge Everything, and Retry Purge show a
|
||||
Review warning and a recorded-only Result when sending is unavailable. Requests
|
||||
for enabled providers remain visible with `SKIPPED_NONE_PROVIDER` or
|
||||
`SKIPPED_STAGE_OFF`; with no enabled provider, no new request row is created.
|
||||
The home Connection and Health cards explain the unconfigured state, and purge
|
||||
activity labels recorded-only requests when disconnected.
|
||||
|
||||
This version does not run origin header checks. The Origin headers card says
|
||||
nothing has been checked until the host application writes results to the
|
||||
standard `cdn_origin_check` table; those results continue to appear normally.
|
||||
@@ -0,0 +1,7 @@
|
||||
# Upgrading
|
||||
|
||||
## Unreleased
|
||||
|
||||
- No schema change. Hosts get the help text, the Setup card and the new actions automatically.
|
||||
- The nightly origin check runs on hosts with a scheduler (02:45 UTC) over hostnames that are not Off. It makes HTTPS requests to the provider's origin host (or the hostname) for a few pages; allow that egress or leave hostnames Off.
|
||||
- Run Create default provider on hosts that have no provider if you want purges recorded as would-be purges.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Revalidation hook migration (F11 / qbit-cdn #15)
|
||||
|
||||
Reference: core source read directly at tag `v4.1.0-RC.5`, including
|
||||
`WebhookSigner`, `WebhookSecret`, `WebhookVerifier`, `SafeHttpClient`,
|
||||
`SafeHttpPolicy`, and `AddressRules`. No temporary core bridge is needed.
|
||||
|
||||
## Receiver found locally
|
||||
|
||||
The plain `Website-Frontend` and `Website-Publisher` checkouts have no match for
|
||||
`X-Revalidate-Signature`. The implementation is in
|
||||
`R:/Git.Local/KofTwentyTwo/Website-Frontend-wt-isr`, branch `feature/cdn-isr`,
|
||||
HEAD `476318b` at inspection (working files inspected):
|
||||
|
||||
- `src/lib/revalidate.ts:5`: names `x-revalidate-signature`.
|
||||
- `src/lib/revalidate.ts:9`: `signBody` returns `sha256=` plus hex HMAC-SHA256
|
||||
over the body using the raw string key (the calculation is line 10).
|
||||
- `src/lib/revalidate.ts:14`: `verifySignature` compares this value in constant
|
||||
time; line 21 lowercases the incoming legacy header.
|
||||
- `src/app/api/revalidate/route.ts:30`: reads `CDN_REVALIDATE_SECRET`.
|
||||
- `src/app/api/revalidate/route.ts:36`: reads the body before JSON parsing.
|
||||
- `src/app/api/revalidate/route.ts:40`: reads only the legacy signature header
|
||||
and calls the verifier; standard requests currently return 401.
|
||||
- `src/__tests__/revalidate-route.test.ts:12`: test requests send only the legacy header.
|
||||
|
||||
These receiver files were inspected, not modified.
|
||||
|
||||
## Deployment and receiver changes
|
||||
|
||||
1. Add the three hook columns documented in README. Existing rows (null
|
||||
`signatureFormat`) keep the LEGACY signature; switch a hook to `STANDARD_WEBHOOKS`
|
||||
after its receiver verifies Standard Webhooks. New rows default to `STANDARD_WEBHOOKS`.
|
||||
For internal HTTP endpoints also explicitly allow HTTP and the intended host.
|
||||
2. Update `src/lib/revalidate.ts:14` to verify Standard Webhooks: require all
|
||||
three headers, validate an integer Unix timestamp within 300 seconds of the
|
||||
receiver clock (reject malformed/non-finite values), then HMAC-SHA256 the
|
||||
bytes `<webhook-id>.<webhook-timestamp>.<raw body>` and compare decoded
|
||||
signatures in constant time. Accept any matching `v1,<base64>` entry in the
|
||||
space-separated signature list. Do not lowercase base64 signatures.
|
||||
Match core secret parsing: trim text; decode `whsec_` base64 keys, otherwise
|
||||
UTF-8; use at least 16 key bytes. Rotate short existing keys on both sides.
|
||||
3. At `src/app/api/revalidate/route.ts:36`, retain raw body bytes for verification;
|
||||
at line 40 pass all three headers to the new verifier before parsing JSON.
|
||||
Update its header import/documentation (lines 5 and 22). During migration the
|
||||
receiver may explicitly support both formats; do not fall back to legacy
|
||||
when standard headers are present but invalid.
|
||||
4. Update `src/__tests__/revalidate-route.test.ts:12` and helper tests to cover
|
||||
valid standard requests, altered body/id/timestamp, missing headers, stale
|
||||
or future timestamps, malformed timestamps, encoded keys, and legacy mode.
|
||||
5. Deploy the receiver, then switch its hook to `STANDARD_WEBHOOKS`.
|
||||
|
||||
The JSON payload remains `{"tags":[...]}`. One UUID identifies a hook invocation,
|
||||
shared across its fan-out requests; each request gets its own current timestamp.
|
||||
A later dispatcher invocation generates a new UUID. Revalidation remains
|
||||
idempotent, so receivers must not rely on this ID to deduplicate purge retries;
|
||||
any optional deduplication must remain local to each pod, not suppress other pods.
|
||||
|
||||
## Compatibility and verification
|
||||
|
||||
The legacy HMAC helper remains solely for the explicitly supported legacy format,
|
||||
not as a temporary replacement for core. Golden tests use fixed digests calculated
|
||||
independently of that helper, including a short key and a whitespace/prefix key.
|
||||
Standard requests are checked using RC.5 `WebhookVerifier`; transport tests cover
|
||||
HTTP/private-address refusal, explicit local exceptions and mixed-address fan-out.
|
||||
|
||||
The test/collaborator constructor now takes `(secrets, resolver)`; callers no longer
|
||||
inject `java.net.http.HttpClient`. The production no-argument constructor remains.
|
||||
The only existing test adaptation outside the runner tests is `PurgePipelineTest`
|
||||
using this constructor. The RC.5 bump required no unrelated code fixes.
|
||||
@@ -18,17 +18,19 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<revision>0.1.0</revision>
|
||||
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
|
||||
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
|
||||
<revision>0.1.0-RC.1</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
<aws.sdk.version>2.41.10</aws.sdk.version>
|
||||
<aws.sdk.version>2.41.34</aws.sdk.version>
|
||||
</properties>
|
||||
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>2.1.0</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
@@ -192,7 +194,7 @@
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
|
||||
<qqq.snapshot.version>4.1.0-RC.6</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -149,6 +149,8 @@ public class CdnQBitProducer implements QBitMetaDataProducer<CdnQBitConfig>
|
||||
}
|
||||
List<QProcessMetaData> processes = new ArrayList<>(CdnProcesses.processes(qBitConfig));
|
||||
processes.addAll(com.kingsrook.qbits.cdn.processes.CdnProviderProcesses.processes(qBitConfig));
|
||||
processes.addAll(com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses.processes(qBitConfig));
|
||||
processes.addAll(com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses.processes());
|
||||
for(QProcessMetaData process : processes)
|
||||
{
|
||||
process.setSourceQBitName(qBit.getName());
|
||||
@@ -160,6 +162,7 @@ public class CdnQBitProducer implements QBitMetaDataProducer<CdnQBitConfig>
|
||||
}
|
||||
output.add(CdnWidgets.app(qBitConfig, tables));
|
||||
output.addSelfToInstance(qInstance);
|
||||
com.kingsrook.qbits.cdn.model.CdnHelp.apply(qInstance, qBitConfig.getAppName());
|
||||
|
||||
if(!qBitConfig.getPurgeableTables().isEmpty())
|
||||
{
|
||||
|
||||
@@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
*******************************************************************************/
|
||||
public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostnames, List<StatView> stats, List<QRecord> recentPurges,
|
||||
int failed24h, int dead24h, int failed7d, int dead7d, List<BudgetView> budgets, List<QRecord> originChecks,
|
||||
Map<String, Integer> openDriftByArea, Instant lastDriftCheck, Instant now)
|
||||
Map<String, Integer> openDriftByArea, Instant lastDriftCheck, Instant now, List<CdnSetupStatus.Item> setup)
|
||||
{
|
||||
public static final int ROTATION_WARNING_DAYS = 75;
|
||||
public static final int ROTATION_DAYS = 90;
|
||||
@@ -61,14 +61,14 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
** A provider row as shown.
|
||||
*******************************************************************************/
|
||||
public record ProviderView(Integer id, String name, String type, String distributionId, String dnsTarget, String originHost,
|
||||
String lastHealthStatus, String lastHealthMessage, Instant lastSuccessfulTest, Long credentialAgeDays)
|
||||
String lastHealthStatus, String lastHealthMessage, Instant lastSuccessfulTest, Long credentialAgeDays, Boolean enabled)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Whether this is a real CDN (not the NONE placeholder).
|
||||
*******************************************************************************/
|
||||
public boolean isConnected()
|
||||
{
|
||||
return (!CdnProviderType.NONE.name().equals(type));
|
||||
return (Boolean.TRUE.equals(enabled) && !CdnProviderType.NONE.name().equals(type));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
LocalDate rotated = ValueUtils.getValueAsLocalDate(row.getValue("credentialRotatedDate"));
|
||||
providers.add(new ProviderView(id, row.getValueString("name"), row.getValueString("providerType"), row.getValueString("zoneOrDistributionId"),
|
||||
row.getValueString("dnsTarget"), row.getValueString("originHost"), row.getValueString("lastHealthStatus"), row.getValueString("lastHealthMessage"),
|
||||
ValueUtils.getValueAsInstant(row.getValue("lastSuccessfulTestDate")), rotated == null ? null : ChronoUnit.DAYS.between(rotated, today)));
|
||||
ValueUtils.getValueAsInstant(row.getValue("lastSuccessfulTestDate")), rotated == null ? null : ChronoUnit.DAYS.between(rotated, today), row.getValueBoolean("enabled")));
|
||||
if(CdnProviderType.CLOUDFRONT.name().equals(row.getValueString("providerType")))
|
||||
{
|
||||
PathPlannerSettings settings = PathPlannerSettings.from(CdnPurgeDispatcher.configOf(row));
|
||||
@@ -237,7 +237,7 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
}
|
||||
|
||||
return (new CdnHomeData(providers, hostnames, stats, recent, failed24, dead24, failed7, dead7, budgets, new ArrayList<>(latestOrigin.values()),
|
||||
drift, lastDrift, now));
|
||||
drift, lastDrift, now, CdnSetupStatus.load()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -61,6 +61,7 @@ public final class CdnHomeView
|
||||
{
|
||||
StringBuilder html = new StringBuilder("<div class=\"cdn-home\" style=\"" + GRID + "\">");
|
||||
connection(html, data, appPath);
|
||||
setup(html, data, appPath);
|
||||
performance(html, data);
|
||||
purges(html, data, appPath);
|
||||
budget(html, data);
|
||||
@@ -80,7 +81,7 @@ public final class CdnHomeView
|
||||
String text = "All CDN checks are OK.";
|
||||
if("NOT_CONNECTED".equals(state))
|
||||
{
|
||||
text = "No CDN connected yet.";
|
||||
text = "No CDN provider configured: purges are recorded only.";
|
||||
}
|
||||
else if("ATTENTION".equals(state))
|
||||
{
|
||||
@@ -99,7 +100,7 @@ public final class CdnHomeView
|
||||
open(html, "Connection");
|
||||
if(!data.isConnected())
|
||||
{
|
||||
empty(html, "No CDN connected yet. The CloudFront distribution is created in the cutover window.");
|
||||
empty(html, "No CDN provider configured: purges are recorded only. The CloudFront distribution is created in the cutover window.");
|
||||
}
|
||||
for(CdnHomeData.ProviderView provider : data.providers())
|
||||
{
|
||||
@@ -182,6 +183,7 @@ public final class CdnHomeView
|
||||
Instant created = ValueUtils.getValueAsInstant(row.getValue("createDate"));
|
||||
html.append("<li><a href=\"").append(esc(appPath + "/" + CdnTables.PURGE_REQUEST + "/" + row.getValue("id"))).append("\">#")
|
||||
.append(esc(String.valueOf(row.getValue("id")))).append("</a> ").append(esc(row.getValueString("status")))
|
||||
.append(data.isConnected() ? "" : " - recorded only (not sent)")
|
||||
.append(created == null ? "" : ", " + WHEN.format(created)).append("</li>");
|
||||
}
|
||||
html.append("</ul>");
|
||||
@@ -216,6 +218,50 @@ public final class CdnHomeView
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Setup card: each configuration table, whether it is set up, and the next
|
||||
** step, linked to its table.
|
||||
*******************************************************************************/
|
||||
private static void setup(StringBuilder html, CdnHomeData data, String appPath)
|
||||
{
|
||||
if(data.setup() == null || data.setup().isEmpty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
open(html, "Setup");
|
||||
long todo = data.setup().stream().filter(i -> i.state() == CdnSetupStatus.State.TODO).count();
|
||||
html.append("<p>").append(todo == 0 ? "Everything the CDN needs is set up." : esc(todo + (todo == 1 ? " item needs" : " items need") + " attention.")).append("</p><ul style=\"list-style:none;margin:0;padding:0;\">");
|
||||
for(CdnSetupStatus.Item item : data.setup())
|
||||
{
|
||||
String color = switch(item.state())
|
||||
{
|
||||
case OK -> "#16a34a";
|
||||
case TODO -> "#dc2626";
|
||||
case DEFAULT -> "#d97706";
|
||||
case OPTIONAL -> "#9ca3af";
|
||||
};
|
||||
String label = switch(item.state())
|
||||
{
|
||||
case OK -> "Set up";
|
||||
case TODO -> "Needs setup";
|
||||
case DEFAULT -> "Using defaults";
|
||||
case OPTIONAL -> "Optional";
|
||||
};
|
||||
html.append("<li style=\"margin:0 0 .75rem;\"><strong><a href=\"").append(esc(appPath + "/" + item.tableName())).append("\">").append(esc(item.label())).append("</a></strong> ")
|
||||
.append("<span style=\"white-space:nowrap;\"><span aria-hidden=\"true\" style=\"width:.55rem;height:.55rem;border-radius:50%;background:").append(color)
|
||||
.append(";display:inline-block;\"></span> ").append(esc(label)).append("</span><div>").append(esc(item.detail())).append("</div>");
|
||||
if(item.nextStep() != null)
|
||||
{
|
||||
html.append("<div style=\"").append(MUTED).append("\">Next: ").append(esc(item.nextStep())).append("</div>");
|
||||
}
|
||||
html.append("</li>");
|
||||
}
|
||||
html.append("</ul>");
|
||||
close(html);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Origin header card.
|
||||
*******************************************************************************/
|
||||
@@ -224,7 +270,7 @@ public final class CdnHomeView
|
||||
open(html, "Origin headers");
|
||||
if(data.originChecks().isEmpty())
|
||||
{
|
||||
empty(html, "No origin header check has run yet.");
|
||||
empty(html, "No origin check yet. Run Check Origin Headers on Hostnames; hostnames that are not Off are also checked every night.");
|
||||
}
|
||||
for(QRecord check : data.originChecks())
|
||||
{
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.app;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.cdn.CdnQBitConfig;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.model.CdnHostnameStage;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Setup card on the CDN home: each configuration table, whether it is set up,
|
||||
** and the next step. Reads the small configuration tables the same way the runtime
|
||||
** does (enabled rows only where the runtime filters on enabled, the application's
|
||||
** purgeable tables from the qbit config).
|
||||
*******************************************************************************/
|
||||
public final class CdnSetupStatus
|
||||
{
|
||||
/*******************************************************************************
|
||||
** OK: in place. TODO: needs doing. DEFAULT: working on defaults. OPTIONAL: off
|
||||
** until set up.
|
||||
*******************************************************************************/
|
||||
public enum State
|
||||
{
|
||||
OK, TODO, DEFAULT, OPTIONAL
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One line of the card.
|
||||
*******************************************************************************/
|
||||
public record Item(String label, String tableName, State state, String detail, String nextStep)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnSetupStatus()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Build the items from the tables.
|
||||
*******************************************************************************/
|
||||
public static List<Item> load() throws QException
|
||||
{
|
||||
CdnQBitConfig config = CdnRuntime.getConfig();
|
||||
List<QRecord> providers = all(CdnTables.PROVIDER);
|
||||
List<QRecord> hostnames = all(CdnTables.HOSTNAME);
|
||||
List<QRecord> policies = all(CdnTables.CACHE_POLICY);
|
||||
List<QRecord> rules = all(CdnTables.TABLE_TAG_RULE);
|
||||
List<QRecord> hooks = all(CdnTables.REVALIDATION_HOOK);
|
||||
long checks = new CountAction().execute(new CountInput(CdnTables.ORIGIN_CHECK)).getCount();
|
||||
boolean scheduled = config != null && config.getSchedulerName() != null;
|
||||
return (build(providers, hostnames, policies, rules, hooks, checks, config == null ? List.of() : config.getPurgeableTables(), scheduled));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The items for given rows (no queries; for tests and reuse).
|
||||
*******************************************************************************/
|
||||
public static List<Item> build(List<QRecord> providers, List<QRecord> hostnames, List<QRecord> policies, List<QRecord> rules, List<QRecord> hooks,
|
||||
long checks, List<String> purgeableTables, boolean scheduled)
|
||||
{
|
||||
List<Item> items = new ArrayList<>();
|
||||
|
||||
//////////////
|
||||
// provider //
|
||||
//////////////
|
||||
List<QRecord> enabledProviders = providers.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("enabled"))).toList();
|
||||
List<QRecord> real = enabledProviders.stream().filter(p -> !CdnProviderType.NONE.name().equals(p.getValueString("providerType"))).toList();
|
||||
if(providers.isEmpty())
|
||||
{
|
||||
items.add(new Item("Providers", CdnTables.PROVIDER, State.TODO, "No provider: changes are not even recorded as purges.",
|
||||
"Run Create default provider (No CDN: records purges, sends nothing), or add your CDN with New."));
|
||||
}
|
||||
else if(enabledProviders.isEmpty())
|
||||
{
|
||||
items.add(new Item("Providers", CdnTables.PROVIDER, State.TODO, "Every provider is turned off.", "Turn one on (Enabled)."));
|
||||
}
|
||||
else if(real.isEmpty())
|
||||
{
|
||||
items.add(new Item("Providers", CdnTables.PROVIDER, State.DEFAULT, "Only No CDN: purges are recorded and nothing is sent.",
|
||||
"Add your CDN with New (type, distribution id, and the environment variable that holds its key), then Test Connection."));
|
||||
}
|
||||
else
|
||||
{
|
||||
boolean failing = real.stream().anyMatch(p -> "FAILED".equals(p.getValueString("lastHealthStatus")));
|
||||
items.add(failing
|
||||
? new Item("Providers", CdnTables.PROVIDER, State.TODO, "A provider's last connection test failed.", "Check its key variable and run Test Connection.")
|
||||
: new Item("Providers", CdnTables.PROVIDER, State.OK, real.size() + " CDN provider" + (real.size() == 1 ? "" : "s") + " turned on.", null));
|
||||
}
|
||||
|
||||
///////////////
|
||||
// hostnames //
|
||||
///////////////
|
||||
if(hostnames.isEmpty())
|
||||
{
|
||||
items.add(real.isEmpty()
|
||||
? new Item("Hostnames", CdnTables.HOSTNAME, State.OPTIONAL, "None: nothing sits behind a CDN yet.", "Add each site hostname once a CDN provider exists.")
|
||||
: new Item("Hostnames", CdnTables.HOSTNAME, State.TODO, "A CDN is set up but no hostname uses it, so nothing is purged.", "Add the hostnames behind it, at stage Off, and move them to Headers only, then Live."));
|
||||
}
|
||||
else
|
||||
{
|
||||
long live = hostnames.stream().filter(h -> (CdnHostnameStage.LIVE.name().equals(h.getValueString("stage")) || CdnHostnameStage.LOCKED.name().equals(h.getValueString("stage")))
|
||||
&& Boolean.TRUE.equals(h.getValueBoolean("purgeEnabled"))).count();
|
||||
items.add(live == 0
|
||||
? new Item("Hostnames", CdnTables.HOSTNAME, State.DEFAULT, hostnames.size() + " hostname" + (hostnames.size() == 1 ? "" : "s") + ", none Live: purges are recorded, not sent.",
|
||||
"When the CDN serves a hostname, move it to Live (and leave Purge enabled on).")
|
||||
: new Item("Hostnames", CdnTables.HOSTNAME, State.OK, live + " of " + hostnames.size() + " hostnames Live.", null));
|
||||
}
|
||||
|
||||
////////////////////////
|
||||
// cache policies //
|
||||
////////////////////////
|
||||
long enabledPolicies = policies.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("enabled"))).count();
|
||||
items.add(policies.isEmpty()
|
||||
? new Item("Cache policies", CdnTables.CACHE_POLICY, State.OPTIONAL, "None: responses keep the headers your application sets.",
|
||||
"Run Create default policies for examples. Policies only change responses when the application calls CdnResponseHeaderPolicy.")
|
||||
: enabledPolicies == 0
|
||||
? new Item("Cache policies", CdnTables.CACHE_POLICY, State.DEFAULT, policies.size() + " polic" + (policies.size() == 1 ? "y" : "ies") + ", all turned off.",
|
||||
"Edit one and turn it on. It takes effect only where the application calls CdnResponseHeaderPolicy.")
|
||||
: new Item("Cache policies", CdnTables.CACHE_POLICY, State.OK, enabledPolicies + " polic" + (enabledPolicies == 1 ? "y" : "ies") + " turned on (applied where the application calls CdnResponseHeaderPolicy).", null));
|
||||
|
||||
////////////////////////////////////////
|
||||
// tag rules: every purgeable table //
|
||||
////////////////////////////////////////
|
||||
Set<String> covered = new HashSet<>();
|
||||
rules.stream().filter(r -> Boolean.TRUE.equals(r.getValueBoolean("enabled"))).forEach(r -> covered.add(r.getValueString("tableName")));
|
||||
List<String> uncovered = purgeableTables.stream().filter(t -> !covered.contains(t)).toList();
|
||||
if(purgeableTables.isEmpty())
|
||||
{
|
||||
items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.OPTIONAL, "The application lists no purgeable tables, so record changes purge nothing.", "List them in the qbit config (purgeableTables)."));
|
||||
}
|
||||
else if(uncovered.isEmpty())
|
||||
{
|
||||
items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.OK, "Every purgeable table has a rule (" + String.join(", ", purgeableTables) + ").", null));
|
||||
}
|
||||
else
|
||||
{
|
||||
items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.TODO, "No rule for " + String.join(", ", uncovered) + ": changes to them purge nothing.",
|
||||
"Run Create default rules (tags only), then add invalidation paths where your CDN purges by path."));
|
||||
}
|
||||
|
||||
///////////////////////
|
||||
// revalidation hooks //
|
||||
///////////////////////
|
||||
long enabledHooks = hooks.stream().filter(h -> Boolean.TRUE.equals(h.getValueBoolean("enabled"))).count();
|
||||
items.add(enabledHooks == 0
|
||||
? new Item("Revalidation hooks", CdnTables.REVALIDATION_HOOK, State.OPTIONAL, "None.", "Add one if a front end keeps its own cache (for example Next.js) that must refresh before the CDN is purged.")
|
||||
: new Item("Revalidation hooks", CdnTables.REVALIDATION_HOOK, State.OK, enabledHooks + " hook" + (enabledHooks == 1 ? "" : "s") + " turned on.", null));
|
||||
|
||||
////////////////////
|
||||
// origin checks //
|
||||
////////////////////
|
||||
items.add(checks == 0
|
||||
? new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OPTIONAL, "Never run.",
|
||||
"Run Check Origin Headers on Hostnames." + (scheduled ? " Hostnames that are not Off are also checked every night." : " No scheduler is configured, so nothing runs it automatically."))
|
||||
: new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OK, checks + " check" + (checks == 1 ? "" : "s") + " recorded.", null));
|
||||
return (items);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> all(String table) throws QException
|
||||
{
|
||||
return (new QueryAction().execute(new QueryInput(table)).getRecords());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,575 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.checks;
|
||||
|
||||
|
||||
import javax.net.ssl.SNIHostName;
|
||||
import javax.net.ssl.SSLParameters;
|
||||
import javax.net.ssl.SSLSocket;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import java.io.BufferedInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Socket;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.ScheduledFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.headers.CdnResponseHeaderPolicy;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The origin header check (design spec section 3.2): requests a few sample paths
|
||||
** of a hostname from its origin and records whether each answers with the
|
||||
** Cache-Control the cache policies say it should. One cdn_origin_check row is
|
||||
** written per hostname and run: objects checked, correct, missing or wrong, and
|
||||
** unreadable (the request failed, or the origin answered 4xx or 5xx).
|
||||
**
|
||||
** The sample paths are "/", the path of every enabled PREFIX policy that applies
|
||||
** to the hostname, and the literal (placeholder-free) invalidation paths of the
|
||||
** enabled tag rules, up to MAX_PATHS. The origin is the provider's originHost when
|
||||
** it has one, otherwise the hostname itself. Requests are HTTPS GETs, never follow
|
||||
** redirects, and refuse hosts that resolve to private, loopback or link-local
|
||||
** addresses.
|
||||
**
|
||||
** A response is correct when it has a Cache-Control header and, when an enabled
|
||||
** policy matches the path, the header equals what the policy would send (compared
|
||||
** ignoring case and spaces). With no enabled policy for the path, any Cache-Control
|
||||
** header is correct: the check proves the origin sets one at all.
|
||||
*******************************************************************************/
|
||||
public final class CdnOriginCheck
|
||||
{
|
||||
public static final int MAX_PATHS = 20;
|
||||
|
||||
private static volatile Fetcher fetcher = new HttpFetcher();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One origin answer: status and lower-cased header names.
|
||||
*******************************************************************************/
|
||||
public record Response(int status, Map<String, String> headers)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** How the origin is asked (replaceable in tests).
|
||||
*******************************************************************************/
|
||||
public interface Fetcher
|
||||
{
|
||||
/*******************************************************************************
|
||||
** GET https://host/path; throws when the origin cannot be read at all.
|
||||
*******************************************************************************/
|
||||
Response fetch(String host, String path) throws IOException;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The outcome of checking one hostname.
|
||||
*******************************************************************************/
|
||||
public record Result(String scope, int checked, int correct, int wrong, int unreadable, List<String> problems)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnOriginCheck()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Replace how origins are asked; null restores the real HTTP fetcher.
|
||||
*******************************************************************************/
|
||||
public static void setFetcher(Fetcher newFetcher)
|
||||
{
|
||||
fetcher = newFetcher == null ? new HttpFetcher() : newFetcher;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The sample paths of a hostname.
|
||||
*******************************************************************************/
|
||||
public static List<String> samplePaths(String hostname) throws QException
|
||||
{
|
||||
Set<String> paths = new LinkedHashSet<>();
|
||||
paths.add("/");
|
||||
for(CdnResponseHeaderPolicy.Policy policy : CdnResponseHeaderPolicy.current())
|
||||
{
|
||||
if("PREFIX".equals(policy.matchType()) && policy.matchValue().startsWith("/") && (policy.hostname() == null || policy.hostname().equalsIgnoreCase(hostname)))
|
||||
{
|
||||
paths.add(policy.matchValue());
|
||||
}
|
||||
}
|
||||
for(QRecord rule : new QueryAction().execute(new QueryInput(CdnTables.TABLE_TAG_RULE)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)))).getRecords())
|
||||
{
|
||||
String text = rule.getValueString("invalidationPaths");
|
||||
for(String line : text == null ? new String[0] : text.split("\\R"))
|
||||
{
|
||||
String path = line.strip();
|
||||
if(path.startsWith("/") && !path.contains("{") && !path.contains("*"))
|
||||
{
|
||||
paths.add(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
return (new ArrayList<>(paths).subList(0, Math.min(MAX_PATHS, paths.size())));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check one hostname row (and record nothing; see record()).
|
||||
*******************************************************************************/
|
||||
public static Result check(QRecord hostname) throws QException
|
||||
{
|
||||
String name = hostname.getValueString("hostname");
|
||||
String origin = originOf(hostname, name);
|
||||
int correct = 0;
|
||||
int wrong = 0;
|
||||
int unread = 0;
|
||||
List<String> problems = new ArrayList<>();
|
||||
List<String> paths = samplePaths(name);
|
||||
for(String path : paths)
|
||||
{
|
||||
Response response;
|
||||
try
|
||||
{
|
||||
response = fetcher.fetch(origin, path);
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
unread++;
|
||||
problems.add(path + ": could not be read (" + e.getMessage() + ")");
|
||||
continue;
|
||||
}
|
||||
if(response.status() >= 400)
|
||||
{
|
||||
unread++;
|
||||
problems.add(path + ": the origin answered " + response.status());
|
||||
continue;
|
||||
}
|
||||
String actual = response.headers().get("cache-control");
|
||||
if(!StringUtils.hasContent(actual))
|
||||
{
|
||||
wrong++;
|
||||
problems.add(path + ": no Cache-Control header");
|
||||
continue;
|
||||
}
|
||||
CdnResponseHeaderPolicy.Decision decision = CdnResponseHeaderPolicy.decide(
|
||||
new CdnResponseHeaderPolicy.Request(name, path, null, null, false, false, List.of(), List.of()), null);
|
||||
if(decision.policyName() != null && !normalize(decision.cacheControl()).equals(normalize(actual)))
|
||||
{
|
||||
wrong++;
|
||||
problems.add(path + ": Cache-Control is \"" + actual + "\" but the policy " + decision.policyName() + " says \"" + decision.cacheControl() + "\"");
|
||||
}
|
||||
else
|
||||
{
|
||||
correct++;
|
||||
}
|
||||
}
|
||||
return (new Result(name, paths.size(), correct, wrong, unread, problems));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check a hostname and write its cdn_origin_check row; returns the result.
|
||||
*******************************************************************************/
|
||||
public static Result checkAndRecord(QRecord hostname) throws QException
|
||||
{
|
||||
Result result = check(hostname);
|
||||
new InsertAction().execute(new InsertInput(CdnTables.ORIGIN_CHECK).withRecord(new QRecord()
|
||||
.withValue("checkDate", CdnRuntime.clock().instant())
|
||||
.withValue("scope", result.scope())
|
||||
.withValue("objectsChecked", result.checked())
|
||||
.withValue("correct", result.correct())
|
||||
.withValue("missingOrWrong", result.wrong())
|
||||
.withValue("unreadable", result.unreadable())));
|
||||
return (result);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The host to ask: the provider's origin host, else the hostname.
|
||||
*******************************************************************************/
|
||||
private static String originOf(QRecord hostname, String fallback) throws QException
|
||||
{
|
||||
java.io.Serializable providerId = hostname.getValue("cdnProviderId");
|
||||
if(providerId != null)
|
||||
{
|
||||
for(QRecord provider : new QueryAction().execute(new QueryInput(CdnTables.PROVIDER)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, providerId)))).getRecords())
|
||||
{
|
||||
String originHost = provider.getValueString("originHost");
|
||||
if(StringUtils.hasContent(originHost))
|
||||
{
|
||||
return (stripPort(originHost.strip()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return (fallback);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The host without a port: "[v6]:443" gives v6, "name:443" gives name, and a bare
|
||||
** IPv6 literal or a plain name is left alone.
|
||||
*******************************************************************************/
|
||||
static String stripPort(String hostAndPort)
|
||||
{
|
||||
if(hostAndPort.startsWith("[") && hostAndPort.contains("]"))
|
||||
{
|
||||
return (hostAndPort.substring(1, hostAndPort.indexOf(']')));
|
||||
}
|
||||
int colon = hostAndPort.indexOf(':');
|
||||
if(colon > 0 && colon == hostAndPort.lastIndexOf(':') && hostAndPort.substring(colon + 1).matches("\\d+"))
|
||||
{
|
||||
return (hostAndPort.substring(0, colon));
|
||||
}
|
||||
return (hostAndPort);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Lower case, no spaces.
|
||||
*******************************************************************************/
|
||||
private static String normalize(String cacheControl)
|
||||
{
|
||||
return (cacheControl == null ? "" : cacheControl.toLowerCase(Locale.ROOT).replace(" ", ""));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether an address is one an origin check may connect to: public unicast only.
|
||||
** IPv4 refuses loopback, private, link-local, carrier-grade NAT (100.64.0.0/10),
|
||||
** 0.0.0.0/8, benchmarking (198.18.0.0/15), documentation, multicast and reserved.
|
||||
** IPv6 is an allowlist: only global unicast 2000::/3 passes, minus Teredo
|
||||
** (2001::/32) and documentation (2001:db8::/32); 6to4 (2002::/16) passes only
|
||||
** when its embedded IPv4 is public. The forms that embed an IPv4 address (::/96,
|
||||
** v4-mapped, IPv4-translated, NAT64 64:ff9b::/96) are unwrapped and judged as
|
||||
** IPv4; the local-use NAT64 prefix 64:ff9b:1::/48 is refused.
|
||||
*******************************************************************************/
|
||||
public static boolean isPublic(InetAddress address)
|
||||
{
|
||||
byte[] b = address.getAddress();
|
||||
if(b.length == 4)
|
||||
{
|
||||
return (isPublicV4(b[0] & 0xff, b[1] & 0xff, b[2] & 0xff));
|
||||
}
|
||||
return (isPublicV6(b));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
static boolean isPublicV6(byte[] b)
|
||||
{
|
||||
if(zero(b, 0, 12) || (zero(b, 0, 10) && ff(b, 10)) || (zero(b, 0, 8) && ff(b, 8) && b[10] == 0 && b[11] == 0))
|
||||
{
|
||||
return (isPublicV4(b[12] & 0xff, b[13] & 0xff, b[14] & 0xff));
|
||||
}
|
||||
if((b[0] & 0xff) == 0x00 && (b[1] & 0xff) == 0x64 && (b[2] & 0xff) == 0xff && (b[3] & 0xff) == 0x9b)
|
||||
{
|
||||
return (zero(b, 4, 12) && isPublicV4(b[12] & 0xff, b[13] & 0xff, b[14] & 0xff));
|
||||
}
|
||||
if((b[0] & 0xe0) != 0x20)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if((b[0] & 0xff) == 0x20 && (b[1] & 0xff) == 0x01 && ((b[2] == 0 && b[3] == 0) || ((b[2] & 0xff) == 0x0d && (b[3] & 0xff) == 0xb8)))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if((b[0] & 0xff) == 0x20 && (b[1] & 0xff) == 0x02)
|
||||
{
|
||||
return (isPublicV4(b[2] & 0xff, b[3] & 0xff, b[4] & 0xff));
|
||||
}
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether bytes from (inclusive) to (exclusive) are all zero.
|
||||
*******************************************************************************/
|
||||
private static boolean zero(byte[] b, int from, int to)
|
||||
{
|
||||
for(int i = from; i < to; i++)
|
||||
{
|
||||
if(b[i] != 0)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
return (true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether two bytes at an index are ff ff.
|
||||
*******************************************************************************/
|
||||
private static boolean ff(byte[] b, int at)
|
||||
{
|
||||
return ((b[at] & 0xff) == 0xff && (b[at + 1] & 0xff) == 0xff);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean isPublicV4(int a, int b, int c)
|
||||
{
|
||||
if(a == 0 || a == 10 || a == 127 || a >= 224)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 100 && b >= 64 && b <= 127)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 169 && b == 254)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 172 && b >= 16 && b <= 31)
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 192 && (b == 168 || (b == 0 && (c == 0 || c == 2)) || (b == 88 && c == 99)))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
if(a == 198 && (b == 18 || b == 19 || (b == 51 && c == 100)))
|
||||
{
|
||||
return (false);
|
||||
}
|
||||
return (!(a == 203 && b == 0 && c == 113));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The address to connect to: every address the name resolves to must be public,
|
||||
** and the first one is used, so the connection goes to what was checked.
|
||||
*******************************************************************************/
|
||||
static InetAddress pick(InetAddress[] resolved) throws IOException
|
||||
{
|
||||
if(resolved == null || resolved.length == 0)
|
||||
{
|
||||
throw (new IOException("the host did not resolve"));
|
||||
}
|
||||
for(InetAddress address : resolved)
|
||||
{
|
||||
if(!isPublic(address))
|
||||
{
|
||||
throw (new IOException("the host resolves to a non-public address"));
|
||||
}
|
||||
}
|
||||
return (resolved[0]);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read a status line and headers (and nothing else) from an HTTP/1.1 response,
|
||||
** within a total deadline and a size limit.
|
||||
*******************************************************************************/
|
||||
static Response readResponse(InputStream in, long deadlineNanos) throws IOException
|
||||
{
|
||||
String status = readLine(in, deadlineNanos);
|
||||
int total = status.length();
|
||||
String[] parts = status.split(" ", 3);
|
||||
if(parts.length < 2 || !parts[0].startsWith("HTTP/"))
|
||||
{
|
||||
throw (new IOException("not an HTTP response"));
|
||||
}
|
||||
int code;
|
||||
try
|
||||
{
|
||||
code = Integer.parseInt(parts[1]);
|
||||
}
|
||||
catch(NumberFormatException e)
|
||||
{
|
||||
throw (new IOException("bad status line"));
|
||||
}
|
||||
Map<String, String> headers = new LinkedHashMap<>();
|
||||
while(true)
|
||||
{
|
||||
String line = readLine(in, deadlineNanos);
|
||||
total += line.length();
|
||||
if(line.isEmpty())
|
||||
{
|
||||
return (new Response(code, headers));
|
||||
}
|
||||
if(total > 16_384)
|
||||
{
|
||||
throw (new IOException("the headers are too large"));
|
||||
}
|
||||
int colon = line.indexOf(':');
|
||||
if(colon > 0)
|
||||
{
|
||||
headers.merge(line.substring(0, colon).strip().toLowerCase(Locale.ROOT), line.substring(colon + 1).strip(), (x, y) -> x + ", " + y);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static String readLine(InputStream in, long deadlineNanos) throws IOException
|
||||
{
|
||||
StringBuilder line = new StringBuilder();
|
||||
while(true)
|
||||
{
|
||||
if(System.nanoTime() > deadlineNanos)
|
||||
{
|
||||
throw (new IOException("the origin was too slow"));
|
||||
}
|
||||
int c = in.read();
|
||||
if(c < 0)
|
||||
{
|
||||
throw (new IOException("the connection closed early"));
|
||||
}
|
||||
if(c == '\n')
|
||||
{
|
||||
return (line.toString().stripTrailing());
|
||||
}
|
||||
line.append((char) c);
|
||||
if(line.length() > 8192)
|
||||
{
|
||||
throw (new IOException("a header line is too long"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The real fetcher: resolve once, refuse non-public addresses, connect to the
|
||||
** address that was checked (TLS with the name as SNI and for verification), send
|
||||
** one GET, read the status line and headers only, and close. No redirects, no body.
|
||||
*******************************************************************************/
|
||||
static final class HttpFetcher implements Fetcher
|
||||
{
|
||||
private static final ScheduledExecutorService WATCHDOG = Executors.newSingleThreadScheduledExecutor(runnable ->
|
||||
{
|
||||
Thread thread = new Thread(runnable, "cdn-origin-check-watchdog");
|
||||
thread.setDaemon(true);
|
||||
return (thread);
|
||||
});
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Response fetch(String host, String path) throws IOException
|
||||
{
|
||||
InetAddress address = pick(InetAddress.getAllByName(host));
|
||||
return (fetch(address, 443, host, path, Duration.ofSeconds(10)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fetch from one checked address within one hard deadline that covers connect,
|
||||
** the TLS handshake and the reads: at the deadline the socket is closed, which
|
||||
** fails whatever read or write is blocked.
|
||||
*******************************************************************************/
|
||||
Response fetch(InetAddress address, int port, String host, String path, Duration limit) throws IOException
|
||||
{
|
||||
long deadline = System.nanoTime() + limit.toNanos();
|
||||
try(Socket plain = new Socket())
|
||||
{
|
||||
ScheduledFuture<?> guard = WATCHDOG.schedule(() ->
|
||||
{
|
||||
try
|
||||
{
|
||||
plain.close();
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
// already closed
|
||||
}
|
||||
}, limit.toMillis(), TimeUnit.MILLISECONDS);
|
||||
try
|
||||
{
|
||||
plain.connect(new InetSocketAddress(address, port), (int) Math.min(5000, limit.toMillis()));
|
||||
plain.setSoTimeout(5000);
|
||||
SSLSocket socket = (SSLSocket) ((SSLSocketFactory) SSLSocketFactory.getDefault()).createSocket(plain, host, port, true);
|
||||
SSLParameters parameters = socket.getSSLParameters();
|
||||
parameters.setServerNames(List.of(new SNIHostName(host)));
|
||||
parameters.setEndpointIdentificationAlgorithm("HTTPS");
|
||||
socket.setSSLParameters(parameters);
|
||||
socket.startHandshake();
|
||||
socket.getOutputStream().write(("GET " + path + " HTTP/1.1\r\nHost: " + host + "\r\nUser-Agent: qbit-cdn-origin-check\r\nAccept: */*\r\nConnection: close\r\n\r\n")
|
||||
.getBytes(StandardCharsets.US_ASCII));
|
||||
socket.getOutputStream().flush();
|
||||
return (readResponse(new BufferedInputStream(socket.getInputStream()), deadline));
|
||||
}
|
||||
finally
|
||||
{
|
||||
guard.cancel(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.model;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Applies help/qbit-cdn.yaml (the QQQ help-bundle format) to the CDN app
|
||||
** and its cdn_* tables: what each table is for and how its rows get created,
|
||||
** shown on the table screen and when it is empty.
|
||||
** STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.8): replace with HelpBundles.register once the pinned QQQ
|
||||
** release includes it; the bundle file stays the same.
|
||||
*******************************************************************************/
|
||||
public final class CdnHelp
|
||||
{
|
||||
public static final String RESOURCE = "/help/qbit-cdn.yaml";
|
||||
public static final String SLOT = "help";
|
||||
public static final String APP_KEY = "app:cdnAdmin;slot:help";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnHelp()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The bundle's entries by key, validated (fails startup if malformed).
|
||||
*******************************************************************************/
|
||||
public static Map<String, JsonNode> entries() throws QException
|
||||
{
|
||||
try(InputStream stream = CdnHelp.class.getResourceAsStream(RESOURCE))
|
||||
{
|
||||
if(stream == null)
|
||||
{
|
||||
throw (new IOException("missing resource"));
|
||||
}
|
||||
JsonNode bundle = new ObjectMapper(new YAMLFactory()).readTree(stream);
|
||||
if(!"qbit-cdn".equals(bundle.path("bundle").asText()) || !"QBIT".equals(bundle.path("layer").asText()))
|
||||
{
|
||||
throw (new IOException("invalid bundle header"));
|
||||
}
|
||||
Map<String, JsonNode> entries = new LinkedHashMap<>();
|
||||
for(JsonNode entry : bundle.path("entries"))
|
||||
{
|
||||
String key = entry.path("key").asText();
|
||||
String summary = entry.path("summary").asText();
|
||||
if(key.isBlank() || summary.isBlank() || summary.length() > 120 || entry.path("content").asText().isBlank() || entries.put(key, entry) != null)
|
||||
{
|
||||
throw (new IOException("invalid or duplicate entry: " + key));
|
||||
}
|
||||
}
|
||||
return (entries);
|
||||
}
|
||||
catch(IOException | RuntimeException e)
|
||||
{
|
||||
throw (new QException("Could not load " + RESOURCE, e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Apply the help to the app and every cdn_* table in the instance. A host's
|
||||
** own help in the slot is kept (the bundle only fills it).
|
||||
*******************************************************************************/
|
||||
public static void apply(QInstance qInstance, String appName) throws QException
|
||||
{
|
||||
Map<String, JsonNode> entries = entries();
|
||||
for(QTableMetaData table : qInstance.getTables().values())
|
||||
{
|
||||
JsonNode entry = entries.get("table:" + table.getName() + ";slot:" + SLOT);
|
||||
if(entry != null && (table.getHelpContent() == null || !table.getHelpContent().containsKey(SLOT)))
|
||||
{
|
||||
table.withHelpContent(SLOT, content(entry));
|
||||
}
|
||||
}
|
||||
QAppMetaData app = qInstance.getApp(appName);
|
||||
JsonNode appEntry = entries.get(APP_KEY);
|
||||
if(app != null && appEntry != null && (app.getHelpContent() == null || !app.getHelpContent().containsKey(SLOT)))
|
||||
{
|
||||
app.withHelpContent(SLOT, content(appEntry));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Summary as the first paragraph, then the content, as markdown.
|
||||
*******************************************************************************/
|
||||
static QHelpContent content(JsonNode entry)
|
||||
{
|
||||
return (new QHelpContent().withContentAsMarkdown(entry.path("summary").asText().strip() + "\n\n" + entry.path("content").asText().strip()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.model;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Signature format selected on each revalidation hook.
|
||||
*******************************************************************************/
|
||||
public enum CdnSignatureFormat implements PossibleValueEnum<String>
|
||||
{
|
||||
STANDARD_WEBHOOKS("STANDARD_WEBHOOKS", "Standard Webhooks"),
|
||||
LEGACY("LEGACY", "Legacy X-Revalidate-Signature");
|
||||
|
||||
private final String id;
|
||||
private final String label;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
CdnSignatureFormat(String id, String label)
|
||||
{
|
||||
this.id = id;
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for possibleValueId.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getPossibleValueId()
|
||||
{
|
||||
return (id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for possibleValueLabel.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (label);
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
@@ -54,11 +55,12 @@ public final class CdnTables
|
||||
public static final List<String> ALL = List.of(PROVIDER, HOSTNAME, CACHE_POLICY, TABLE_TAG_RULE, PURGE_REQUEST, REVALIDATION_HOOK,
|
||||
EDGE_STAT_DAILY, ORIGIN_CHECK, DRIFT_FINDING);
|
||||
|
||||
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
|
||||
public static final String PVS_STAGE = "cdn_hostname_stage";
|
||||
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
|
||||
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
|
||||
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
|
||||
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
|
||||
public static final String PVS_STAGE = "cdn_hostname_stage";
|
||||
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
|
||||
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
|
||||
public static final String PVS_SIGNATURE_FORMAT = "cdn_signature_format";
|
||||
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
|
||||
|
||||
|
||||
|
||||
@@ -95,6 +97,7 @@ public final class CdnTables
|
||||
QPossibleValueSource.newForEnum(PVS_PURGE_STATUS, CdnPurgeStatus.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_PURGE_SOURCE, CdnPurgeSource.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_POLICY_CLASS, CdnPolicyClass.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_SIGNATURE_FORMAT, CdnSignatureFormat.values()),
|
||||
QPossibleValueSource.newForTable(PROVIDER),
|
||||
QPossibleValueSource.newForTable(HOSTNAME)));
|
||||
}
|
||||
@@ -228,7 +231,8 @@ public final class CdnTables
|
||||
.withRecordLabelFormatAndFields("%s", "hostname")
|
||||
.withField(new QFieldMetaData("hostname", QFieldType.STRING).withIsRequired(true).withMaxLength(255))
|
||||
.withField(new QFieldMetaData("cdnProviderId", QFieldType.INTEGER).withIsRequired(true).withPossibleValueSourceName(PROVIDER))
|
||||
.withField(new QFieldMetaData("hostnameRole", QFieldType.STRING).withMaxLength(20))
|
||||
.withField(new QFieldMetaData("hostnameRole", QFieldType.STRING).withMaxLength(20)
|
||||
.withHelpContent(new QHelpContent("A label for your own reference (for example WEB, API or ASSETS). It is not used to change any behavior.")))
|
||||
.withField(new QFieldMetaData("stage", QFieldType.STRING).withIsRequired(true).withPossibleValueSourceName(PVS_STAGE).withDefaultValue(CdnHostnameStage.OFF.name()))
|
||||
.withField(new QFieldMetaData("purgeEnabled", QFieldType.BOOLEAN).withDefaultValue(true))
|
||||
.withField(new QFieldMetaData("notes", QFieldType.TEXT)));
|
||||
@@ -271,7 +275,8 @@ public final class CdnTables
|
||||
.withField(new QFieldMetaData("tableName", QFieldType.STRING).withIsRequired(true).withMaxLength(100))
|
||||
.withField(new QFieldMetaData("tagTemplates", QFieldType.TEXT))
|
||||
.withField(new QFieldMetaData("invalidationPaths", QFieldType.TEXT))
|
||||
.withField(new QFieldMetaData("onlyWhenFieldsChange", QFieldType.STRING).withMaxLength(1000))
|
||||
.withField(new QFieldMetaData("onlyWhenFieldsChange", QFieldType.STRING).withMaxLength(1000)
|
||||
.withHelpContent(new QHelpContent("Comma-separated, case-sensitive field names. Plain entries allow only those changes; !viewCount ignores viewCount changes. With only ignored entries, any other changed field counts. In mixed lists, ignored fields never count. Inserts and deletes are unaffected by this filter.")))
|
||||
.withField(new QFieldMetaData("publishedField", QFieldType.STRING).withMaxLength(100))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)));
|
||||
}
|
||||
@@ -318,6 +323,9 @@ public final class CdnTables
|
||||
.withField(new QFieldMetaData("url", QFieldType.STRING).withIsRequired(true).withMaxLength(500))
|
||||
.withField(new QFieldMetaData("fanOutDns", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData("secretRef", QFieldType.STRING).withMaxLength(100))
|
||||
.withField(new QFieldMetaData("signatureFormat", QFieldType.STRING).withPossibleValueSourceName(PVS_SIGNATURE_FORMAT).withDefaultValue(CdnSignatureFormat.STANDARD_WEBHOOKS.name()))
|
||||
.withField(new QFieldMetaData("allowPlainHttp", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData("internalAllowlist", QFieldType.TEXT))
|
||||
.withField(new QFieldMetaData("timeoutMs", QFieldType.INTEGER).withDefaultValue(5000))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import com.kingsrook.qbits.cdn.CdnQBitConfig;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.permissions.CdnPermissions;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import static com.kingsrook.qbits.cdn.processes.CdnProcesses.line;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The "Create defaults" table actions that give an empty CDN setup something to
|
||||
** start from. Each is a standard streamed ETL (Review lists the rows that will
|
||||
** be added, Result the rows that were), needs no selected records, needs
|
||||
** cdn.admin, and is insert-only: a row (or the whole kind of row) that exists is
|
||||
** never changed or duplicated.
|
||||
** <ul>
|
||||
** <li>Create default provider (cdn_provider): a "No CDN" provider, so changes
|
||||
** are recorded as would-be purges and nothing is sent, until a real CDN is
|
||||
** added;</li>
|
||||
** <li>Create default policies (cdn_cache_policy): two disabled examples
|
||||
** (immutable assets, never cache) to edit and turn on;</li>
|
||||
** <li>Create default rules (cdn_table_tag_rule): one tag rule per purgeable table
|
||||
** that has none (tags only; paths are the application's to add).</li>
|
||||
** </ul>
|
||||
** These are actions, not a database seed, because the qbit ships no DDL and a
|
||||
** host chooses its own column names; they work on whatever the backend is.
|
||||
*******************************************************************************/
|
||||
public final class CdnDefaultsProcesses
|
||||
{
|
||||
public static final String CREATE_PROVIDER = "cdnCreateDefaultProvider";
|
||||
public static final String CREATE_POLICIES = "cdnCreateDefaultPolicies";
|
||||
public static final String CREATE_RULES = "cdnCreateDefaultRules";
|
||||
|
||||
public static final String NONE_PROVIDER_NAME = "No CDN (record only)";
|
||||
public static final String ASSETS_POLICY_NAME = "Static assets (immutable)";
|
||||
public static final String NEVER_POLICY_NAME = "Never cache (default)";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnDefaultsProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The three actions.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes()
|
||||
{
|
||||
return (List.of(
|
||||
define(CREATE_PROVIDER, "Create default provider", CdnTables.PROVIDER, ProviderExtract.class,
|
||||
"This provider will be added. It records purges and sends nothing, until you add a real CDN."),
|
||||
define(CREATE_POLICIES, "Create default policies", CdnTables.CACHE_POLICY, PolicyExtract.class,
|
||||
"These policies will be added turned off, as examples to edit and turn on."),
|
||||
define(CREATE_RULES, "Create default rules", CdnTables.TABLE_TAG_RULE, RuleExtract.class,
|
||||
"These tag rules will be added, one for each purgeable table that has none.")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One table action.
|
||||
*******************************************************************************/
|
||||
private static QProcessMetaData define(String name, String label, String table, Class<? extends AbstractExtractStep> extract, String preview)
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, preview + " Existing rows are never changed.");
|
||||
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(extract, DefaultsTransform.class, DefaultsLoad.class, values);
|
||||
process.setName(name);
|
||||
process.setLabel(label);
|
||||
process.setIcon(new QIcon("playlist_add"));
|
||||
process.setTableName(table);
|
||||
process.setMinInputRecords(0);
|
||||
process.setPermissionRules(CdnTables.permissionRules());
|
||||
return (CdnProcesses.permission(process, CdnPermissions.ADMIN));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The rows the provider action would add: the No CDN provider, when the table
|
||||
** has no NONE provider at all.
|
||||
*******************************************************************************/
|
||||
public static List<Map<String, Serializable>> missingProviders() throws QException
|
||||
{
|
||||
for(QRecord row : all(CdnTables.PROVIDER))
|
||||
{
|
||||
if(CdnProviderType.NONE.name().equals(row.getValueString("providerType")))
|
||||
{
|
||||
return (List.of());
|
||||
}
|
||||
}
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("name", NONE_PROVIDER_NAME);
|
||||
row.put("providerType", CdnProviderType.NONE.name());
|
||||
row.put("enabled", true);
|
||||
return (List.of(row));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The policies the action would add, by name: both disabled, so adding them
|
||||
** changes no response until an administrator turns one on.
|
||||
*******************************************************************************/
|
||||
public static List<Map<String, Serializable>> missingPolicies() throws QException
|
||||
{
|
||||
Set<String> existing = new HashSet<>();
|
||||
all(CdnTables.CACHE_POLICY).forEach(row -> existing.add(row.getValueString("name")));
|
||||
List<Map<String, Serializable>> rows = new ArrayList<>();
|
||||
if(!existing.contains(ASSETS_POLICY_NAME))
|
||||
{
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("name", ASSETS_POLICY_NAME);
|
||||
row.put("matchType", "GLOB");
|
||||
row.put("matchValue", "/assets/*");
|
||||
row.put("policyClass", "IMMUTABLE");
|
||||
row.put("browserMaxAge", 31536000);
|
||||
row.put("edgeMaxAge", 31536000);
|
||||
row.put("immutable", true);
|
||||
row.put("noStore", false);
|
||||
row.put("priority", 10);
|
||||
row.put("enabled", false);
|
||||
row.put("seeded", true);
|
||||
rows.add(row);
|
||||
}
|
||||
if(!existing.contains(NEVER_POLICY_NAME))
|
||||
{
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("name", NEVER_POLICY_NAME);
|
||||
row.put("matchType", "PREFIX");
|
||||
row.put("matchValue", "/");
|
||||
row.put("policyClass", "NEVER");
|
||||
row.put("immutable", false);
|
||||
row.put("noStore", true);
|
||||
row.put("priority", 1000);
|
||||
row.put("enabled", false);
|
||||
row.put("seeded", true);
|
||||
rows.add(row);
|
||||
}
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The rules the action would add: tags "table" and "table:id" for each purgeable
|
||||
** table without a rule (any rule, enabled or not).
|
||||
*******************************************************************************/
|
||||
public static List<Map<String, Serializable>> missingRules() throws QException
|
||||
{
|
||||
CdnQBitConfig config = CdnRuntime.getConfig();
|
||||
Set<String> have = new HashSet<>();
|
||||
all(CdnTables.TABLE_TAG_RULE).forEach(row -> have.add(row.getValueString("tableName")));
|
||||
List<Map<String, Serializable>> rows = new ArrayList<>();
|
||||
for(String table : config == null ? List.<String>of() : config.getPurgeableTables())
|
||||
{
|
||||
if(!have.contains(table))
|
||||
{
|
||||
Map<String, Serializable> row = new LinkedHashMap<>();
|
||||
row.put("tableName", table);
|
||||
row.put("tagTemplates", table + ":{id}\n" + table);
|
||||
row.put("enabled", true);
|
||||
rows.add(row);
|
||||
}
|
||||
}
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> all(String table) throws QException
|
||||
{
|
||||
return (new QueryAction().execute(new QueryInput(table)).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Extract the rows to add (computed; nothing is read from the selected records).
|
||||
*******************************************************************************/
|
||||
public abstract static class DefaultsExtract extends AbstractExtractStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** The table the rows go in.
|
||||
*******************************************************************************/
|
||||
protected abstract String table();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The rows that are missing.
|
||||
*******************************************************************************/
|
||||
protected abstract List<Map<String, Serializable>> missing() throws QException;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(Map<String, Serializable> values : missing())
|
||||
{
|
||||
QRecord record = new QRecord().withTableName(table());
|
||||
record.setValues(new LinkedHashMap<>(values));
|
||||
getRecordPipe().addRecord(record);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The provider rows.
|
||||
*******************************************************************************/
|
||||
public static class ProviderExtract extends DefaultsExtract
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected String table()
|
||||
{
|
||||
return (CdnTables.PROVIDER);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected List<Map<String, Serializable>> missing() throws QException
|
||||
{
|
||||
return (missingProviders());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The cache policy rows.
|
||||
*******************************************************************************/
|
||||
public static class PolicyExtract extends DefaultsExtract
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected String table()
|
||||
{
|
||||
return (CdnTables.CACHE_POLICY);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected List<Map<String, Serializable>> missing() throws QException
|
||||
{
|
||||
return (missingPolicies());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The tag rule rows.
|
||||
*******************************************************************************/
|
||||
public static class RuleExtract extends DefaultsExtract
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected String table()
|
||||
{
|
||||
return (CdnTables.TABLE_TAG_RULE);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
protected List<Map<String, Serializable>> missing() throws QException
|
||||
{
|
||||
return (missingRules());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review: the rows that will be added.
|
||||
*******************************************************************************/
|
||||
public static class DefaultsTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine added = line(Status.OK, "default row will be added", "default rows will be added", "default row was added", "default rows were added");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
added.incrementCount();
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
added.addSelfToListIfAnyCount(lines);
|
||||
if(lines.isEmpty())
|
||||
{
|
||||
lines.add(new ProcessSummaryLine(Status.INFO, null, "Nothing to add: the defaults are already there."));
|
||||
}
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result: insert through the normal insert, so the user's own permissions apply.
|
||||
*******************************************************************************/
|
||||
public static class DefaultsLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine added = line(Status.OK, "default row will be added", "default rows will be added", "default row was added", "default rows were added");
|
||||
private final ProcessSummaryLine failed = line(Status.ERROR, "default row could not be added", "default rows could not be added", "default row could not be added", "default rows could not be added");
|
||||
private final List<String> problems = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord inserted : new InsertAction().execute(new InsertInput(input.getTableName()).withRecords(input.getRecords())).getRecords())
|
||||
{
|
||||
if(inserted.getErrors() == null || inserted.getErrors().isEmpty())
|
||||
{
|
||||
added.incrementCountAndAddPrimaryKey(inserted.getValue("id"));
|
||||
output.getRecords().add(inserted);
|
||||
}
|
||||
else
|
||||
{
|
||||
failed.incrementCount();
|
||||
inserted.getErrors().forEach(error -> problems.add(error.getMessage()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
added.addSelfToListIfAnyCount(lines);
|
||||
failed.setBulletsOfText(new ArrayList<>(problems));
|
||||
failed.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.processes;
|
||||
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.CdnQBitConfig;
|
||||
import com.kingsrook.qbits.cdn.checks.CdnOriginCheck;
|
||||
import com.kingsrook.qbits.cdn.model.CdnHostnameStage;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.permissions.CdnPermissions;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.logging.QLogger;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import static com.kingsrook.qbits.cdn.processes.CdnProcesses.line;
|
||||
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Check Origin Headers: a record action on cdn_hostname (standard streamed ETL:
|
||||
** Review lists the hostnames, Result says how many sample objects had the right
|
||||
** Cache-Control) and a daily scheduled run over every hostname that is not Off.
|
||||
** The work is CdnOriginCheck; each run writes one cdn_origin_check row per hostname.
|
||||
*******************************************************************************/
|
||||
public final class CdnOriginCheckProcesses
|
||||
{
|
||||
private static final QLogger LOG = QLogger.getLogger(CdnOriginCheckProcesses.class);
|
||||
|
||||
public static final String CHECK_ORIGIN = "cdnCheckOrigin";
|
||||
public static final String SCHEDULED_ORIGIN = "cdnScheduledCheckOrigin";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private CdnOriginCheckProcesses()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The processes.
|
||||
*******************************************************************************/
|
||||
public static List<QProcessMetaData> processes(CdnQBitConfig config)
|
||||
{
|
||||
List<QProcessMetaData> processes = new ArrayList<>();
|
||||
QProcessMetaData action = CdnProcesses.etl(CHECK_ORIGIN, "Check Origin Headers", CdnTables.HOSTNAME, CheckTransform.class, CheckLoad.class,
|
||||
"This is a preview of the hostnames whose origin will be asked for a few sample pages, to check their Cache-Control headers.");
|
||||
action.setIcon(new QIcon("fact_check"));
|
||||
processes.add(CdnProcesses.permission(action, CdnPermissions.ADMIN));
|
||||
|
||||
QProcessMetaData scheduled = new QProcessMetaData().withName(SCHEDULED_ORIGIN).withLabel("Check CDN Origin Headers").withIcon(new QIcon("schedule")).withIsHidden(true)
|
||||
.withPermissionRules(CdnTables.permissionRules())
|
||||
.withStep(new QBackendStepMetaData().withName("run").withCode(new QCodeReference(ScheduledStep.class)));
|
||||
if(config.getSchedulerName() != null)
|
||||
{
|
||||
scheduled.withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName()).withCronExpression("0 45 2 * * ?").withCronTimeZoneId("UTC"));
|
||||
}
|
||||
processes.add(CdnProcesses.permission(scheduled, CdnPermissions.ADMIN));
|
||||
return (processes);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Review: the hostnames that will be checked.
|
||||
*******************************************************************************/
|
||||
public static class CheckTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine hosts = line(Status.OK, "hostname will be checked", "hostnames will be checked", "hostname was checked", "hostnames were checked");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord hostname : input.getRecords())
|
||||
{
|
||||
hosts.incrementCountAndAddPrimaryKey(hostname.getValue("id"));
|
||||
output.getRecords().add(hostname);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
hosts.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result: check each hostname and record the row.
|
||||
*******************************************************************************/
|
||||
public static class CheckLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine clean = line(Status.OK, "hostname will be checked", "hostnames will be checked", "hostname has the right headers", "hostnames have the right headers");
|
||||
private final ProcessSummaryLine wrong = line(Status.WARNING, "page has a missing or wrong Cache-Control", "pages have a missing or wrong Cache-Control",
|
||||
"page has a missing or wrong Cache-Control", "pages have a missing or wrong Cache-Control");
|
||||
private final ProcessSummaryLine unreadable = line(Status.WARNING, "page could not be read", "pages could not be read", "page could not be read", "pages could not be read");
|
||||
private final List<String> problems = new ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord hostname : input.getRecords())
|
||||
{
|
||||
CdnOriginCheck.Result result = CdnOriginCheck.checkAndRecord(hostname);
|
||||
if(result.wrong() == 0 && result.unreadable() == 0)
|
||||
{
|
||||
clean.incrementCountAndAddPrimaryKey(hostname.getValue("id"));
|
||||
}
|
||||
wrong.incrementCount(result.wrong());
|
||||
unreadable.incrementCount(result.unreadable());
|
||||
result.problems().forEach(problem -> problems.add(result.scope() + problem));
|
||||
output.getRecords().add(hostname);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
clean.addSelfToListIfAnyCount(lines);
|
||||
wrong.setBulletsOfText(new ArrayList<>(problems));
|
||||
wrong.addSelfToListIfAnyCount(lines);
|
||||
unreadable.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled: every hostname that is not Off.
|
||||
*******************************************************************************/
|
||||
public static class ScheduledStep implements BackendStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
** {@inheritDoc}
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord hostname : new QueryAction().execute(new QueryInput(CdnTables.HOSTNAME)
|
||||
.withFilter(new QQueryFilter(new QFilterCriteria("stage", QCriteriaOperator.NOT_EQUALS, CdnHostnameStage.OFF.name())))).getRecords())
|
||||
{
|
||||
try
|
||||
{
|
||||
CdnOriginCheck.checkAndRecord(hostname);
|
||||
}
|
||||
catch(RuntimeException | QException e)
|
||||
{
|
||||
LOG.warn("An origin check failed", e, logPair("hostname", hostname.getValueString("hostname")));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -167,6 +167,56 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Summary counts for requests that cannot be sent, shared by all actions.
|
||||
*******************************************************************************/
|
||||
private static class UnsentSummary
|
||||
{
|
||||
private final ProcessSummaryLine absent = line(Status.WARNING,
|
||||
"record has no enabled CDN provider", "records have no enabled CDN provider",
|
||||
"record had no enabled CDN provider (no purge request recorded)", "records had no enabled CDN provider (no purge requests recorded)");
|
||||
private final ProcessSummaryLine none = line(Status.WARNING,
|
||||
"No CDN provider configured: purges are recorded only, not sent.", "No CDN provider configured: purges are recorded only, not sent.",
|
||||
"purge request was recorded (not sent: no CDN provider)", "purge requests were recorded (not sent: no CDN provider)");
|
||||
private final ProcessSummaryLine off = line(Status.WARNING,
|
||||
"CDN hostnames are not live: purges are recorded only, not sent.", "CDN hostnames are not live: purges are recorded only, not sent.",
|
||||
"purge request was recorded (not sent: CDN hostnames are not live)", "purge requests were recorded (not sent: CDN hostnames are not live)");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count a skipped request; returns whether it cannot be sent.
|
||||
*******************************************************************************/
|
||||
boolean add(CdnPurgeStatus status)
|
||||
{
|
||||
if(status == CdnPurgeStatus.SKIPPED_NONE_PROVIDER)
|
||||
{
|
||||
none.incrementCount(1);
|
||||
return (true);
|
||||
}
|
||||
if(status == CdnPurgeStatus.SKIPPED_STAGE_OFF)
|
||||
{
|
||||
off.incrementCount(1);
|
||||
return (true);
|
||||
}
|
||||
return (false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Include only reasons encountered by this action.
|
||||
*******************************************************************************/
|
||||
void append(ArrayList<ProcessSummaryLineInterface> lines)
|
||||
{
|
||||
absent.addSelfToListIfAnyCount(lines);
|
||||
none.addSelfToListIfAnyCount(lines);
|
||||
off.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled: send due purges and poll in-progress ones.
|
||||
*******************************************************************************/
|
||||
@@ -189,6 +239,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class PurgeRecordsTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine records = line(Status.OK, "record will be purged", "records will be purged",
|
||||
"record was purged", "records were purged");
|
||||
private final ProcessSummaryLine noRule = line(Status.INFO, "record has no CDN tag rule (no change)", "records have no CDN tag rule (no change)",
|
||||
@@ -207,6 +258,7 @@ public final class CdnProcesses
|
||||
String table = input.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
|
||||
List<QRecord> rules = rules(table);
|
||||
TagRuleEngine engine = new TagRuleEngine();
|
||||
CdnPurgeStatus sending = CdnPurgeService.sendingStatus(null);
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
List<String> tags = new ArrayList<>();
|
||||
@@ -226,7 +278,10 @@ public final class CdnProcesses
|
||||
recordPaths.forEach(path -> paths.add(path.path()));
|
||||
record.setValue("cdnTags", String.join("\n", tags));
|
||||
record.setValue("cdnPaths", com.kingsrook.qbits.cdn.purge.PurgeText.paths(recordPaths));
|
||||
records.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(sending))
|
||||
{
|
||||
records.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -249,6 +304,7 @@ public final class CdnProcesses
|
||||
lines.add(pathLine);
|
||||
}
|
||||
noRule.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -256,10 +312,11 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Purge From CDN, execute: one purge request (sent at once, hubs included).
|
||||
** Purge From CDN, execute: record requests with hubs included.
|
||||
*******************************************************************************/
|
||||
public static class PurgeRecordsLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"purge request was recorded (sent within seconds)", "purge requests were recorded (sent within seconds)");
|
||||
|
||||
@@ -277,7 +334,17 @@ public final class CdnProcesses
|
||||
List<QRecord> rows = CdnPurgeService.enqueue(new CdnPurgeService.Request(CdnPurgeKind.TAG.name(),
|
||||
com.kingsrook.qbits.cdn.purge.PurgeText.tags(record.getValueString("cdnTags")),
|
||||
com.kingsrook.qbits.cdn.purge.PurgeText.paths(record.getValueString("cdnPaths")), true, CdnPurgeSource.RECORD_ACTION, table, record.getValue("id")));
|
||||
requested.incrementCount(rows.size());
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
unsent.absent.incrementCount(1);
|
||||
}
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
if(!unsent.add(CdnPurgeStatus.valueOf(row.getValueString("status"))))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -296,6 +363,7 @@ public final class CdnProcesses
|
||||
{
|
||||
transform.noRule.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -307,6 +375,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class RetryTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine retry = line(Status.OK, "purge request will be sent again", "purge requests will be sent again",
|
||||
"purge request was queued again", "purge requests were queued again");
|
||||
private final ProcessSummaryLine skip = line(Status.INFO, "purge request has not failed (no change)", "purge requests have not failed (no change)",
|
||||
@@ -325,7 +394,10 @@ public final class CdnProcesses
|
||||
String status = record.getValueString("status");
|
||||
if(CdnPurgeStatus.FAILED.name().equals(status) || CdnPurgeStatus.DEAD.name().equals(status) || CdnPurgeStatus.DEFERRED_BUDGET.name().equals(status))
|
||||
{
|
||||
retry.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(CdnPurgeService.sendingStatus(record.getValue("cdnProviderId"))))
|
||||
{
|
||||
retry.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
else
|
||||
@@ -346,6 +418,7 @@ public final class CdnProcesses
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
retry.addSelfToListIfAnyCount(lines);
|
||||
skip.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -353,10 +426,16 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Retry Purge, execute: back to PENDING (sent on the next dispatcher pass).
|
||||
** Retry Purge, execute: queue live providers or record why sending is skipped.
|
||||
*******************************************************************************/
|
||||
public static class RetryLoad extends AbstractLoadStep
|
||||
public static class RetryLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"purge request was queued again", "purge requests were queued again");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reset the rows.
|
||||
*******************************************************************************/
|
||||
@@ -366,7 +445,12 @@ public final class CdnProcesses
|
||||
List<QRecord> updates = new ArrayList<>();
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("status", CdnPurgeStatus.PENDING.name())
|
||||
CdnPurgeStatus status = CdnPurgeService.sendingStatus(record.getValue("cdnProviderId"));
|
||||
if(!unsent.add(status))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("status", status.name())
|
||||
.withValue("source", CdnPurgeSource.RETRY.name()).withValue("attempts", 0).withValue("nextAttemptDate", null)
|
||||
.withValue("errorMessage", null).withValue("completedDate", null).withValue("modifyDate", CdnRuntime.clock().instant()));
|
||||
}
|
||||
@@ -375,6 +459,24 @@ public final class CdnProcesses
|
||||
output.getRecords().addAll(new UpdateAction().execute(new UpdateInput(CdnTables.PURGE_REQUEST).withRecords(updates)).getRecords());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result lines from the requests recorded during execution.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
requested.addSelfToListIfAnyCount(lines);
|
||||
if(getTransformStep() instanceof RetryTransform transform)
|
||||
{
|
||||
transform.skip.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -384,6 +486,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class PurgeHostnameTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine providers = line(Status.WARNING,
|
||||
"CDN distribution will be purged completely (every hostname on it)", "CDN distributions will be purged completely (every hostname on them)",
|
||||
"CDN distribution purge was recorded", "CDN distribution purges were recorded");
|
||||
@@ -402,7 +505,10 @@ public final class CdnProcesses
|
||||
Serializable providerId = record.getValue("cdnProviderId");
|
||||
if(providerId != null && seen.add(providerId))
|
||||
{
|
||||
providers.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(CdnPurgeService.sendingStatus(providerId)))
|
||||
{
|
||||
providers.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -418,6 +524,7 @@ public final class CdnProcesses
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
providers.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -427,8 +534,14 @@ public final class CdnProcesses
|
||||
/*******************************************************************************
|
||||
** Purge Everything, execute.
|
||||
*******************************************************************************/
|
||||
public static class PurgeHostnameLoad extends AbstractLoadStep
|
||||
public static class PurgeHostnameLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"CDN distribution purge was recorded", "CDN distribution purges were recorded");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record an ALL purge for each provider.
|
||||
*******************************************************************************/
|
||||
@@ -437,11 +550,36 @@ public final class CdnProcesses
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
CdnPurgeService.enqueueFor(record.getValue("cdnProviderId"), new CdnPurgeService.Request(CdnPurgeKind.ALL.name(), List.of(), List.of(), true,
|
||||
List<QRecord> rows = CdnPurgeService.enqueueFor(record.getValue("cdnProviderId"), new CdnPurgeService.Request(CdnPurgeKind.ALL.name(), List.of(), List.of(), true,
|
||||
CdnPurgeSource.MANUAL, CdnTables.HOSTNAME, record.getValue("id")));
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
unsent.absent.incrementCount(1);
|
||||
}
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
if(!unsent.add(CdnPurgeStatus.valueOf(row.getValueString("status"))))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result lines from the requests recorded during execution.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
requested.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -120,6 +120,36 @@ public final class CdnPurgeService
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sending eligibility for one provider, or all enabled providers when null.
|
||||
** A pending status means at least one provider can send; otherwise the
|
||||
** status gives the same reason used when recording a new request.
|
||||
*******************************************************************************/
|
||||
public static CdnPurgeStatus sendingStatus(Serializable providerId) throws QException
|
||||
{
|
||||
QQueryFilter filter = new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true));
|
||||
if(providerId != null)
|
||||
{
|
||||
filter.addCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, providerId));
|
||||
}
|
||||
CdnPurgeStatus status = CdnPurgeStatus.SKIPPED_NONE_PROVIDER;
|
||||
for(QRecord provider : new QueryAction().execute(new QueryInput(CdnTables.PROVIDER).withFilter(filter)).getRecords())
|
||||
{
|
||||
CdnPurgeStatus candidate = initialStatus(provider);
|
||||
if(candidate == CdnPurgeStatus.PENDING)
|
||||
{
|
||||
return (candidate);
|
||||
}
|
||||
if(candidate == CdnPurgeStatus.SKIPPED_STAGE_OFF)
|
||||
{
|
||||
status = candidate;
|
||||
}
|
||||
}
|
||||
return (status);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Status of a new request for a provider.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -21,16 +21,18 @@ import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.cdn.model.CdnSignatureFormat;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
@@ -41,17 +43,20 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpClient;
|
||||
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpPolicy;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSigner;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Calls every enabled cdn_revalidation_hook before the CDN is purged, so the
|
||||
** edge never refetches a stale page (design spec section 4.5 step 3).
|
||||
**
|
||||
** Request: POST {"tags":[...]} with header
|
||||
** "X-Revalidate-Signature: sha256=<hex HMAC-SHA256 of the body>", keyed by
|
||||
** the secret in the environment variable the hook's secretRef names. With
|
||||
** fanOutDns, the hook's host is resolved and every address is called (one
|
||||
** per pod behind a headless Service). Every call must answer 2xx.
|
||||
** Request: POST {"tags":[...]} signed with core Standard Webhooks headers.
|
||||
** LEGACY keeps X-Revalidate-Signature with a hex HMAC of the body. The
|
||||
** secretRef names an environment variable. SafeHttpClient checks and pins
|
||||
** every address; fan-out preserves the original Host and TLS server name.
|
||||
*******************************************************************************/
|
||||
public class RevalidationHookRunner
|
||||
{
|
||||
@@ -59,9 +64,8 @@ public class RevalidationHookRunner
|
||||
|
||||
private static final ObjectMapper JSON = new ObjectMapper();
|
||||
|
||||
private final HttpClient http;
|
||||
private final Function<String, String> secrets;
|
||||
private final Resolver resolver;
|
||||
private final Function<String, String> secrets;
|
||||
private final Resolver resolver;
|
||||
|
||||
|
||||
|
||||
@@ -83,7 +87,7 @@ public class RevalidationHookRunner
|
||||
*******************************************************************************/
|
||||
public RevalidationHookRunner()
|
||||
{
|
||||
this(HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(), System::getenv,
|
||||
this(System::getenv,
|
||||
host -> java.util.Arrays.stream(InetAddress.getAllByName(host)).map(InetAddress::getHostAddress).toList());
|
||||
}
|
||||
|
||||
@@ -92,9 +96,8 @@ public class RevalidationHookRunner
|
||||
/*******************************************************************************
|
||||
** Constructor with collaborators.
|
||||
*******************************************************************************/
|
||||
public RevalidationHookRunner(HttpClient http, Function<String, String> secrets, Resolver resolver)
|
||||
public RevalidationHookRunner(Function<String, String> secrets, Resolver resolver)
|
||||
{
|
||||
this.http = http;
|
||||
this.secrets = secrets;
|
||||
this.resolver = resolver;
|
||||
}
|
||||
@@ -133,42 +136,63 @@ public class RevalidationHookRunner
|
||||
byte[] body = JSON.writeValueAsBytes(java.util.Map.of("tags", tags));
|
||||
String secretRef = hook.getValueString("secretRef");
|
||||
String secret = StringUtils.hasContent(secretRef) ? secrets.apply(secretRef) : null;
|
||||
String formatValue = hook.getValueString("signatureFormat");
|
||||
CdnSignatureFormat format = formatValue == null ? CdnSignatureFormat.LEGACY : CdnSignatureFormat.valueOf(formatValue);
|
||||
WebhookSecret signingSecret = secret != null && format == CdnSignatureFormat.STANDARD_WEBHOOKS ? WebhookSecret.parse(secret) : null;
|
||||
String messageId = UUID.randomUUID().toString();
|
||||
URI uri = URI.create(hook.getValueString("url"));
|
||||
List<URI> targets = new ArrayList<>();
|
||||
if(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns"))))
|
||||
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
|
||||
if(timeout != null && timeout <= 0)
|
||||
{
|
||||
for(String address : resolver.resolve(uri.getHost()))
|
||||
{
|
||||
String host = address.contains(":") ? "[" + address + "]" : address;
|
||||
targets.add(new URI(uri.getScheme(), null, host, uri.getPort(), uri.getPath(), uri.getQuery(), null));
|
||||
}
|
||||
throw (new QException("Revalidation hook timeoutMs must be positive"));
|
||||
}
|
||||
else
|
||||
String allowlist = hook.getValueString("internalAllowlist");
|
||||
SafeHttpPolicy policy = SafeHttpPolicy.defaults()
|
||||
.withAllowPlainHttp(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("allowPlainHttp"))))
|
||||
.withInternalAllowlist(allowlist == null ? List.of() : Arrays.asList(allowlist.split("[,\\s]+")))
|
||||
.withTotalTimeout(Duration.ofMillis(timeout == null ? 5000 : timeout));
|
||||
List<String> resolved = resolver.resolve(uri.getHost());
|
||||
InetAddress[] addresses = new InetAddress[resolved.size()];
|
||||
for(int i = 0; i < resolved.size(); i++)
|
||||
{
|
||||
targets.add(uri);
|
||||
addresses[i] = InetAddress.getByName(resolved.get(i));
|
||||
}
|
||||
if(targets.isEmpty())
|
||||
if(addresses.length == 0)
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " resolved to no addresses"));
|
||||
}
|
||||
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
|
||||
for(URI target : targets)
|
||||
try(SafeHttpClient validator = new SafeHttpClient(policy, host -> addresses))
|
||||
{
|
||||
HttpRequest.Builder request = HttpRequest.newBuilder(target)
|
||||
.timeout(Duration.ofMillis(timeout == null ? 5000 : timeout))
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofByteArray(body));
|
||||
validator.checkUrl(uri.toString());
|
||||
}
|
||||
boolean fanOut = Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns")));
|
||||
int calls = fanOut ? addresses.length : 1;
|
||||
for(int i = 0; i < calls; i++)
|
||||
{
|
||||
InetAddress[] targetAddresses = fanOut ? new InetAddress[] {addresses[i]} : addresses;
|
||||
Map<String, String> headers = new LinkedHashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
if(secret != null)
|
||||
{
|
||||
request.header(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
|
||||
if(format == CdnSignatureFormat.LEGACY)
|
||||
{
|
||||
headers.put(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
|
||||
}
|
||||
else
|
||||
{
|
||||
headers.putAll(WebhookSigner.headers(messageId, Instant.now(), body, List.of(signingSecret)));
|
||||
}
|
||||
}
|
||||
HttpResponse<String> response = http.send(request.build(), HttpResponse.BodyHandlers.ofString());
|
||||
if(response.statusCode() / 100 != 2)
|
||||
try(SafeHttpClient client = new SafeHttpClient(policy, host -> targetAddresses))
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " answered " + response.statusCode() + " from " + target.getHost()));
|
||||
int status = client.post(uri.toString(), headers, body).statusCode();
|
||||
if(status / 100 != 2)
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " answered " + status + " from " + uri.getHost()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return (targets.size());
|
||||
return (calls);
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
@@ -188,7 +212,7 @@ public class RevalidationHookRunner
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Hex HMAC-SHA256.
|
||||
** Legacy compatibility: hex HMAC-SHA256 over the body with raw UTF-8 key bytes.
|
||||
*******************************************************************************/
|
||||
public static String sign(String secret, byte[] body) throws Exception
|
||||
{
|
||||
|
||||
@@ -55,8 +55,8 @@ import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
|
||||
** expires by TTL instead of being invalidated;
|
||||
** - publishedField: when set, only records that are (or were) published
|
||||
** count, so draft edits purge nothing;
|
||||
** - onlyWhenFieldsChange: comma list; an update that changes none of them
|
||||
** purges nothing.
|
||||
** - onlyWhenFieldsChange: comma list of watched fields; !field ignores a
|
||||
** field. With only ignored fields, any other changed field counts.
|
||||
**
|
||||
** A dotted placeholder "{blog.slug}" reads field "slug" of the record that
|
||||
** field "blogId" points to (the xId convention). The joined table is the one
|
||||
@@ -159,7 +159,27 @@ public class TagRuleEngine
|
||||
String watched = rule.getValueString("onlyWhenFieldsChange");
|
||||
if(current != null && old != null && StringUtils.hasContent(watched))
|
||||
{
|
||||
return (Arrays.stream(watched.split(",")).map(String::strip).filter(StringUtils::hasContent)
|
||||
Set<String> included = new LinkedHashSet<>();
|
||||
Set<String> ignored = new LinkedHashSet<>();
|
||||
for(String entry : watched.split(","))
|
||||
{
|
||||
String field = entry.strip();
|
||||
if(field.startsWith("!"))
|
||||
{
|
||||
ignored.add(field.substring(1).strip());
|
||||
}
|
||||
else if(StringUtils.hasContent(field))
|
||||
{
|
||||
included.add(field);
|
||||
}
|
||||
}
|
||||
if(included.isEmpty() && !ignored.isEmpty())
|
||||
{
|
||||
included.addAll(current.getValues().keySet());
|
||||
included.addAll(old.getValues().keySet());
|
||||
}
|
||||
included.removeAll(ignored);
|
||||
return (included.stream()
|
||||
.anyMatch(field -> !Objects.equals(current.getValue(field), old.getValue(field))));
|
||||
}
|
||||
return (true);
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
bundle: qbit-cdn
|
||||
version: 1
|
||||
layer: QBIT
|
||||
# Help for the CDN app and every cdn_* table, in the QQQ help-bundle format
|
||||
# (qqq#1198). Each table entry says what the table is for and how its rows get
|
||||
# created, so an empty table explains itself. CdnHelp applies it until the core
|
||||
# HelpBundles registry ships in a QQQ release (STAND-IN(core=#1198)).
|
||||
# Summaries stay under 120 characters (CdnTablesGuardrailTest).
|
||||
entries:
|
||||
- key: app:cdnAdmin;slot:help
|
||||
summary: "Put a CDN in front of the application: cache headers, change-driven purges, edge statistics and checks."
|
||||
content: |
|
||||
The Setup card on the home lists each part (provider, hostnames, cache policies, tag rules, revalidation hooks, origin checks) with its status and the next step. A new installation has nothing: run Create default provider on Providers (No CDN records purges and sends nothing), Create default rules on Tag rules, and add your CDN when you have one.
|
||||
|
||||
- key: table:cdn_provider;slot:help
|
||||
summary: "A CDN the application purges: its type, distribution id and the variable that holds its key."
|
||||
content: |
|
||||
**How rows get created:** an administrator adds one with New (pick the type, the distribution or zone id, and name the environment variable that holds the API key; the key itself is never stored), or runs Create default provider, which adds "No CDN (record only)": every record change is then recorded as a would-be purge and nothing is sent. Test Connection, Import Edge Stats and Check Config Drift are actions on a provider.
|
||||
|
||||
- key: table:cdn_hostname;slot:help
|
||||
summary: "A hostname behind a provider and its rollout stage: Off, Headers only, Live or Locked."
|
||||
content: |
|
||||
**How rows get created:** an administrator adds one with New for each hostname the CDN serves, at stage Off, then moves it to Headers only and Live as the rollout proceeds. Only Live and Locked hostnames with purging enabled are sent purges. Check Origin Headers is an action on a hostname, and hostnames that are not Off are checked every night. The role field is a label for your own reference and changes nothing.
|
||||
|
||||
- key: table:cdn_cache_policy;slot:help
|
||||
summary: "Cache-Control rules by path, route class or table; the lowest priority number that matches wins."
|
||||
content: |
|
||||
**How rows get created:** an administrator adds one with New, or runs Create default policies, which adds two turned-off examples (immutable static assets, never cache) to edit and turn on. A policy changes a response only where the application calls CdnResponseHeaderPolicy.decide when it answers; the qbit does not rewrite responses on its own. A response that sets a cookie or answers a signed-in user is always private, whatever the policies say.
|
||||
|
||||
- key: table:cdn_table_tag_rule;slot:help
|
||||
summary: "How a table's record changes become cache tags and invalidation paths for purges."
|
||||
content: |
|
||||
**How rows get created:** an administrator adds one with New for each purgeable table the application lists, or runs Create default rules, which adds a tags-only rule for every purgeable table that has none (tags "table:id" and "table"). Add invalidation paths where your CDN purges by path, and a published field so draft edits purge nothing.
|
||||
|
||||
- key: table:cdn_purge_request;slot:help
|
||||
summary: "Every purge the qbit recorded or sent, with its status, paths and tags."
|
||||
content: |
|
||||
**How rows get created:** only the qbit writes it: a change to a purgeable record, Purge From CDN on a record, or Purge Everything on a hostname. Without a provider nothing is recorded, so run Create default provider first. Retry Purge is an action here.
|
||||
|
||||
- key: table:cdn_revalidation_hook;slot:help
|
||||
summary: "A signed webhook called before a CDN purge, so a front-end cache (for example Next.js) refreshes first."
|
||||
content: |
|
||||
**How rows get created:** an administrator adds one with New: the URL, the signature format and the environment variable that holds the signing secret (never the secret). It is optional; add one only when a front end keeps its own cache.
|
||||
|
||||
- key: table:cdn_edge_stat_daily;slot:help
|
||||
summary: "Daily edge statistics imported from the CDN: requests, bytes, hit rate and error rates."
|
||||
content: |
|
||||
**How rows get created:** Import Edge Stats on a provider writes it, and a daily scheduled run does the same for every provider that supports it. A No CDN provider has no statistics.
|
||||
|
||||
- key: table:cdn_origin_check;slot:help
|
||||
summary: "Results of asking a hostname's origin for sample pages and comparing their Cache-Control headers."
|
||||
content: |
|
||||
**How rows get created:** Check Origin Headers on a hostname writes one row per hostname, and a nightly scheduled run does the same for every hostname that is not Off. It requests the home page, the paths of prefix cache policies and the literal paths of tag rules over HTTPS and counts the right, missing or wrong, and unreadable ones.
|
||||
|
||||
- key: table:cdn_drift_finding;slot:help
|
||||
summary: "Differences between the CDN's expected configuration and what the provider reports."
|
||||
content: |
|
||||
**How rows get created:** Check Config Drift on a provider writes it, and a nightly scheduled run does the same. It needs the expected configuration in the environment variable the provider names (CDN_EXPECTED_CONFIG by default); a finding is resolved when it no longer appears.
|
||||
@@ -58,10 +58,10 @@ class CdnHomeTest
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
String html = render(CdnWidgets.HOME_WIDGET);
|
||||
assertThat(html).contains("No CDN connected yet.").contains("No edge statistics yet.").contains("No purges recorded yet.")
|
||||
.contains("No path-billed provider connected yet.").contains("No origin header check has run yet.").contains("Nothing to compare yet.")
|
||||
assertThat(html).contains("No CDN provider configured: purges are recorded only.").contains("No edge statistics yet.").contains("No purges recorded yet.")
|
||||
.contains("No path-billed provider connected yet.").contains("No origin check yet.").contains("Nothing to compare yet.")
|
||||
.contains("minmax(min(100%,22rem),1fr)");
|
||||
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN connected yet.").contains("href=\"/cdnAdmin\"");
|
||||
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN provider configured: purges are recorded only.").contains("href=\"/cdnAdmin\"");
|
||||
}
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ class CdnHomeTest
|
||||
String html = CdnHomeView.render(data, "/cdnAdmin");
|
||||
assertThat(html).contains("d1.cloudfront.net").contains(age + " days (rotate every 90) - rotation due").contains("1,500").contains("91.5%")
|
||||
.contains("<polyline").contains("396 of 396").contains("BEHAVIOR").contains("1 open").contains("/cdnAdmin/cdn_purge_request/")
|
||||
.contains("www.example.com");
|
||||
.contains("www.example.com").doesNotContain("recorded only (not sent)", "nothing has been checked");
|
||||
assertThat(CdnHomeView.renderHealthCard(data, "/cdnAdmin")).contains("Needs attention");
|
||||
}
|
||||
|
||||
@@ -119,4 +119,31 @@ class CdnHomeTest
|
||||
assertEquals("50.0%", CdnHomeView.percent(new BigDecimal("0.5")));
|
||||
assertEquals("91.5%", CdnHomeView.percent(new BigDecimal("91.5")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** NONE and disabled providers do not imply a connection or a sent purge.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void recordedOnlyHome() throws Exception
|
||||
{
|
||||
for(boolean none : List.of(true, false))
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
Integer provider = CdnTestSupport.liveProvider(none ? CdnProviderType.NONE : CdnProviderType.CLOUDFRONT, null);
|
||||
if(!none)
|
||||
{
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.PROVIDER)
|
||||
.withRecord(new QRecord().withValue("id", provider).withValue("enabled", false)));
|
||||
}
|
||||
CdnTestSupport.insert(CdnTables.PURGE_REQUEST, new QRecord().withValue("cdnProviderId", provider).withValue("status", "SKIPPED_NONE_PROVIDER"));
|
||||
assertThat(render(CdnWidgets.HOME_WIDGET)).contains("No CDN provider configured: purges are recorded only.")
|
||||
.contains("The CloudFront distribution is created in the cutover window.").contains("recorded only (not sent)")
|
||||
.contains("No origin check yet.");
|
||||
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN provider configured: purges are recorded only.");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.checks;
|
||||
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.net.InetAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.List;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The safety of the origin check's own HTTP code: which addresses it refuses
|
||||
** (every literal is parsed without DNS), that it connects to the address it
|
||||
** checked, and that it reads only a bounded status line and headers.
|
||||
*******************************************************************************/
|
||||
class CdnOriginCheckSafetyTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static boolean isPublic(String literal) throws Exception
|
||||
{
|
||||
return (CdnOriginCheck.isPublic(InetAddress.getByName(literal)));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Public unicast is allowed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void publicAddressesAreAllowed() throws Exception
|
||||
{
|
||||
for(String literal : List.of("93.184.216.34", "8.8.8.8", "100.63.255.255", "100.128.0.1", "172.15.0.1", "172.32.0.1", "198.17.0.1", "198.20.0.1", "2606:4700:4700::1111", "2001:4860:4860::8888"))
|
||||
{
|
||||
assertThat(isPublic(literal)).as(literal).isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Everything non-public is refused, including cloud metadata addresses and the
|
||||
** IPv6 forms that carry an IPv4 address.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void nonPublicAddressesAreRefused() throws Exception
|
||||
{
|
||||
for(String literal : List.of("127.0.0.1", "10.1.2.3", "172.16.0.1", "172.31.255.255", "192.168.1.1", "169.254.169.254", "100.64.0.1", "100.100.100.200", "100.127.255.255",
|
||||
"0.0.0.0", "0.1.2.3", "198.18.0.1", "198.19.255.255", "224.0.0.1", "255.255.255.255", "192.0.2.1", "203.0.113.9",
|
||||
"::1", "::", "fe80::1", "fc00::1", "fd00:ec2::254", "fdff::1", "ff02::1",
|
||||
"::ffff:127.0.0.1", "::ffff:169.254.169.254", "::ffff:10.0.0.1", "::127.0.0.1", "64:ff9b::7f00:1", "64:ff9b::a9fe:a9fe", "2002:7f00:1::1", "2002:a9fe:a9fe::1"))
|
||||
{
|
||||
assertThat(isPublic(literal)).as(literal).isFalse();
|
||||
}
|
||||
assertThat(isPublic("::ffff:8.8.8.8")).as("a mapped public address is judged by its IPv4 part").isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** IPv6 is an allowlist, built from raw bytes so the IPv6 path is really used (Java
|
||||
** turns ::ffff:a.b.c.d literals into IPv4 addresses).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ipv6IsAnAllowlistOfGlobalUnicast() throws Exception
|
||||
{
|
||||
for(String hex : List.of("00000000000000000000ffff7f000001", "00000000000000000000ffffa9fea9fe", "0000000000000000ffff00007f000001", "0064ff9b0000000000000000a9fea9fe",
|
||||
"0064ff9b0001000000000000" + "7f000001", "20010db8000000000000000000000001", "01000000000000000000000000000001", "20010000000000000000000000000001",
|
||||
"2002 7f00 0001 0000 0000 0000 0000 0001".replace(" ", ""), "fc000000000000000000000000000001", "fe800000000000000000000000000001", "ff020000000000000000000000000001",
|
||||
"00000000000000000000000000000001", "00000000000000000000000000000000", "40000000000000000000000000000001"))
|
||||
{
|
||||
assertThat(CdnOriginCheck.isPublicV6(bytes(hex))).as(hex).isFalse();
|
||||
}
|
||||
for(String hex : List.of("2606470047000000000000000000" + "1111", "20014860486000000000000000008888", "0064ff9b000000000000000008080808", "00000000000000000000ffff08080808",
|
||||
"2002080808080000000000000000" + "0001"))
|
||||
{
|
||||
assertThat(CdnOriginCheck.isPublicV6(bytes(hex))).as(hex).isTrue();
|
||||
}
|
||||
assertThat(CdnOriginCheck.isPublic(java.net.Inet6Address.getByAddress(bytes("20014860486000000000000000008888")))).isTrue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An origin that drips one byte at a time (here, never finishing the TLS handshake)
|
||||
** is cut off at the hard deadline, not after hours.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void aDrippingOriginIsCutOffAtTheDeadline() throws Exception
|
||||
{
|
||||
try(java.net.ServerSocket server = new java.net.ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")))
|
||||
{
|
||||
Thread dripper = new Thread(() ->
|
||||
{
|
||||
try(java.net.Socket client = server.accept())
|
||||
{
|
||||
for(int i = 0; i < 60; i++)
|
||||
{
|
||||
client.getOutputStream().write(0x16);
|
||||
client.getOutputStream().flush();
|
||||
Thread.sleep(300);
|
||||
}
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
// the client gave up
|
||||
}
|
||||
});
|
||||
dripper.setDaemon(true);
|
||||
dripper.start();
|
||||
long start = System.nanoTime();
|
||||
assertThatThrownBy(() -> new CdnOriginCheck.HttpFetcher().fetch(InetAddress.getByName("127.0.0.1"), server.getLocalPort(), "localhost", "/", java.time.Duration.ofSeconds(2)))
|
||||
.isInstanceOf(IOException.class);
|
||||
long seconds = (System.nanoTime() - start) / 1_000_000_000L;
|
||||
assertThat(seconds).as("gave up at about the 2 second deadline").isLessThan(6);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A port is stripped from name:port and [v6]:port, never from a bare IPv6 literal.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void stripsPortsOnlyWhenThereIsOne()
|
||||
{
|
||||
assertThat(CdnOriginCheck.stripPort("origin.example.com:443")).isEqualTo("origin.example.com");
|
||||
assertThat(CdnOriginCheck.stripPort("origin.example.com")).isEqualTo("origin.example.com");
|
||||
assertThat(CdnOriginCheck.stripPort("[2606:4700::1111]:443")).isEqualTo("2606:4700::1111");
|
||||
assertThat(CdnOriginCheck.stripPort("2606:4700::1111")).isEqualTo("2606:4700::1111");
|
||||
assertThat(CdnOriginCheck.stripPort("2606:4700:4700::1")).isEqualTo("2606:4700:4700::1");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Hex to 16 bytes.
|
||||
*******************************************************************************/
|
||||
private static byte[] bytes(String hex)
|
||||
{
|
||||
assertThat(hex).hasSize(32);
|
||||
byte[] out = new byte[16];
|
||||
for(int i = 0; i < 16; i++)
|
||||
{
|
||||
out[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16);
|
||||
}
|
||||
return (out);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One bad address in a name's answer refuses the whole name, and the address
|
||||
** used is one that was checked (the first).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void pickRefusesAnyBadAddressAndReturnsACheckedOne() throws Exception
|
||||
{
|
||||
InetAddress good = InetAddress.getByName("93.184.216.34");
|
||||
assertThat(CdnOriginCheck.pick(new InetAddress[] {good, InetAddress.getByName("8.8.8.8")})).isSameAs(good);
|
||||
assertThatThrownBy(() -> CdnOriginCheck.pick(new InetAddress[] {good, InetAddress.getByName("169.254.169.254")})).isInstanceOf(IOException.class).hasMessageContaining("non-public");
|
||||
assertThatThrownBy(() -> CdnOriginCheck.pick(new InetAddress[0])).isInstanceOf(IOException.class);
|
||||
assertThatThrownBy(() -> new CdnOriginCheck.HttpFetcher().fetch("localhost", "/")).isInstanceOf(IOException.class).hasMessageContaining("non-public");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Only the status line and headers are read, names are lower-cased, repeats are
|
||||
** joined, and the body is never touched.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void readsOnlyStatusAndHeaders() throws Exception
|
||||
{
|
||||
InputStream in = new ByteArrayInputStream("HTTP/1.1 200 OK\r\nCache-Control: public, max-age=60\r\nVary: a\r\nVary: b\r\n\r\nBODY".getBytes(StandardCharsets.US_ASCII));
|
||||
CdnOriginCheck.Response response = CdnOriginCheck.readResponse(in, System.nanoTime() + 1_000_000_000L);
|
||||
assertThat(response.status()).isEqualTo(200);
|
||||
assertThat(response.headers()).containsEntry("cache-control", "public, max-age=60").containsEntry("vary", "a, b");
|
||||
assertThat(in.read()).as("the body is left unread").isEqualTo('B');
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Malformed, oversized, truncated and too-slow responses are refused.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void refusesBadResponses() throws Exception
|
||||
{
|
||||
long deadline = System.nanoTime() + 1_000_000_000L;
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("garbage\r\n\r\n"), deadline)).isInstanceOf(IOException.class);
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 abc OK\r\n\r\n"), deadline)).isInstanceOf(IOException.class);
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\nA: b\r\n"), deadline)).hasMessageContaining("closed early");
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\nA: " + "x".repeat(9000) + "\r\n\r\n"), deadline)).hasMessageContaining("too long");
|
||||
StringBuilder many = new StringBuilder("HTTP/1.1 200 OK\r\n");
|
||||
for(int i = 0; i < 400; i++)
|
||||
{
|
||||
many.append("H").append(i).append(": ").append("v".repeat(60)).append("\r\n");
|
||||
}
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream(many.append("\r\n").toString()), deadline)).hasMessageContaining("too large");
|
||||
assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\n\r\n"), System.nanoTime() - 1)).hasMessageContaining("too slow");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static InputStream stream(String text)
|
||||
{
|
||||
return (new ByteArrayInputStream(text.getBytes(StandardCharsets.US_ASCII)));
|
||||
}
|
||||
}
|
||||
@@ -70,6 +70,16 @@ class CdnProcessesTest
|
||||
RunProcessOutput review = new RunProcessAction().execute(input);
|
||||
List<ProcessSummaryLineInterface> reviewLines = (List<ProcessSummaryLineInterface>) review.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
|
||||
for(ProcessSummaryLineInterface reviewLine : reviewLines)
|
||||
{
|
||||
ProcessSummaryLine summary = (ProcessSummaryLine) reviewLine;
|
||||
summary.pickMessage(false);
|
||||
if(summary.getMessage().contains("purges are recorded only"))
|
||||
{
|
||||
assertEquals(com.kingsrook.qqq.backend.core.model.actions.processes.Status.WARNING, summary.getStatus());
|
||||
}
|
||||
}
|
||||
|
||||
RunProcessInput go = new RunProcessInput();
|
||||
go.setProcessName(process);
|
||||
go.setProcessUUID(review.getProcessUUID());
|
||||
@@ -137,6 +147,7 @@ class CdnProcessesTest
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "SUCCEEDED"));
|
||||
List<List<String>> lines = run(CdnProcesses.RETRY_PURGE, rows.get(0).getValue("id") + "," + rows.get(1).getValue("id"));
|
||||
assertThat(lines.get(0)).containsExactly("1 purge request will be sent again", "1 purge request has not failed (no change)");
|
||||
assertThat(lines.get(1)).contains("1 purge request was queued again");
|
||||
QRecord retried = purges().get(0);
|
||||
assertEquals("PENDING", retried.getValueString("status"));
|
||||
assertEquals(0, retried.getValueInteger("attempts"));
|
||||
@@ -155,9 +166,99 @@ class CdnProcessesTest
|
||||
CdnTestSupport.insert(CdnTables.PROVIDER, new QRecord().withValue("name", "other").withValue("providerType", "CLOUDFRONT").withValue("enabled", true));
|
||||
List<List<String>> lines = run(CdnProcesses.PURGE_HOSTNAME, "1");
|
||||
assertThat(lines.get(0)).containsExactly("1 CDN distribution will be purged completely (every hostname on it)");
|
||||
assertThat(lines.get(1)).containsExactly("1 CDN distribution purge was recorded");
|
||||
List<QRecord> rows = purges();
|
||||
assertThat(rows).hasSize(1);
|
||||
assertEquals("ALL", rows.get(0).getValueString("kind"));
|
||||
assertEquals(1, rows.get(0).getValueInteger("cdnProviderId"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every action reports NONE and OFF honestly, including plural results.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void recordedOnlySummaries() throws Exception
|
||||
{
|
||||
for(boolean none : List.of(true, false))
|
||||
{
|
||||
for(String process : List.of(CdnProcesses.PURGE_PREFIX + "post", CdnProcesses.PURGE_HOSTNAME, CdnProcesses.RETRY_PURGE))
|
||||
{
|
||||
setUp();
|
||||
String table = none ? CdnTables.PROVIDER : CdnTables.HOSTNAME;
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(table)
|
||||
.withRecord(new QRecord().withValue("id", 1).withValue(none ? "providerType" : "stage", none ? "NONE" : "OFF")));
|
||||
String ids = "1";
|
||||
if(process.equals(CdnProcesses.PURGE_PREFIX + "post"))
|
||||
{
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "one"), new QRecord().withValue("slug", "two"));
|
||||
ids = "1,2";
|
||||
}
|
||||
else if(process.equals(CdnProcesses.RETRY_PURGE))
|
||||
{
|
||||
CdnTestSupport.insert(CdnTables.PURGE_REQUEST,
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "FAILED"),
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "DEAD"));
|
||||
ids = "1,2";
|
||||
}
|
||||
List<List<String>> lines = run(process, ids);
|
||||
String warning = none ? "No CDN provider configured" : "CDN hostnames are not live";
|
||||
assertThat(lines.get(0)).anyMatch(line -> line.contains(warning + ": purges are recorded only, not sent."));
|
||||
assertThat(lines.get(0)).noneMatch(line -> line.contains("will be purged") || line.contains("will be sent again"));
|
||||
String reason = none ? "no CDN provider" : "CDN hostnames are not live";
|
||||
assertThat(lines.get(1)).containsExactly(process.equals(CdnProcesses.PURGE_HOSTNAME)
|
||||
? "1 purge request was recorded (not sent: " + reason + ")"
|
||||
: "2 purge requests were recorded (not sent: " + reason + ")");
|
||||
assertThat(purges()).allMatch(row -> (none ? "SKIPPED_NONE_PROVIDER" : "SKIPPED_STAGE_OFF").equals(row.getValueString("status")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Eligibility considers enabled providers and their purge-enabled hosts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void sendingEligibility() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_NONE_PROVIDER,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
CdnTestSupport.liveProvider(CdnProviderType.NONE, null);
|
||||
Integer live = CdnTestSupport.liveProvider(CdnProviderType.CLOUDFRONT, null);
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.PENDING,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.HOSTNAME)
|
||||
.withRecord(new QRecord().withValue("id", 2).withValue("purgeEnabled", false)));
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_STAGE_OFF,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(live));
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.PROVIDER)
|
||||
.withRecord(new QRecord().withValue("id", live).withValue("enabled", false)));
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_NONE_PROVIDER,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without an enabled provider, do not claim a request row was recorded.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void absentProvider() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
CdnTestSupport.postRule();
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "one"));
|
||||
List<List<String>> lines = run(CdnProcesses.PURGE_PREFIX + "post", "1");
|
||||
assertThat(lines.get(0)).contains("1 No CDN provider configured: purges are recorded only, not sent.");
|
||||
assertThat(lines.get(1)).containsExactly("1 record had no enabled CDN provider (no purge request recorded)");
|
||||
assertThat(purges()).isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -17,23 +17,25 @@
|
||||
package com.kingsrook.qbits.cdn.purge;
|
||||
|
||||
|
||||
import java.net.http.HttpClient;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport.FakePathProvider;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport.FakeTagProvider;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnException;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnPurgeKind;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
@@ -46,7 +48,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
*******************************************************************************/
|
||||
class PurgePipelineTest
|
||||
{
|
||||
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of()));
|
||||
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(name -> null, host -> List.of()));
|
||||
|
||||
|
||||
|
||||
@@ -139,6 +141,32 @@ class PurgePipelineTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record-change listener skips counter updates but records title changes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ignoredCounterChangeRecordsNoPurge() throws Exception
|
||||
{
|
||||
QContext.getQInstance().getTable("post").withField(new QFieldMetaData("viewCount", QFieldType.INTEGER));
|
||||
CdnTestSupport.liveProvider(CdnProviderType.CLOUDFRONT, null);
|
||||
QRecord rule = new QueryAction().execute(new QueryInput(CdnTables.TABLE_TAG_RULE)).getRecords().get(0);
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.TABLE_TAG_RULE).withRecord(new QRecord().withValue("id", rule.getValue("id"))
|
||||
.withValue("onlyWhenFieldsChange", "!viewCount")));
|
||||
QRecord post = CdnTestSupport.insert("post", new QRecord().withValue("slug", "counter-post").withValue("title", "Old")
|
||||
.withValue("blogId", 1).withValue("isPublished", true).withValue("viewCount", 10)).get(0);
|
||||
assertThat(purges()).hasSize(1);
|
||||
|
||||
new UpdateAction().execute(new UpdateInput("post").withRecord(new QRecord().withValue("id", post.getValue("id")).withValue("viewCount", 11)));
|
||||
assertThat(purges()).as("counter-only update adds no purge request").hasSize(1);
|
||||
|
||||
new UpdateAction().execute(new UpdateInput("post").withRecord(new QRecord().withValue("id", post.getValue("id")).withValue("title", "New")));
|
||||
assertThat(purges()).hasSize(2);
|
||||
assertThat(purges().get(1).getValueString("paths")).contains("/blog/quick-bits/counter-post*");
|
||||
assertThat(purges().get(1).getValueString("tags")).contains("post:" + post.getValue("id"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Retryable failures back off and end DEAD; permanent ones go DEAD at once.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -18,15 +18,19 @@ package com.kingsrook.qbits.cdn.purge;
|
||||
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.http.HttpClient;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerificationException;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerifier;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -46,6 +50,7 @@ class RevalidationHookRunnerTest
|
||||
private final List<String> signatures = new CopyOnWriteArrayList<>();
|
||||
private final AtomicInteger status = new AtomicInteger(200);
|
||||
private HttpServer server;
|
||||
private final List<Map<String, String>> headers = new CopyOnWriteArrayList<>();
|
||||
|
||||
|
||||
|
||||
@@ -59,6 +64,9 @@ class RevalidationHookRunnerTest
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/api/revalidate", exchange ->
|
||||
{
|
||||
Map<String, String> received = new LinkedHashMap<>();
|
||||
exchange.getRequestHeaders().forEach((key, values) -> received.put(key.toLowerCase(java.util.Locale.ROOT), values.get(0)));
|
||||
headers.add(received);
|
||||
bodies.add(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
|
||||
String signature = exchange.getRequestHeaders().getFirst(RevalidationHookRunner.SIGNATURE_HEADER);
|
||||
signatures.add(signature == null ? "none" : signature);
|
||||
@@ -88,13 +96,21 @@ class RevalidationHookRunnerTest
|
||||
void signsAndFansOut() throws Exception
|
||||
{
|
||||
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true).withValue("fanOutDns", true)
|
||||
.withValue("secretRef", "SECRET").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> "SECRET".equals(name) ? "s3cret" : null,
|
||||
.withValue("secretRef", "SECRET").withValue("allowPlainHttp", true).withValue("internalAllowlist", "frontend.invalid").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "SECRET".equals(name) ? "test-secret-at-least-16-bytes" : null,
|
||||
host -> List.of("127.0.0.1", "127.0.0.1"));
|
||||
assertEquals(2, runner.run(List.of("post:1")));
|
||||
assertThat(bodies).hasSize(2).allMatch("{\"tags\":[\"post:1\"]}"::equals);
|
||||
String expected = "sha256=" + RevalidationHookRunner.sign("s3cret", bodies.get(0).getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(signatures).allMatch(expected::equals);
|
||||
WebhookVerifier verifier = new WebhookVerifier(List.of(WebhookSecret.parse("test-secret-at-least-16-bytes")));
|
||||
for(int i = 0; i < bodies.size(); i++)
|
||||
{
|
||||
verifier.verify(headers.get(i), bodies.get(i).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
assertThat(signatures).containsExactly("none", "none");
|
||||
assertThat(headers.get(0).get("webhook-id")).isEqualTo(headers.get(1).get("webhook-id"));
|
||||
assertThat(headers.get(0).get("host")).startsWith("frontend.invalid:");
|
||||
assertThatThrownBy(() -> verifier.verify(headers.get(0), "tampered".getBytes(StandardCharsets.UTF_8)))
|
||||
.isInstanceOf(WebhookVerificationException.class);
|
||||
assertEquals(0, runner.run(List.of()));
|
||||
}
|
||||
|
||||
@@ -108,12 +124,94 @@ class RevalidationHookRunnerTest
|
||||
{
|
||||
status.set(500);
|
||||
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true)
|
||||
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of());
|
||||
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1").withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.run(List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("500");
|
||||
assertThat(signatures).containsExactly("none");
|
||||
|
||||
QRecord fanOut = new QRecord().withValue("name", "empty").withValue("fanOutDns", true).withValue("url", "http://x.invalid/api/revalidate");
|
||||
assertThatThrownBy(() -> runner.call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
|
||||
assertThatThrownBy(() -> new RevalidationHookRunner(name -> null, host -> List.of()).call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Legacy mode preserves fixed signatures, including whitespace and whsec_ keys.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void legacyGoldenVectors() throws Exception
|
||||
{
|
||||
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
|
||||
assertThat(signatures.get(0)).isEqualTo("sha256=540457336a45ef962b165f8732f71ebf327ffa67c0cb1047c446df0231fa0db2");
|
||||
new RevalidationHookRunner(name -> " whsec_test-key-with-spaces ", host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
|
||||
assertThat(signatures.get(1)).isEqualTo("sha256=a4dd022bffe6f799633102c415fdc18e22f078bfc98357f909b5df58d5318b05");
|
||||
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
|
||||
.call(hook(), List.of("post:1"));
|
||||
assertThat(signatures.get(2)).as("a null format (existing rows) keeps the legacy signature").isEqualTo(signatures.get(0));
|
||||
assertThat(bodies).containsExactly("{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}");
|
||||
assertThat(headers).allSatisfy(value -> assertThat(value).doesNotContainKeys("webhook-id", "webhook-timestamp", "webhook-signature"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An explicit standard format accepts encoded secrets.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void explicitStandardFormat() throws Exception
|
||||
{
|
||||
WebhookSecret secret = WebhookSecret.generate();
|
||||
new RevalidationHookRunner(name -> secret.reveal(), host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"));
|
||||
new WebhookVerifier(List.of(secret)).verify(headers.get(0), bodies.get(0).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalid format and short standard secrets fail before sending.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void invalidSigningConfiguration()
|
||||
{
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "short", host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("at least 16 bytes");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "UNKNOWN"), List.of("post:1"))).isInstanceOf(QException.class);
|
||||
assertThat(bodies).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Safe defaults reject HTTP, private addresses and mixed fan-out before sending.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void safeHttpPolicy()
|
||||
{
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("allowPlainHttp", null), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Only https");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("internalAllowlist", null), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
|
||||
RevalidationHookRunner mixed = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1", "169.254.169.254"));
|
||||
assertThatThrownBy(() -> mixed.call(hook().withValue("fanOutDns", true), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("timeoutMs", 0), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("positive");
|
||||
assertThat(bodies).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A local test hook explicitly opts into HTTP and loopback.
|
||||
*******************************************************************************/
|
||||
private QRecord hook()
|
||||
{
|
||||
return (new QRecord().withValue("name", "test").withValue("secretRef", "SECRET")
|
||||
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1")
|
||||
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
}
|
||||
}
|
||||
@@ -135,6 +135,67 @@ class TagRuleEngineTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Ignore-only rules compare actual values after merging partial updates.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void ignoresCounterOnlyUpdates() throws Exception
|
||||
{
|
||||
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", " !viewCount, !lastIndexedAt ").withValue("tagTemplates", "post:{id}");
|
||||
QRecord old = new QRecord().withValue("id", 1).withValue("title", "Old").withValue("viewCount", 10).withValue("lastIndexedAt", "yesterday");
|
||||
TagRuleEngine engine = new TagRuleEngine();
|
||||
assertTrue(engine.evaluate(rule, List.of(new QRecord(old)), List.of(old)).isEmpty());
|
||||
assertTrue(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("viewCount", 11).withValue("lastIndexedAt", "today")), List.of(old)).isEmpty());
|
||||
assertTrue(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", "Old")), List.of(old)).isEmpty());
|
||||
assertThat(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", "New").withValue("viewCount", 11)), List.of(old)).tags()).containsExactly("post:1");
|
||||
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("title", null)), List.of(old)).isEmpty());
|
||||
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("newField", "value")), List.of(old)).isEmpty());
|
||||
assertFalse(engine.evaluate(rule, List.of(new QRecord().withValue("id", 1).withValue("ViewCount", 11)), List.of(old)).isEmpty());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Plain lists retain allow-list behavior; ignored entries always win.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void appliesAllowListAndMixedRules()
|
||||
{
|
||||
QRecord old = new QRecord().withValue("title", "Old").withValue("viewCount", 10);
|
||||
QRecord counterChange = new QRecord(old).withValue("viewCount", 11);
|
||||
QRecord titleChange = new QRecord(old).withValue("title", "New");
|
||||
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", " title, ,slug ");
|
||||
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
|
||||
assertTrue(TagRuleEngine.counts(rule, titleChange, old));
|
||||
assertFalse(TagRuleEngine.counts(rule, new QRecord(old), old));
|
||||
rule.setValue("onlyWhenFieldsChange", "title, viewCount, !viewCount");
|
||||
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
|
||||
assertTrue(TagRuleEngine.counts(rule, titleChange, old));
|
||||
assertFalse(TagRuleEngine.counts(rule, new QRecord(old).withValue("slug", "new"), old));
|
||||
rule.setValue("onlyWhenFieldsChange", "viewCount,!viewCount");
|
||||
assertFalse(TagRuleEngine.counts(rule, titleChange, old));
|
||||
assertFalse(TagRuleEngine.counts(rule, counterChange, old));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Changed-field filters never suppress inserts or deletes.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void insertsAndDeletesIgnoreFieldFilters() throws Exception
|
||||
{
|
||||
QRecord post = new QRecord().withValue("id", 1).withValue("viewCount", 10);
|
||||
for(String fields : List.of("!viewCount", "title", "title,!viewCount"))
|
||||
{
|
||||
QRecord rule = new QRecord().withValue("onlyWhenFieldsChange", fields).withValue("tagTemplates", "post:{id}");
|
||||
assertThat(new TagRuleEngine().evaluate(rule, List.of(post), List.of()).tags()).containsExactly("post:1");
|
||||
assertThat(new TagRuleEngine().evaluate(rule, List.of(), List.of(post)).tags()).containsExactly("post:1");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Stored text round-trips; invalid lines are dropped.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.tables;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.app.CdnSetupStatus;
|
||||
import com.kingsrook.qbits.cdn.checks.CdnOriginCheck;
|
||||
import com.kingsrook.qbits.cdn.headers.CdnResponseHeaderPolicy;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses;
|
||||
import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The "Create default ..." table actions (insert-only, change nothing that is
|
||||
** there) and the origin header check (the writer of cdn_origin_check, which had
|
||||
** none): what it counts, what it records, and the action and nightly entry points.
|
||||
*******************************************************************************/
|
||||
class CdnDefaultsAndOriginCheckTest
|
||||
{
|
||||
private final Map<String, CdnOriginCheck.Response> origin = new LinkedHashMap<>();
|
||||
private final List<String> asked = new java.util.ArrayList<>();
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
CdnOriginCheck.setFetcher((host, path) ->
|
||||
{
|
||||
asked.add(host + path);
|
||||
CdnOriginCheck.Response response = origin.get(path);
|
||||
if(response == null)
|
||||
{
|
||||
throw (new IOException("connection refused"));
|
||||
}
|
||||
return (response);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@AfterEach
|
||||
void tearDown()
|
||||
{
|
||||
CdnOriginCheck.setFetcher(null);
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Run a table action through Review and then Result; returns the result lines.
|
||||
*******************************************************************************/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static List<String> run(String process, String recordIds) throws Exception
|
||||
{
|
||||
RunProcessInput input = new RunProcessInput();
|
||||
input.setProcessName(process);
|
||||
if(recordIds != null)
|
||||
{
|
||||
input.addValue("recordsParam", "recordIds");
|
||||
input.addValue("recordIds", recordIds);
|
||||
}
|
||||
input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK);
|
||||
RunProcessOutput review = new RunProcessAction().execute(input);
|
||||
RunProcessInput go = new RunProcessInput();
|
||||
go.setProcessName(process);
|
||||
go.setProcessUUID(review.getProcessUUID());
|
||||
go.setStartAfterStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW);
|
||||
go.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP);
|
||||
RunProcessOutput result = new RunProcessAction().execute(go);
|
||||
List<ProcessSummaryLineInterface> lines = (List<ProcessSummaryLineInterface>) result.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY);
|
||||
return (lines == null ? List.of() : lines.stream().map(line ->
|
||||
{
|
||||
ProcessSummaryLine summary = (ProcessSummaryLine) line;
|
||||
summary.pickMessage(true);
|
||||
return (summary.getCount() + " " + summary.getMessage());
|
||||
}).toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static List<QRecord> rows(String table) throws Exception
|
||||
{
|
||||
return (new QueryAction().execute(new QueryInput(table)).getRecords());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create default provider adds No CDN once; and with it a record change is
|
||||
** recorded as a would-be purge (with no provider it is not recorded at all).
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void defaultProviderMakesPurgesRecorded() throws Exception
|
||||
{
|
||||
CdnTestSupport.postRule();
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "a").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true));
|
||||
assertThat(rows(CdnTables.PURGE_REQUEST)).as("no provider: nothing is recorded").isEmpty();
|
||||
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_PROVIDER, null)).containsExactly("1 default row was added");
|
||||
assertThat(rows(CdnTables.PROVIDER)).singleElement().satisfies(p ->
|
||||
{
|
||||
assertThat(p.getValueString("name")).isEqualTo(CdnDefaultsProcesses.NONE_PROVIDER_NAME);
|
||||
assertThat(p.getValueString("providerType")).isEqualTo("NONE");
|
||||
assertThat(p.getValueBoolean("enabled")).isTrue();
|
||||
});
|
||||
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "b").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true));
|
||||
assertThat(rows(CdnTables.PURGE_REQUEST)).singleElement().satisfies(r -> assertThat(r.getValueString("status")).isEqualTo("SKIPPED_NONE_PROVIDER"));
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// insert-only: again changes nothing, and an edited row is left alone //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.PROVIDER).withRecord(new QRecord().withValue("id", rows(CdnTables.PROVIDER).get(0).getValue("id")).withValue("name", "Ours")));
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_PROVIDER, null)).containsExactly("null Nothing to add: the defaults are already there.");
|
||||
assertThat(rows(CdnTables.PROVIDER)).singleElement().satisfies(p -> assertThat(p.getValueString("name")).isEqualTo("Ours"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create default policies adds two turned-off policies (so nothing changes until
|
||||
** one is turned on), by name; an edited policy survives a second run; turning one
|
||||
** on makes the header decision change.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void defaultPoliciesAreOffUntilTurnedOn() throws Exception
|
||||
{
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_POLICIES, null)).containsExactly("2 default rows were added");
|
||||
List<QRecord> policies = rows(CdnTables.CACHE_POLICY);
|
||||
assertThat(policies).hasSize(2).allSatisfy(p -> assertThat(p.getValueBoolean("enabled")).isFalse());
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
assertThat(CdnResponseHeaderPolicy.current()).as("turned-off policies change no response").isEmpty();
|
||||
|
||||
QRecord assets = policies.stream().filter(p -> CdnDefaultsProcesses.ASSETS_POLICY_NAME.equals(p.getValueString("name"))).findFirst().orElseThrow();
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.CACHE_POLICY).withRecord(new QRecord().withValue("id", assets.getValue("id")).withValue("enabled", true).withValue("browserMaxAge", 600)));
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
CdnResponseHeaderPolicy.Decision decision = CdnResponseHeaderPolicy.decide(new CdnResponseHeaderPolicy.Request("www.example.com", "/assets/app.js", null, null, false, false, List.of(), List.of()), null);
|
||||
assertThat(decision.policyName()).isEqualTo(CdnDefaultsProcesses.ASSETS_POLICY_NAME);
|
||||
assertThat(decision.cacheControl()).startsWith("public, max-age=600").endsWith("immutable");
|
||||
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_POLICIES, null)).containsExactly("null Nothing to add: the defaults are already there.");
|
||||
assertThat(rows(CdnTables.CACHE_POLICY)).hasSize(2).anySatisfy(p -> assertThat(p.getValueInteger("browserMaxAge")).isEqualTo(600));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Create default rules adds a tags-only rule for each purgeable table without
|
||||
** one, and the rule makes a change record tags.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void defaultRulesCoverPurgeableTables() throws Exception
|
||||
{
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_RULES, null)).containsExactly("1 default row was added");
|
||||
assertThat(rows(CdnTables.TABLE_TAG_RULE)).singleElement().satisfies(r ->
|
||||
{
|
||||
assertThat(r.getValueString("tableName")).isEqualTo("post");
|
||||
assertThat(r.getValueString("tagTemplates")).isEqualTo("post:{id}\npost");
|
||||
assertThat(r.getValueBoolean("enabled")).isTrue();
|
||||
});
|
||||
CdnTestSupport.insert(CdnTables.PROVIDER, new QRecord().withValue("name", "none").withValue("providerType", "NONE").withValue("enabled", true));
|
||||
QRecord post = CdnTestSupport.insert("post", new QRecord().withValue("slug", "a").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true)).get(0);
|
||||
assertThat(rows(CdnTables.PURGE_REQUEST)).singleElement().satisfies(r -> assertThat(r.getValueString("tags")).contains("post:" + post.getValue("id")));
|
||||
|
||||
assertThat(run(CdnDefaultsProcesses.CREATE_RULES, null)).containsExactly("null Nothing to add: the defaults are already there.");
|
||||
assertThat(rows(CdnTables.TABLE_TAG_RULE)).hasSize(1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The origin check: a good page, a page with no Cache-Control, a page whose
|
||||
** Cache-Control differs from its policy, a 404 and an unreachable page.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void originCheckCountsRightWrongAndUnreadable() throws Exception
|
||||
{
|
||||
Integer provider = CdnTestSupport.liveProvider(com.kingsrook.qbits.cdn.spi.CdnProviderType.CLOUDFRONT, null);
|
||||
new UpdateAction().execute(new UpdateInput(CdnTables.PROVIDER).withRecord(new QRecord().withValue("id", provider).withValue("originHost", "origin.example.com:443")));
|
||||
CdnTestSupport.insert(CdnTables.CACHE_POLICY,
|
||||
new QRecord().withValue("name", "blog").withValue("matchType", "PREFIX").withValue("matchValue", "/blog").withValue("policyClass", "LEAF").withValue("browserMaxAge", 60).withValue("enabled", true),
|
||||
new QRecord().withValue("name", "news").withValue("matchType", "PREFIX").withValue("matchValue", "/news").withValue("policyClass", "LEAF").withValue("browserMaxAge", 60).withValue("enabled", true));
|
||||
CdnTestSupport.insert(CdnTables.TABLE_TAG_RULE, new QRecord().withValue("tableName", "post").withValue("enabled", true)
|
||||
.withValue("invalidationPaths", "/about\n/blog/{slug}*\nhub:/blog"));
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
|
||||
origin.put("/", new CdnOriginCheck.Response(200, Map.of("cache-control", "max-age=5")));
|
||||
origin.put("/blog", new CdnOriginCheck.Response(200, Map.of("cache-control", "Public, max-age=60")));
|
||||
origin.put("/news", new CdnOriginCheck.Response(200, Map.of("cache-control", "public, max-age=999")));
|
||||
origin.put("/about", new CdnOriginCheck.Response(200, Map.of()));
|
||||
CdnTestSupport.insert(CdnTables.CACHE_POLICY, new QRecord().withValue("name", "gone").withValue("matchType", "PREFIX").withValue("matchValue", "/gone").withValue("policyClass", "LEAF").withValue("enabled", true),
|
||||
new QRecord().withValue("name", "down").withValue("matchType", "PREFIX").withValue("matchValue", "/down").withValue("policyClass", "LEAF").withValue("enabled", true));
|
||||
CdnResponseHeaderPolicy.clearCache();
|
||||
origin.put("/gone", new CdnOriginCheck.Response(404, Map.of()));
|
||||
|
||||
QRecord hostname = rows(CdnTables.HOSTNAME).get(0);
|
||||
CdnOriginCheck.Result result = CdnOriginCheck.checkAndRecord(hostname);
|
||||
|
||||
assertThat(asked).as("the provider's origin host (without a port) is asked, not the public name").allMatch(url -> url.startsWith("origin.example.com/"));
|
||||
assertThat(result.checked()).isEqualTo(6);
|
||||
assertThat(result.correct()).as("/ (any Cache-Control, no policy) and /blog (equals its policy, ignoring case)").isEqualTo(2);
|
||||
assertThat(result.wrong()).as("/about has no header; /news differs from its policy").isEqualTo(2);
|
||||
assertThat(result.unreadable()).as("/gone answered 404; /down is unreachable").isEqualTo(2);
|
||||
assertThat(result.problems()).anyMatch(p -> p.startsWith("/about: no Cache-Control")).anyMatch(p -> p.startsWith("/news") && p.contains("public, max-age=60"))
|
||||
.anyMatch(p -> p.startsWith("/gone: the origin answered 404")).anyMatch(p -> p.startsWith("/down: could not be read"));
|
||||
|
||||
assertThat(rows(CdnTables.ORIGIN_CHECK)).singleElement().satisfies(row ->
|
||||
{
|
||||
assertThat(row.getValueString("scope")).isEqualTo("www.example.com");
|
||||
assertThat(row.getValueInteger("objectsChecked")).isEqualTo(6);
|
||||
assertThat(row.getValueInteger("correct")).isEqualTo(2);
|
||||
assertThat(row.getValueInteger("missingOrWrong")).isEqualTo(2);
|
||||
assertThat(row.getValueInteger("unreadable")).isEqualTo(2);
|
||||
assertThat(row.getValue("checkDate")).isNotNull();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sample paths are bounded and only literal paths are used.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void samplePathsAreLiteralAndBounded() throws Exception
|
||||
{
|
||||
StringBuilder paths = new StringBuilder("/blog/{slug}*");
|
||||
for(int i = 0; i < 40; i++)
|
||||
{
|
||||
paths.append("\n/page-").append(i);
|
||||
}
|
||||
CdnTestSupport.insert(CdnTables.TABLE_TAG_RULE, new QRecord().withValue("tableName", "post").withValue("enabled", true).withValue("invalidationPaths", paths.toString()));
|
||||
List<String> sample = CdnOriginCheck.samplePaths("www.example.com");
|
||||
assertThat(sample).hasSize(CdnOriginCheck.MAX_PATHS).startsWith("/", "/page-0").noneMatch(p -> p.contains("{") || p.contains("*"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The record action writes a row per selected hostname; the nightly entry point
|
||||
** skips hostnames that are Off.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void actionAndNightlyRun() throws Exception
|
||||
{
|
||||
CdnTestSupport.liveProvider(com.kingsrook.qbits.cdn.spi.CdnProviderType.CLOUDFRONT, null);
|
||||
CdnTestSupport.insert(CdnTables.HOSTNAME, new QRecord().withValue("hostname", "off.example.com").withValue("cdnProviderId", 1).withValue("stage", "OFF"));
|
||||
origin.put("/", new CdnOriginCheck.Response(200, Map.of("cache-control", "max-age=5")));
|
||||
|
||||
List<String> lines = run(CdnOriginCheckProcesses.CHECK_ORIGIN, "1");
|
||||
assertThat(lines).containsExactly("1 hostname has the right headers");
|
||||
assertThat(rows(CdnTables.ORIGIN_CHECK)).hasSize(1);
|
||||
|
||||
new CdnOriginCheckProcesses.ScheduledStep().run(new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput(), new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput());
|
||||
assertThat(rows(CdnTables.ORIGIN_CHECK)).extracting(r -> r.getValueString("scope")).containsExactly("www.example.com", "www.example.com");
|
||||
|
||||
CdnRuntime.setClock(null);
|
||||
assertThat(CdnSetupStatus.load()).anySatisfy(item ->
|
||||
{
|
||||
assertThat(item.tableName()).isEqualTo(CdnTables.ORIGIN_CHECK);
|
||||
assertThat(item.state()).isEqualTo(CdnSetupStatus.State.OK);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.tables;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.app.CdnSetupStatus;
|
||||
import com.kingsrook.qbits.cdn.app.CdnSetupStatus.Item;
|
||||
import com.kingsrook.qbits.cdn.app.CdnSetupStatus.State;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The Setup card of the CDN home: empty, on the No CDN default, and fully set up.
|
||||
*******************************************************************************/
|
||||
class CdnSetupStatusTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private static Item item(List<Item> items, String label)
|
||||
{
|
||||
return (items.stream().filter(i -> i.label().equals(label)).findFirst().orElseThrow());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Nothing set up: the next steps name the Create default actions, and the home shows them.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void emptySetupPointsAtTheDefaults() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
List<Item> items = CdnSetupStatus.load();
|
||||
assertThat(item(items, "Providers").state()).isEqualTo(State.TODO);
|
||||
assertThat(item(items, "Providers").nextStep()).contains("Create default provider");
|
||||
assertThat(item(items, "Tag rules").state()).isEqualTo(State.TODO);
|
||||
assertThat(item(items, "Tag rules").nextStep()).contains("Create default rules");
|
||||
assertThat(item(items, "Cache policies").nextStep()).contains("Create default policies");
|
||||
assertThat(items).extracting(Item::tableName).contains(CdnTables.PROVIDER, CdnTables.HOSTNAME, CdnTables.CACHE_POLICY, CdnTables.TABLE_TAG_RULE,
|
||||
CdnTables.REVALIDATION_HOOK, CdnTables.ORIGIN_CHECK);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A real CDN, a Live hostname, a policy that is on, a rule for each table and a hook.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void aCompleteSetupIsOk()
|
||||
{
|
||||
List<QRecord> providers = List.of(new QRecord().withValue("providerType", "CLOUDFRONT").withValue("enabled", true).withValue("lastHealthStatus", "OK"));
|
||||
List<QRecord> hostnames = List.of(new QRecord().withValue("stage", "LIVE").withValue("purgeEnabled", true));
|
||||
List<QRecord> policies = List.of(new QRecord().withValue("enabled", true));
|
||||
List<QRecord> rules = List.of(new QRecord().withValue("tableName", "post").withValue("enabled", true));
|
||||
List<QRecord> hooks = List.of(new QRecord().withValue("enabled", true));
|
||||
List<Item> items = CdnSetupStatus.build(providers, hostnames, policies, rules, hooks, 1, List.of("post"), true);
|
||||
assertThat(items).extracting(Item::state).containsOnly(State.OK);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The softer states: No CDN only, hostnames not Live, a failing provider, a table without a rule.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void softerStatesAreNamed()
|
||||
{
|
||||
QRecord none = new QRecord().withValue("providerType", "NONE").withValue("enabled", true);
|
||||
List<Item> items = CdnSetupStatus.build(List.of(none), List.of(new QRecord().withValue("stage", "OFF")), List.of(new QRecord().withValue("enabled", false)),
|
||||
List.of(), List.of(), 0, List.of("post", "page"), false);
|
||||
assertThat(item(items, "Providers").state()).isEqualTo(State.DEFAULT);
|
||||
assertThat(item(items, "Hostnames").state()).isEqualTo(State.DEFAULT);
|
||||
assertThat(item(items, "Cache policies").state()).isEqualTo(State.DEFAULT);
|
||||
assertThat(item(items, "Tag rules").detail()).contains("post, page");
|
||||
assertThat(item(items, "Origin checks").nextStep()).contains("No scheduler is configured");
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
// like the runtime: Live needs purge enabled, and a null enabled counts as off //
|
||||
///////////////////////////////////////////////////////////////////////////////////
|
||||
List<Item> unpurgeable = CdnSetupStatus.build(List.of(new QRecord().withValue("providerType", "CLOUDFRONT")), List.of(new QRecord().withValue("stage", "LIVE").withValue("purgeEnabled", false)),
|
||||
List.of(), List.of(), List.of(), 0, List.of(), true);
|
||||
assertThat(item(unpurgeable, "Providers").state()).isEqualTo(State.TODO);
|
||||
assertThat(item(unpurgeable, "Hostnames").state()).isEqualTo(State.DEFAULT);
|
||||
assertThat(item(unpurgeable, "Origin checks").nextStep()).contains("every night");
|
||||
|
||||
QRecord failing = new QRecord().withValue("providerType", "FASTLY").withValue("enabled", true).withValue("lastHealthStatus", "FAILED");
|
||||
assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), 0, List.of(), false), "Providers").state()).isEqualTo(State.TODO);
|
||||
assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), 0, List.of(), false), "Hostnames").state()).isEqualTo(State.TODO);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.tables;
|
||||
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Stream;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnHelp;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses;
|
||||
import com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Guardrail (owner requirement 2026-10-14: "all tables should be populated and
|
||||
** the qbit should be fully using them"). For every cdn_* table:
|
||||
** - runtime code (not just its model, metadata or home screen) reads or writes it;
|
||||
** - it is listed here with why it starts empty and, where there is one, the table
|
||||
** action that sets it up (this qbit ships no DDL and a host chooses its column
|
||||
** names, so defaults are actions, not a database seed);
|
||||
** - it has bundle help that says what it is for and how its rows get created
|
||||
** (shown on the table screen and when it is empty).
|
||||
** A new table fails this test until it is wired and explained.
|
||||
*******************************************************************************/
|
||||
class CdnTablesGuardrailTest
|
||||
{
|
||||
private static final Path MAIN = Path.of("src", "main", "java", "com", "kingsrook", "qbits", "cdn");
|
||||
|
||||
/*******************************************************************************
|
||||
** Why each table starts empty, and where its rows come from.
|
||||
*******************************************************************************/
|
||||
static final Map<String, String> STARTS_EMPTY = Map.ofEntries(
|
||||
Map.entry(CdnTables.PROVIDER, "host connection config: the Create default provider action adds No CDN; a real CDN needs the host's account and key variable"),
|
||||
Map.entry(CdnTables.HOSTNAME, "host-specific: the hostnames the host serves"),
|
||||
Map.entry(CdnTables.CACHE_POLICY, "the Create default policies action adds two turned-off examples; turning one on is the host's decision"),
|
||||
Map.entry(CdnTables.TABLE_TAG_RULE, "the Create default rules action adds a tags-only rule per purgeable table; paths are the host's"),
|
||||
Map.entry(CdnTables.PURGE_REQUEST, "runtime: a record change or a purge action"),
|
||||
Map.entry(CdnTables.REVALIDATION_HOOK, "host-specific and optional: a front end with its own cache"),
|
||||
Map.entry(CdnTables.EDGE_STAT_DAILY, "runtime: Import Edge Stats and the daily import"),
|
||||
Map.entry(CdnTables.ORIGIN_CHECK, "runtime: Check Origin Headers and the nightly check"),
|
||||
Map.entry(CdnTables.DRIFT_FINDING, "runtime: Check Config Drift and the nightly check"));
|
||||
|
||||
/*******************************************************************************
|
||||
** The table action that sets a table up, by table.
|
||||
*******************************************************************************/
|
||||
static final Map<String, String> SETUP_ACTIONS = Map.of(
|
||||
CdnTables.PROVIDER, CdnDefaultsProcesses.CREATE_PROVIDER,
|
||||
CdnTables.CACHE_POLICY, CdnDefaultsProcesses.CREATE_POLICIES,
|
||||
CdnTables.TABLE_TAG_RULE, CdnDefaultsProcesses.CREATE_RULES);
|
||||
|
||||
private QInstance qInstance;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void setUp() throws Exception
|
||||
{
|
||||
qInstance = CdnTestSupport.setUp();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table is used by runtime code.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableIsUsedAtRuntime() throws Exception
|
||||
{
|
||||
List<String> runtimeSource = new ArrayList<>();
|
||||
try(Stream<Path> files = Files.walk(MAIN))
|
||||
{
|
||||
for(Path file : files.filter(p -> p.toString().endsWith(".java")).filter(p -> !p.startsWith(MAIN.resolve("model")) && !p.startsWith(MAIN.resolve("app"))).toList())
|
||||
{
|
||||
runtimeSource.add(Files.readString(file, StandardCharsets.UTF_8));
|
||||
}
|
||||
}
|
||||
for(String table : cdnTables())
|
||||
{
|
||||
String constant = constantFor(table);
|
||||
Pattern use = Pattern.compile("\\bCdnTables\\." + constant + "\\b");
|
||||
assertThat(runtimeSource.stream().anyMatch(source -> use.matcher(source).find()))
|
||||
.as(table + " (CdnTables." + constant + ") must be read or written by runtime code, not only declared").isTrue();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table is explained, and the setup actions it names exist.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableIsExplained()
|
||||
{
|
||||
assertThat(cdnTables()).containsExactlyInAnyOrderElementsOf(STARTS_EMPTY.keySet());
|
||||
assertThat(CdnTables.ALL).containsExactlyInAnyOrderElementsOf(STARTS_EMPTY.keySet());
|
||||
SETUP_ACTIONS.forEach((table, action) ->
|
||||
{
|
||||
assertThat(qInstance.getProcess(action)).as(action).isNotNull();
|
||||
assertThat(qInstance.getProcess(action).getTableName()).isEqualTo(table);
|
||||
assertThat(qInstance.getProcess(action).getMinInputRecords()).as(action + " needs no selected records").isEqualTo(0);
|
||||
});
|
||||
assertThat(qInstance.getProcess(CdnOriginCheckProcesses.CHECK_ORIGIN)).isNotNull();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every table and the app have bundle help that says how rows get created, and
|
||||
** the help of a table with a setup action names it.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void everyTableHasEmptyStateHelp() throws Exception
|
||||
{
|
||||
assertThat(CdnHelp.entries()).allSatisfy((key, entry) -> assertThat(entry.path("summary").asText().length()).as(key).isLessThanOrEqualTo(120));
|
||||
for(String table : cdnTables())
|
||||
{
|
||||
List<QHelpContent> help = qInstance.getTable(table).getHelpContent() == null ? null : qInstance.getTable(table).getHelpContent().get(CdnHelp.SLOT);
|
||||
assertThat(help).as(table + " table help").isNotEmpty();
|
||||
assertThat(help.get(0).getContent()).as(table + " help says how rows get created").contains("How rows get created");
|
||||
if(SETUP_ACTIONS.containsKey(table))
|
||||
{
|
||||
assertThat(help.get(0).getContent()).as(table + " help names its setup action").contains(qInstance.getProcess(SETUP_ACTIONS.get(table)).getLabel());
|
||||
}
|
||||
}
|
||||
assertThat(qInstance.getApp("cdnAdmin").getHelpContent()).containsKey(CdnHelp.SLOT);
|
||||
assertThat(CdnHelp.entries().keySet()).allMatch(key -> !key.startsWith("table:") || qInstance.getTable(key.substring(6, key.indexOf(';'))) != null,
|
||||
"help names only real tables");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A host's own help in the slot is never replaced by the bundle.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void hostHelpIsKept() throws Exception
|
||||
{
|
||||
QTableMetaData table = qInstance.getTable(CdnTables.PROVIDER);
|
||||
table.withHelpContent(CdnHelp.SLOT, new QHelpContent("Host help for providers."));
|
||||
qInstance.getApp("cdnAdmin").withHelpContent(CdnHelp.SLOT, new QHelpContent("Host help for the app."));
|
||||
CdnHelp.apply(qInstance, "cdnAdmin");
|
||||
assertThat(table.getHelpContent().get(CdnHelp.SLOT)).singleElement().satisfies(h -> assertThat(h.getContent()).isEqualTo("Host help for providers."));
|
||||
assertThat(qInstance.getApp("cdnAdmin").getHelpContent().get(CdnHelp.SLOT)).singleElement().satisfies(h -> assertThat(h.getContent()).isEqualTo("Host help for the app."));
|
||||
assertThat(qInstance.getTable(CdnTables.HOSTNAME).getHelpContent().get(CdnHelp.SLOT).get(0).getContent()).contains("How rows get created");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
private List<String> cdnTables()
|
||||
{
|
||||
List<String> tables = qInstance.getTables().values().stream().map(QTableMetaData::getName).filter(n -> n.startsWith("cdn_")).sorted().toList();
|
||||
assertThat(tables).hasSize(9);
|
||||
return (tables);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The CdnTables constant whose value is the table name.
|
||||
*******************************************************************************/
|
||||
private static String constantFor(String table) throws Exception
|
||||
{
|
||||
for(Field field : CdnTables.class.getFields())
|
||||
{
|
||||
if(field.getType() == String.class && table.equals(field.get(null)) && !field.getName().startsWith("PVS_"))
|
||||
{
|
||||
return (field.getName());
|
||||
}
|
||||
}
|
||||
throw (new IllegalStateException("no constant for " + table));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user