Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
440f69fc5c
|
||
|
|
37fb879f51 | ||
|
|
ecceaf31ee
|
||
|
|
0092574b6e
|
||
|
|
fb4a536494 | ||
|
|
967c52f8bc
|
||
|
|
253f862ecd
|
||
|
|
5801ff2ebd | ||
|
|
7510392166
|
||
|
|
59fc98ccb2 | ||
|
|
1e2af2acf1
|
||
|
|
2c41409fb2
|
||
|
|
b1d7d9543f | ||
|
|
15439d2fba |
No files matched your search
+57
-11
@@ -1,14 +1,60 @@
|
||||
version: 2.1
|
||||
setup: true
|
||||
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
workflows:
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release\/.*)/
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only: [develop]
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only: [/release\/.*/]
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [/^v\d+\.\d+\.\d+$/]
|
||||
@@ -1,25 +0,0 @@
|
||||
# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
|
||||
name: qrun-release
|
||||
on:
|
||||
push: {}
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
jobs:
|
||||
release:
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
docker-runner: container-build
|
||||
browsers: true
|
||||
browser-runner: browser-build
|
||||
postpublish:
|
||||
if: ${{ needs.release.outputs.capture == 'true' }}
|
||||
needs: release
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
@@ -1,23 +0,0 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -0,0 +1,25 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release-no-hotfix
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
hotfix: true
|
||||
asciidoc: false
|
||||
@@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository.
|
||||
## Dependencies
|
||||
|
||||
- Java 21.
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.4` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.5` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
|
||||
- Tests: see pom.xml.
|
||||
|
||||
## Release
|
||||
|
||||
@@ -4,7 +4,13 @@ All notable changes to this project are documented here. The format follows Keep
|
||||
|
||||
## [0.1.1] - Unreleased
|
||||
|
||||
### Changed
|
||||
- Require released QQQ BOM 4.1.0-RC.5; revalidation hooks default to core Standard Webhooks signing. Per-hook `signatureFormat=LEGACY` preserves the prior header and signature bytes.
|
||||
- Use core `SafeHttpClient` for revalidation requests: HTTPS/public targets by default, explicit `allowPlainHttp`/`internalAllowlist` exceptions, bounded responses and checked DNS pinning. Fan-out preserves Host/TLS identity. Host schema migration and receiver migration are described in README and `docs/revalidation-hook-migration.md`.
|
||||
|
||||
|
||||
### Fixed
|
||||
- Purge actions warn when providers are NONE or hostnames are OFF, and report requests as recorded but not sent. The home explains recorded-only purges and that this version does not run origin header checks.
|
||||
- Tag rule `{prefix.field}` placeholders now resolve the joined table from the `<prefix>Id` field's possible-value source, so prefixed tables (`cms_blog`) work; a missing joined table is logged instead of silently skipped.
|
||||
|
||||
## [0.1.0] - Unreleased
|
||||
|
||||
@@ -58,6 +58,36 @@ CloudFront credentials: `<secretRef>_ACCESS_KEY_ID` / `<secretRef>_SECRET_ACCESS
|
||||
Drift: the OpenTofu `expected_config` JSON in `CDN_EXPECTED_CONFIG` (or the
|
||||
variable named by the provider setting `expectedConfigEnv`).
|
||||
|
||||
## Revalidation hooks
|
||||
|
||||
Requires QQQ **4.1.0-RC.5** (the default `qqq-snapshot` profile imports the released BOM;
|
||||
its historical property name does not select a core SNAPSHOT).
|
||||
|
||||
New `cdn_revalidation_hook` rows default `signatureFormat` to `STANDARD_WEBHOOKS`;
|
||||
existing rows with a null format keep the `LEGACY` signature. Signed requests use core `WebhookSigner` and
|
||||
`WebhookSecret`: `webhook-id`, `webhook-timestamp`, and `webhook-signature`.
|
||||
Rows on `LEGACY` (or null) keep sending
|
||||
`X-Revalidate-Signature: sha256=<lowercase hex HMAC-SHA256 of the body>`.
|
||||
Legacy keys remain the exact UTF-8 environment value, with no trimming or decoding.
|
||||
Standard keys must contain at least 16 bytes after core parsing (trimmed raw text,
|
||||
or base64-decoded `whsec_` text). `secretRef` names the environment variable;
|
||||
as before, no resolved secret means no signature headers.
|
||||
|
||||
Core `SafeHttpClient` now enforces HTTPS and rejects internal addresses by default.
|
||||
For an internal HTTP pod endpoint explicitly set `allowPlainHttp=true` and
|
||||
`internalAllowlist` to its exact hostname, a `.suffix`, IP, or CIDR (comma/whitespace
|
||||
separated). Metadata and unspecified addresses remain blocked. Every DNS address
|
||||
is checked before fan-out; the original hostname is retained for Host/TLS, and
|
||||
connections use the checked addresses without resolving again. Redirects fail,
|
||||
responses are bounded to 64 KiB, and `timeoutMs` is a positive per-request total
|
||||
limit (default 5000). These transport rules apply to both signature formats.
|
||||
|
||||
Host apps must migrate `cdn_revalidation_hook` to add `signatureFormat` (string),
|
||||
`allowPlainHttp` (boolean), and `internalAllowlist` (text). Backfill `LEGACY` for
|
||||
old receivers; configure internal network exceptions explicitly before dispatch.
|
||||
See [receiver migration](docs/revalidation-hook-migration.md) for the inspected
|
||||
KoF22 receiver and exact changes required.
|
||||
|
||||
## Build
|
||||
|
||||
```sh
|
||||
@@ -67,3 +97,18 @@ variable named by the provider setting `expectedConfigEnv`).
|
||||
## License
|
||||
|
||||
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
||||
|
||||
### Before a CDN is live
|
||||
|
||||
With no enabled CDN provider (including providers of type `NONE`), purges are
|
||||
recorded only, not sent. A real provider also needs a `LIVE` or `LOCKED` hostname
|
||||
with purging enabled. Purge From CDN, Purge Everything, and Retry Purge show a
|
||||
Review warning and a recorded-only Result when sending is unavailable. Requests
|
||||
for enabled providers remain visible with `SKIPPED_NONE_PROVIDER` or
|
||||
`SKIPPED_STAGE_OFF`; with no enabled provider, no new request row is created.
|
||||
The home Connection and Health cards explain the unconfigured state, and purge
|
||||
activity labels recorded-only requests when disconnected.
|
||||
|
||||
This version does not run origin header checks. The Origin headers card says
|
||||
nothing has been checked until the host application writes results to the
|
||||
standard `cdn_origin_check` table; those results continue to appear normally.
|
||||
@@ -0,0 +1,68 @@
|
||||
# Revalidation hook migration (F11 / qbit-cdn #15)
|
||||
|
||||
Reference: core source read directly at tag `v4.1.0-RC.5`, including
|
||||
`WebhookSigner`, `WebhookSecret`, `WebhookVerifier`, `SafeHttpClient`,
|
||||
`SafeHttpPolicy`, and `AddressRules`. No temporary core bridge is needed.
|
||||
|
||||
## Receiver found locally
|
||||
|
||||
The plain `Website-Frontend` and `Website-Publisher` checkouts have no match for
|
||||
`X-Revalidate-Signature`. The implementation is in
|
||||
`R:/Git.Local/KofTwentyTwo/Website-Frontend-wt-isr`, branch `feature/cdn-isr`,
|
||||
HEAD `476318b` at inspection (working files inspected):
|
||||
|
||||
- `src/lib/revalidate.ts:5`: names `x-revalidate-signature`.
|
||||
- `src/lib/revalidate.ts:9`: `signBody` returns `sha256=` plus hex HMAC-SHA256
|
||||
over the body using the raw string key (the calculation is line 10).
|
||||
- `src/lib/revalidate.ts:14`: `verifySignature` compares this value in constant
|
||||
time; line 21 lowercases the incoming legacy header.
|
||||
- `src/app/api/revalidate/route.ts:30`: reads `CDN_REVALIDATE_SECRET`.
|
||||
- `src/app/api/revalidate/route.ts:36`: reads the body before JSON parsing.
|
||||
- `src/app/api/revalidate/route.ts:40`: reads only the legacy signature header
|
||||
and calls the verifier; standard requests currently return 401.
|
||||
- `src/__tests__/revalidate-route.test.ts:12`: test requests send only the legacy header.
|
||||
|
||||
These receiver files were inspected, not modified.
|
||||
|
||||
## Deployment and receiver changes
|
||||
|
||||
1. Add the three hook columns documented in README. Existing rows (null
|
||||
`signatureFormat`) keep the LEGACY signature; switch a hook to `STANDARD_WEBHOOKS`
|
||||
after its receiver verifies Standard Webhooks. New rows default to `STANDARD_WEBHOOKS`.
|
||||
For internal HTTP endpoints also explicitly allow HTTP and the intended host.
|
||||
2. Update `src/lib/revalidate.ts:14` to verify Standard Webhooks: require all
|
||||
three headers, validate an integer Unix timestamp within 300 seconds of the
|
||||
receiver clock (reject malformed/non-finite values), then HMAC-SHA256 the
|
||||
bytes `<webhook-id>.<webhook-timestamp>.<raw body>` and compare decoded
|
||||
signatures in constant time. Accept any matching `v1,<base64>` entry in the
|
||||
space-separated signature list. Do not lowercase base64 signatures.
|
||||
Match core secret parsing: trim text; decode `whsec_` base64 keys, otherwise
|
||||
UTF-8; use at least 16 key bytes. Rotate short existing keys on both sides.
|
||||
3. At `src/app/api/revalidate/route.ts:36`, retain raw body bytes for verification;
|
||||
at line 40 pass all three headers to the new verifier before parsing JSON.
|
||||
Update its header import/documentation (lines 5 and 22). During migration the
|
||||
receiver may explicitly support both formats; do not fall back to legacy
|
||||
when standard headers are present but invalid.
|
||||
4. Update `src/__tests__/revalidate-route.test.ts:12` and helper tests to cover
|
||||
valid standard requests, altered body/id/timestamp, missing headers, stale
|
||||
or future timestamps, malformed timestamps, encoded keys, and legacy mode.
|
||||
5. Deploy the receiver, then switch its hook to `STANDARD_WEBHOOKS`.
|
||||
|
||||
The JSON payload remains `{"tags":[...]}`. One UUID identifies a hook invocation,
|
||||
shared across its fan-out requests; each request gets its own current timestamp.
|
||||
A later dispatcher invocation generates a new UUID. Revalidation remains
|
||||
idempotent, so receivers must not rely on this ID to deduplicate purge retries;
|
||||
any optional deduplication must remain local to each pod, not suppress other pods.
|
||||
|
||||
## Compatibility and verification
|
||||
|
||||
The legacy HMAC helper remains solely for the explicitly supported legacy format,
|
||||
not as a temporary replacement for core. Golden tests use fixed digests calculated
|
||||
independently of that helper, including a short key and a whitespace/prefix key.
|
||||
Standard requests are checked using RC.5 `WebhookVerifier`; transport tests cover
|
||||
HTTP/private-address refusal, explicit local exceptions and mixed-address fan-out.
|
||||
|
||||
The test/collaborator constructor now takes `(secrets, resolver)`; callers no longer
|
||||
inject `java.net.http.HttpClient`. The production no-argument constructor remains.
|
||||
The only existing test adaptation outside the runner tests is `PurgePipelineTest`
|
||||
using this constructor. The RC.5 bump required no unrelated code fixes.
|
||||
@@ -18,17 +18,19 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<revision>0.1.0</revision>
|
||||
<qbit.guardrails.enforce>true</qbit.guardrails.enforce>
|
||||
<qbit.guardrails.strictUiApiRoutes>true</qbit.guardrails.strictUiApiRoutes>
|
||||
<revision>0.1.0-RC.1</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
<aws.sdk.version>2.41.10</aws.sdk.version>
|
||||
<aws.sdk.version>2.41.34</aws.sdk.version>
|
||||
</properties>
|
||||
|
||||
<parent>
|
||||
<groupId>com.kingsrook</groupId>
|
||||
<artifactId>qbit-build-parent</artifactId>
|
||||
<version>2.0.0</version>
|
||||
<version>2.1.0</version>
|
||||
</parent>
|
||||
|
||||
<groupId>com.kingsrook.qbits</groupId>
|
||||
@@ -192,7 +194,7 @@
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
|
||||
<qqq.snapshot.version>4.1.0-RC.6</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -61,14 +61,14 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
** A provider row as shown.
|
||||
*******************************************************************************/
|
||||
public record ProviderView(Integer id, String name, String type, String distributionId, String dnsTarget, String originHost,
|
||||
String lastHealthStatus, String lastHealthMessage, Instant lastSuccessfulTest, Long credentialAgeDays)
|
||||
String lastHealthStatus, String lastHealthMessage, Instant lastSuccessfulTest, Long credentialAgeDays, Boolean enabled)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Whether this is a real CDN (not the NONE placeholder).
|
||||
*******************************************************************************/
|
||||
public boolean isConnected()
|
||||
{
|
||||
return (!CdnProviderType.NONE.name().equals(type));
|
||||
return (Boolean.TRUE.equals(enabled) && !CdnProviderType.NONE.name().equals(type));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ public record CdnHomeData(List<ProviderView> providers, List<HostnameView> hostn
|
||||
LocalDate rotated = ValueUtils.getValueAsLocalDate(row.getValue("credentialRotatedDate"));
|
||||
providers.add(new ProviderView(id, row.getValueString("name"), row.getValueString("providerType"), row.getValueString("zoneOrDistributionId"),
|
||||
row.getValueString("dnsTarget"), row.getValueString("originHost"), row.getValueString("lastHealthStatus"), row.getValueString("lastHealthMessage"),
|
||||
ValueUtils.getValueAsInstant(row.getValue("lastSuccessfulTestDate")), rotated == null ? null : ChronoUnit.DAYS.between(rotated, today)));
|
||||
ValueUtils.getValueAsInstant(row.getValue("lastSuccessfulTestDate")), rotated == null ? null : ChronoUnit.DAYS.between(rotated, today), row.getValueBoolean("enabled")));
|
||||
if(CdnProviderType.CLOUDFRONT.name().equals(row.getValueString("providerType")))
|
||||
{
|
||||
PathPlannerSettings settings = PathPlannerSettings.from(CdnPurgeDispatcher.configOf(row));
|
||||
|
||||
@@ -80,7 +80,7 @@ public final class CdnHomeView
|
||||
String text = "All CDN checks are OK.";
|
||||
if("NOT_CONNECTED".equals(state))
|
||||
{
|
||||
text = "No CDN connected yet.";
|
||||
text = "No CDN provider configured: purges are recorded only.";
|
||||
}
|
||||
else if("ATTENTION".equals(state))
|
||||
{
|
||||
@@ -99,7 +99,7 @@ public final class CdnHomeView
|
||||
open(html, "Connection");
|
||||
if(!data.isConnected())
|
||||
{
|
||||
empty(html, "No CDN connected yet. The CloudFront distribution is created in the cutover window.");
|
||||
empty(html, "No CDN provider configured: purges are recorded only. The CloudFront distribution is created in the cutover window.");
|
||||
}
|
||||
for(CdnHomeData.ProviderView provider : data.providers())
|
||||
{
|
||||
@@ -182,6 +182,7 @@ public final class CdnHomeView
|
||||
Instant created = ValueUtils.getValueAsInstant(row.getValue("createDate"));
|
||||
html.append("<li><a href=\"").append(esc(appPath + "/" + CdnTables.PURGE_REQUEST + "/" + row.getValue("id"))).append("\">#")
|
||||
.append(esc(String.valueOf(row.getValue("id")))).append("</a> ").append(esc(row.getValueString("status")))
|
||||
.append(data.isConnected() ? "" : " - recorded only (not sent)")
|
||||
.append(created == null ? "" : ", " + WHEN.format(created)).append("</li>");
|
||||
}
|
||||
html.append("</ul>");
|
||||
@@ -224,7 +225,7 @@ public final class CdnHomeView
|
||||
open(html, "Origin headers");
|
||||
if(data.originChecks().isEmpty())
|
||||
{
|
||||
empty(html, "No origin header check has run yet.");
|
||||
empty(html, "Origin header checks are not run by this version; nothing has been checked.");
|
||||
}
|
||||
for(QRecord check : data.originChecks())
|
||||
{
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.model;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Signature format selected on each revalidation hook.
|
||||
*******************************************************************************/
|
||||
public enum CdnSignatureFormat implements PossibleValueEnum<String>
|
||||
{
|
||||
STANDARD_WEBHOOKS("STANDARD_WEBHOOKS", "Standard Webhooks"),
|
||||
LEGACY("LEGACY", "Legacy X-Revalidate-Signature");
|
||||
|
||||
private final String id;
|
||||
private final String label;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
CdnSignatureFormat(String id, String label)
|
||||
{
|
||||
this.id = id;
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for possibleValueId.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getPossibleValueId()
|
||||
{
|
||||
return (id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for possibleValueLabel.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (label);
|
||||
}
|
||||
}
|
||||
@@ -54,11 +54,12 @@ public final class CdnTables
|
||||
public static final List<String> ALL = List.of(PROVIDER, HOSTNAME, CACHE_POLICY, TABLE_TAG_RULE, PURGE_REQUEST, REVALIDATION_HOOK,
|
||||
EDGE_STAT_DAILY, ORIGIN_CHECK, DRIFT_FINDING);
|
||||
|
||||
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
|
||||
public static final String PVS_STAGE = "cdn_hostname_stage";
|
||||
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
|
||||
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
|
||||
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
|
||||
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
|
||||
public static final String PVS_STAGE = "cdn_hostname_stage";
|
||||
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
|
||||
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
|
||||
public static final String PVS_SIGNATURE_FORMAT = "cdn_signature_format";
|
||||
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
|
||||
|
||||
|
||||
|
||||
@@ -95,6 +96,7 @@ public final class CdnTables
|
||||
QPossibleValueSource.newForEnum(PVS_PURGE_STATUS, CdnPurgeStatus.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_PURGE_SOURCE, CdnPurgeSource.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_POLICY_CLASS, CdnPolicyClass.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_SIGNATURE_FORMAT, CdnSignatureFormat.values()),
|
||||
QPossibleValueSource.newForTable(PROVIDER),
|
||||
QPossibleValueSource.newForTable(HOSTNAME)));
|
||||
}
|
||||
@@ -318,6 +320,9 @@ public final class CdnTables
|
||||
.withField(new QFieldMetaData("url", QFieldType.STRING).withIsRequired(true).withMaxLength(500))
|
||||
.withField(new QFieldMetaData("fanOutDns", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData("secretRef", QFieldType.STRING).withMaxLength(100))
|
||||
.withField(new QFieldMetaData("signatureFormat", QFieldType.STRING).withPossibleValueSourceName(PVS_SIGNATURE_FORMAT).withDefaultValue(CdnSignatureFormat.STANDARD_WEBHOOKS.name()))
|
||||
.withField(new QFieldMetaData("allowPlainHttp", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData("internalAllowlist", QFieldType.TEXT))
|
||||
.withField(new QFieldMetaData("timeoutMs", QFieldType.INTEGER).withDefaultValue(5000))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)));
|
||||
}
|
||||
|
||||
@@ -167,6 +167,56 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Summary counts for requests that cannot be sent, shared by all actions.
|
||||
*******************************************************************************/
|
||||
private static class UnsentSummary
|
||||
{
|
||||
private final ProcessSummaryLine absent = line(Status.WARNING,
|
||||
"record has no enabled CDN provider", "records have no enabled CDN provider",
|
||||
"record had no enabled CDN provider (no purge request recorded)", "records had no enabled CDN provider (no purge requests recorded)");
|
||||
private final ProcessSummaryLine none = line(Status.WARNING,
|
||||
"No CDN provider configured: purges are recorded only, not sent.", "No CDN provider configured: purges are recorded only, not sent.",
|
||||
"purge request was recorded (not sent: no CDN provider)", "purge requests were recorded (not sent: no CDN provider)");
|
||||
private final ProcessSummaryLine off = line(Status.WARNING,
|
||||
"CDN hostnames are not live: purges are recorded only, not sent.", "CDN hostnames are not live: purges are recorded only, not sent.",
|
||||
"purge request was recorded (not sent: CDN hostnames are not live)", "purge requests were recorded (not sent: CDN hostnames are not live)");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Count a skipped request; returns whether it cannot be sent.
|
||||
*******************************************************************************/
|
||||
boolean add(CdnPurgeStatus status)
|
||||
{
|
||||
if(status == CdnPurgeStatus.SKIPPED_NONE_PROVIDER)
|
||||
{
|
||||
none.incrementCount(1);
|
||||
return (true);
|
||||
}
|
||||
if(status == CdnPurgeStatus.SKIPPED_STAGE_OFF)
|
||||
{
|
||||
off.incrementCount(1);
|
||||
return (true);
|
||||
}
|
||||
return (false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Include only reasons encountered by this action.
|
||||
*******************************************************************************/
|
||||
void append(ArrayList<ProcessSummaryLineInterface> lines)
|
||||
{
|
||||
absent.addSelfToListIfAnyCount(lines);
|
||||
none.addSelfToListIfAnyCount(lines);
|
||||
off.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Scheduled: send due purges and poll in-progress ones.
|
||||
*******************************************************************************/
|
||||
@@ -189,6 +239,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class PurgeRecordsTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine records = line(Status.OK, "record will be purged", "records will be purged",
|
||||
"record was purged", "records were purged");
|
||||
private final ProcessSummaryLine noRule = line(Status.INFO, "record has no CDN tag rule (no change)", "records have no CDN tag rule (no change)",
|
||||
@@ -207,6 +258,7 @@ public final class CdnProcesses
|
||||
String table = input.getValueString(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE);
|
||||
List<QRecord> rules = rules(table);
|
||||
TagRuleEngine engine = new TagRuleEngine();
|
||||
CdnPurgeStatus sending = CdnPurgeService.sendingStatus(null);
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
List<String> tags = new ArrayList<>();
|
||||
@@ -226,7 +278,10 @@ public final class CdnProcesses
|
||||
recordPaths.forEach(path -> paths.add(path.path()));
|
||||
record.setValue("cdnTags", String.join("\n", tags));
|
||||
record.setValue("cdnPaths", com.kingsrook.qbits.cdn.purge.PurgeText.paths(recordPaths));
|
||||
records.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(sending))
|
||||
{
|
||||
records.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -249,6 +304,7 @@ public final class CdnProcesses
|
||||
lines.add(pathLine);
|
||||
}
|
||||
noRule.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -256,10 +312,11 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Purge From CDN, execute: one purge request (sent at once, hubs included).
|
||||
** Purge From CDN, execute: record requests with hubs included.
|
||||
*******************************************************************************/
|
||||
public static class PurgeRecordsLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"purge request was recorded (sent within seconds)", "purge requests were recorded (sent within seconds)");
|
||||
|
||||
@@ -277,7 +334,17 @@ public final class CdnProcesses
|
||||
List<QRecord> rows = CdnPurgeService.enqueue(new CdnPurgeService.Request(CdnPurgeKind.TAG.name(),
|
||||
com.kingsrook.qbits.cdn.purge.PurgeText.tags(record.getValueString("cdnTags")),
|
||||
com.kingsrook.qbits.cdn.purge.PurgeText.paths(record.getValueString("cdnPaths")), true, CdnPurgeSource.RECORD_ACTION, table, record.getValue("id")));
|
||||
requested.incrementCount(rows.size());
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
unsent.absent.incrementCount(1);
|
||||
}
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
if(!unsent.add(CdnPurgeStatus.valueOf(row.getValueString("status"))))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -296,6 +363,7 @@ public final class CdnProcesses
|
||||
{
|
||||
transform.noRule.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -307,6 +375,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class RetryTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine retry = line(Status.OK, "purge request will be sent again", "purge requests will be sent again",
|
||||
"purge request was queued again", "purge requests were queued again");
|
||||
private final ProcessSummaryLine skip = line(Status.INFO, "purge request has not failed (no change)", "purge requests have not failed (no change)",
|
||||
@@ -325,7 +394,10 @@ public final class CdnProcesses
|
||||
String status = record.getValueString("status");
|
||||
if(CdnPurgeStatus.FAILED.name().equals(status) || CdnPurgeStatus.DEAD.name().equals(status) || CdnPurgeStatus.DEFERRED_BUDGET.name().equals(status))
|
||||
{
|
||||
retry.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(CdnPurgeService.sendingStatus(record.getValue("cdnProviderId"))))
|
||||
{
|
||||
retry.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
else
|
||||
@@ -346,6 +418,7 @@ public final class CdnProcesses
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
retry.addSelfToListIfAnyCount(lines);
|
||||
skip.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -353,10 +426,16 @@ public final class CdnProcesses
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Retry Purge, execute: back to PENDING (sent on the next dispatcher pass).
|
||||
** Retry Purge, execute: queue live providers or record why sending is skipped.
|
||||
*******************************************************************************/
|
||||
public static class RetryLoad extends AbstractLoadStep
|
||||
public static class RetryLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"purge request was queued again", "purge requests were queued again");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Reset the rows.
|
||||
*******************************************************************************/
|
||||
@@ -366,7 +445,12 @@ public final class CdnProcesses
|
||||
List<QRecord> updates = new ArrayList<>();
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("status", CdnPurgeStatus.PENDING.name())
|
||||
CdnPurgeStatus status = CdnPurgeService.sendingStatus(record.getValue("cdnProviderId"));
|
||||
if(!unsent.add(status))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
updates.add(new QRecord().withValue("id", record.getValue("id")).withValue("status", status.name())
|
||||
.withValue("source", CdnPurgeSource.RETRY.name()).withValue("attempts", 0).withValue("nextAttemptDate", null)
|
||||
.withValue("errorMessage", null).withValue("completedDate", null).withValue("modifyDate", CdnRuntime.clock().instant()));
|
||||
}
|
||||
@@ -375,6 +459,24 @@ public final class CdnProcesses
|
||||
output.getRecords().addAll(new UpdateAction().execute(new UpdateInput(CdnTables.PURGE_REQUEST).withRecords(updates)).getRecords());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result lines from the requests recorded during execution.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
requested.addSelfToListIfAnyCount(lines);
|
||||
if(getTransformStep() instanceof RetryTransform transform)
|
||||
{
|
||||
transform.skip.addSelfToListIfAnyCount(lines);
|
||||
}
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -384,6 +486,7 @@ public final class CdnProcesses
|
||||
*******************************************************************************/
|
||||
public static class PurgeHostnameTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine providers = line(Status.WARNING,
|
||||
"CDN distribution will be purged completely (every hostname on it)", "CDN distributions will be purged completely (every hostname on them)",
|
||||
"CDN distribution purge was recorded", "CDN distribution purges were recorded");
|
||||
@@ -402,7 +505,10 @@ public final class CdnProcesses
|
||||
Serializable providerId = record.getValue("cdnProviderId");
|
||||
if(providerId != null && seen.add(providerId))
|
||||
{
|
||||
providers.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
if(!unsent.add(CdnPurgeService.sendingStatus(providerId)))
|
||||
{
|
||||
providers.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
@@ -418,6 +524,7 @@ public final class CdnProcesses
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
providers.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
@@ -427,8 +534,14 @@ public final class CdnProcesses
|
||||
/*******************************************************************************
|
||||
** Purge Everything, execute.
|
||||
*******************************************************************************/
|
||||
public static class PurgeHostnameLoad extends AbstractLoadStep
|
||||
public static class PurgeHostnameLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final UnsentSummary unsent = new UnsentSummary();
|
||||
private final ProcessSummaryLine requested = line(Status.OK, "purge request will be recorded", "purge requests will be recorded",
|
||||
"CDN distribution purge was recorded", "CDN distribution purges were recorded");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Record an ALL purge for each provider.
|
||||
*******************************************************************************/
|
||||
@@ -437,11 +550,36 @@ public final class CdnProcesses
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
CdnPurgeService.enqueueFor(record.getValue("cdnProviderId"), new CdnPurgeService.Request(CdnPurgeKind.ALL.name(), List.of(), List.of(), true,
|
||||
List<QRecord> rows = CdnPurgeService.enqueueFor(record.getValue("cdnProviderId"), new CdnPurgeService.Request(CdnPurgeKind.ALL.name(), List.of(), List.of(), true,
|
||||
CdnPurgeSource.MANUAL, CdnTables.HOSTNAME, record.getValue("id")));
|
||||
if(rows.isEmpty())
|
||||
{
|
||||
unsent.absent.incrementCount(1);
|
||||
}
|
||||
for(QRecord row : rows)
|
||||
{
|
||||
if(!unsent.add(CdnPurgeStatus.valueOf(row.getValueString("status"))))
|
||||
{
|
||||
requested.incrementCount(1);
|
||||
}
|
||||
}
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result lines from the requests recorded during execution.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
requested.addSelfToListIfAnyCount(lines);
|
||||
unsent.append(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -120,6 +120,36 @@ public final class CdnPurgeService
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Sending eligibility for one provider, or all enabled providers when null.
|
||||
** A pending status means at least one provider can send; otherwise the
|
||||
** status gives the same reason used when recording a new request.
|
||||
*******************************************************************************/
|
||||
public static CdnPurgeStatus sendingStatus(Serializable providerId) throws QException
|
||||
{
|
||||
QQueryFilter filter = new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true));
|
||||
if(providerId != null)
|
||||
{
|
||||
filter.addCriteria(new QFilterCriteria("id", QCriteriaOperator.EQUALS, providerId));
|
||||
}
|
||||
CdnPurgeStatus status = CdnPurgeStatus.SKIPPED_NONE_PROVIDER;
|
||||
for(QRecord provider : new QueryAction().execute(new QueryInput(CdnTables.PROVIDER).withFilter(filter)).getRecords())
|
||||
{
|
||||
CdnPurgeStatus candidate = initialStatus(provider);
|
||||
if(candidate == CdnPurgeStatus.PENDING)
|
||||
{
|
||||
return (candidate);
|
||||
}
|
||||
if(candidate == CdnPurgeStatus.SKIPPED_STAGE_OFF)
|
||||
{
|
||||
status = candidate;
|
||||
}
|
||||
}
|
||||
return (status);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Status of a new request for a provider.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -21,16 +21,18 @@ import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.cdn.model.CdnSignatureFormat;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
@@ -41,17 +43,20 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpClient;
|
||||
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpPolicy;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSigner;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Calls every enabled cdn_revalidation_hook before the CDN is purged, so the
|
||||
** edge never refetches a stale page (design spec section 4.5 step 3).
|
||||
**
|
||||
** Request: POST {"tags":[...]} with header
|
||||
** "X-Revalidate-Signature: sha256=<hex HMAC-SHA256 of the body>", keyed by
|
||||
** the secret in the environment variable the hook's secretRef names. With
|
||||
** fanOutDns, the hook's host is resolved and every address is called (one
|
||||
** per pod behind a headless Service). Every call must answer 2xx.
|
||||
** Request: POST {"tags":[...]} signed with core Standard Webhooks headers.
|
||||
** LEGACY keeps X-Revalidate-Signature with a hex HMAC of the body. The
|
||||
** secretRef names an environment variable. SafeHttpClient checks and pins
|
||||
** every address; fan-out preserves the original Host and TLS server name.
|
||||
*******************************************************************************/
|
||||
public class RevalidationHookRunner
|
||||
{
|
||||
@@ -59,9 +64,8 @@ public class RevalidationHookRunner
|
||||
|
||||
private static final ObjectMapper JSON = new ObjectMapper();
|
||||
|
||||
private final HttpClient http;
|
||||
private final Function<String, String> secrets;
|
||||
private final Resolver resolver;
|
||||
private final Function<String, String> secrets;
|
||||
private final Resolver resolver;
|
||||
|
||||
|
||||
|
||||
@@ -83,7 +87,7 @@ public class RevalidationHookRunner
|
||||
*******************************************************************************/
|
||||
public RevalidationHookRunner()
|
||||
{
|
||||
this(HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(), System::getenv,
|
||||
this(System::getenv,
|
||||
host -> java.util.Arrays.stream(InetAddress.getAllByName(host)).map(InetAddress::getHostAddress).toList());
|
||||
}
|
||||
|
||||
@@ -92,9 +96,8 @@ public class RevalidationHookRunner
|
||||
/*******************************************************************************
|
||||
** Constructor with collaborators.
|
||||
*******************************************************************************/
|
||||
public RevalidationHookRunner(HttpClient http, Function<String, String> secrets, Resolver resolver)
|
||||
public RevalidationHookRunner(Function<String, String> secrets, Resolver resolver)
|
||||
{
|
||||
this.http = http;
|
||||
this.secrets = secrets;
|
||||
this.resolver = resolver;
|
||||
}
|
||||
@@ -133,42 +136,63 @@ public class RevalidationHookRunner
|
||||
byte[] body = JSON.writeValueAsBytes(java.util.Map.of("tags", tags));
|
||||
String secretRef = hook.getValueString("secretRef");
|
||||
String secret = StringUtils.hasContent(secretRef) ? secrets.apply(secretRef) : null;
|
||||
String formatValue = hook.getValueString("signatureFormat");
|
||||
CdnSignatureFormat format = formatValue == null ? CdnSignatureFormat.LEGACY : CdnSignatureFormat.valueOf(formatValue);
|
||||
WebhookSecret signingSecret = secret != null && format == CdnSignatureFormat.STANDARD_WEBHOOKS ? WebhookSecret.parse(secret) : null;
|
||||
String messageId = UUID.randomUUID().toString();
|
||||
URI uri = URI.create(hook.getValueString("url"));
|
||||
List<URI> targets = new ArrayList<>();
|
||||
if(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns"))))
|
||||
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
|
||||
if(timeout != null && timeout <= 0)
|
||||
{
|
||||
for(String address : resolver.resolve(uri.getHost()))
|
||||
{
|
||||
String host = address.contains(":") ? "[" + address + "]" : address;
|
||||
targets.add(new URI(uri.getScheme(), null, host, uri.getPort(), uri.getPath(), uri.getQuery(), null));
|
||||
}
|
||||
throw (new QException("Revalidation hook timeoutMs must be positive"));
|
||||
}
|
||||
else
|
||||
String allowlist = hook.getValueString("internalAllowlist");
|
||||
SafeHttpPolicy policy = SafeHttpPolicy.defaults()
|
||||
.withAllowPlainHttp(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("allowPlainHttp"))))
|
||||
.withInternalAllowlist(allowlist == null ? List.of() : Arrays.asList(allowlist.split("[,\\s]+")))
|
||||
.withTotalTimeout(Duration.ofMillis(timeout == null ? 5000 : timeout));
|
||||
List<String> resolved = resolver.resolve(uri.getHost());
|
||||
InetAddress[] addresses = new InetAddress[resolved.size()];
|
||||
for(int i = 0; i < resolved.size(); i++)
|
||||
{
|
||||
targets.add(uri);
|
||||
addresses[i] = InetAddress.getByName(resolved.get(i));
|
||||
}
|
||||
if(targets.isEmpty())
|
||||
if(addresses.length == 0)
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " resolved to no addresses"));
|
||||
}
|
||||
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
|
||||
for(URI target : targets)
|
||||
try(SafeHttpClient validator = new SafeHttpClient(policy, host -> addresses))
|
||||
{
|
||||
HttpRequest.Builder request = HttpRequest.newBuilder(target)
|
||||
.timeout(Duration.ofMillis(timeout == null ? 5000 : timeout))
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofByteArray(body));
|
||||
validator.checkUrl(uri.toString());
|
||||
}
|
||||
boolean fanOut = Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns")));
|
||||
int calls = fanOut ? addresses.length : 1;
|
||||
for(int i = 0; i < calls; i++)
|
||||
{
|
||||
InetAddress[] targetAddresses = fanOut ? new InetAddress[] {addresses[i]} : addresses;
|
||||
Map<String, String> headers = new LinkedHashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
if(secret != null)
|
||||
{
|
||||
request.header(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
|
||||
if(format == CdnSignatureFormat.LEGACY)
|
||||
{
|
||||
headers.put(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
|
||||
}
|
||||
else
|
||||
{
|
||||
headers.putAll(WebhookSigner.headers(messageId, Instant.now(), body, List.of(signingSecret)));
|
||||
}
|
||||
}
|
||||
HttpResponse<String> response = http.send(request.build(), HttpResponse.BodyHandlers.ofString());
|
||||
if(response.statusCode() / 100 != 2)
|
||||
try(SafeHttpClient client = new SafeHttpClient(policy, host -> targetAddresses))
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " answered " + response.statusCode() + " from " + target.getHost()));
|
||||
int status = client.post(uri.toString(), headers, body).statusCode();
|
||||
if(status / 100 != 2)
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " answered " + status + " from " + uri.getHost()));
|
||||
}
|
||||
}
|
||||
}
|
||||
return (targets.size());
|
||||
return (calls);
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
@@ -188,7 +212,7 @@ public class RevalidationHookRunner
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Hex HMAC-SHA256.
|
||||
** Legacy compatibility: hex HMAC-SHA256 over the body with raw UTF-8 key bytes.
|
||||
*******************************************************************************/
|
||||
public static String sign(String secret, byte[] body) throws Exception
|
||||
{
|
||||
|
||||
@@ -58,10 +58,10 @@ class CdnHomeTest
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
String html = render(CdnWidgets.HOME_WIDGET);
|
||||
assertThat(html).contains("No CDN connected yet.").contains("No edge statistics yet.").contains("No purges recorded yet.")
|
||||
.contains("No path-billed provider connected yet.").contains("No origin header check has run yet.").contains("Nothing to compare yet.")
|
||||
assertThat(html).contains("No CDN provider configured: purges are recorded only.").contains("No edge statistics yet.").contains("No purges recorded yet.")
|
||||
.contains("No path-billed provider connected yet.").contains("Origin header checks are not run by this version; nothing has been checked.").contains("Nothing to compare yet.")
|
||||
.contains("minmax(min(100%,22rem),1fr)");
|
||||
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN connected yet.").contains("href=\"/cdnAdmin\"");
|
||||
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN provider configured: purges are recorded only.").contains("href=\"/cdnAdmin\"");
|
||||
}
|
||||
|
||||
|
||||
@@ -99,7 +99,7 @@ class CdnHomeTest
|
||||
String html = CdnHomeView.render(data, "/cdnAdmin");
|
||||
assertThat(html).contains("d1.cloudfront.net").contains(age + " days (rotate every 90) - rotation due").contains("1,500").contains("91.5%")
|
||||
.contains("<polyline").contains("396 of 396").contains("BEHAVIOR").contains("1 open").contains("/cdnAdmin/cdn_purge_request/")
|
||||
.contains("www.example.com");
|
||||
.contains("www.example.com").doesNotContain("recorded only (not sent)", "nothing has been checked");
|
||||
assertThat(CdnHomeView.renderHealthCard(data, "/cdnAdmin")).contains("Needs attention");
|
||||
}
|
||||
|
||||
@@ -119,4 +119,31 @@ class CdnHomeTest
|
||||
assertEquals("50.0%", CdnHomeView.percent(new BigDecimal("0.5")));
|
||||
assertEquals("91.5%", CdnHomeView.percent(new BigDecimal("91.5")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** NONE and disabled providers do not imply a connection or a sent purge.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void recordedOnlyHome() throws Exception
|
||||
{
|
||||
for(boolean none : List.of(true, false))
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
Integer provider = CdnTestSupport.liveProvider(none ? CdnProviderType.NONE : CdnProviderType.CLOUDFRONT, null);
|
||||
if(!none)
|
||||
{
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.PROVIDER)
|
||||
.withRecord(new QRecord().withValue("id", provider).withValue("enabled", false)));
|
||||
}
|
||||
CdnTestSupport.insert(CdnTables.PURGE_REQUEST, new QRecord().withValue("cdnProviderId", provider).withValue("status", "SKIPPED_NONE_PROVIDER"));
|
||||
assertThat(render(CdnWidgets.HOME_WIDGET)).contains("No CDN provider configured: purges are recorded only.")
|
||||
.contains("The CloudFront distribution is created in the cutover window.").contains("recorded only (not sent)")
|
||||
.contains("Origin header checks are not run by this version; nothing has been checked.");
|
||||
assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN provider configured: purges are recorded only.");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -70,6 +70,16 @@ class CdnProcessesTest
|
||||
RunProcessOutput review = new RunProcessAction().execute(input);
|
||||
List<ProcessSummaryLineInterface> reviewLines = (List<ProcessSummaryLineInterface>) review.getValues().get(StreamedETLWithFrontendProcess.FIELD_VALIDATION_SUMMARY);
|
||||
|
||||
for(ProcessSummaryLineInterface reviewLine : reviewLines)
|
||||
{
|
||||
ProcessSummaryLine summary = (ProcessSummaryLine) reviewLine;
|
||||
summary.pickMessage(false);
|
||||
if(summary.getMessage().contains("purges are recorded only"))
|
||||
{
|
||||
assertEquals(com.kingsrook.qqq.backend.core.model.actions.processes.Status.WARNING, summary.getStatus());
|
||||
}
|
||||
}
|
||||
|
||||
RunProcessInput go = new RunProcessInput();
|
||||
go.setProcessName(process);
|
||||
go.setProcessUUID(review.getProcessUUID());
|
||||
@@ -137,6 +147,7 @@ class CdnProcessesTest
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "SUCCEEDED"));
|
||||
List<List<String>> lines = run(CdnProcesses.RETRY_PURGE, rows.get(0).getValue("id") + "," + rows.get(1).getValue("id"));
|
||||
assertThat(lines.get(0)).containsExactly("1 purge request will be sent again", "1 purge request has not failed (no change)");
|
||||
assertThat(lines.get(1)).contains("1 purge request was queued again");
|
||||
QRecord retried = purges().get(0);
|
||||
assertEquals("PENDING", retried.getValueString("status"));
|
||||
assertEquals(0, retried.getValueInteger("attempts"));
|
||||
@@ -155,9 +166,99 @@ class CdnProcessesTest
|
||||
CdnTestSupport.insert(CdnTables.PROVIDER, new QRecord().withValue("name", "other").withValue("providerType", "CLOUDFRONT").withValue("enabled", true));
|
||||
List<List<String>> lines = run(CdnProcesses.PURGE_HOSTNAME, "1");
|
||||
assertThat(lines.get(0)).containsExactly("1 CDN distribution will be purged completely (every hostname on it)");
|
||||
assertThat(lines.get(1)).containsExactly("1 CDN distribution purge was recorded");
|
||||
List<QRecord> rows = purges();
|
||||
assertThat(rows).hasSize(1);
|
||||
assertEquals("ALL", rows.get(0).getValueString("kind"));
|
||||
assertEquals(1, rows.get(0).getValueInteger("cdnProviderId"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Every action reports NONE and OFF honestly, including plural results.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void recordedOnlySummaries() throws Exception
|
||||
{
|
||||
for(boolean none : List.of(true, false))
|
||||
{
|
||||
for(String process : List.of(CdnProcesses.PURGE_PREFIX + "post", CdnProcesses.PURGE_HOSTNAME, CdnProcesses.RETRY_PURGE))
|
||||
{
|
||||
setUp();
|
||||
String table = none ? CdnTables.PROVIDER : CdnTables.HOSTNAME;
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(table)
|
||||
.withRecord(new QRecord().withValue("id", 1).withValue(none ? "providerType" : "stage", none ? "NONE" : "OFF")));
|
||||
String ids = "1";
|
||||
if(process.equals(CdnProcesses.PURGE_PREFIX + "post"))
|
||||
{
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "one"), new QRecord().withValue("slug", "two"));
|
||||
ids = "1,2";
|
||||
}
|
||||
else if(process.equals(CdnProcesses.RETRY_PURGE))
|
||||
{
|
||||
CdnTestSupport.insert(CdnTables.PURGE_REQUEST,
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "FAILED"),
|
||||
new QRecord().withValue("cdnProviderId", 1).withValue("status", "DEAD"));
|
||||
ids = "1,2";
|
||||
}
|
||||
List<List<String>> lines = run(process, ids);
|
||||
String warning = none ? "No CDN provider configured" : "CDN hostnames are not live";
|
||||
assertThat(lines.get(0)).anyMatch(line -> line.contains(warning + ": purges are recorded only, not sent."));
|
||||
assertThat(lines.get(0)).noneMatch(line -> line.contains("will be purged") || line.contains("will be sent again"));
|
||||
String reason = none ? "no CDN provider" : "CDN hostnames are not live";
|
||||
assertThat(lines.get(1)).containsExactly(process.equals(CdnProcesses.PURGE_HOSTNAME)
|
||||
? "1 purge request was recorded (not sent: " + reason + ")"
|
||||
: "2 purge requests were recorded (not sent: " + reason + ")");
|
||||
assertThat(purges()).allMatch(row -> (none ? "SKIPPED_NONE_PROVIDER" : "SKIPPED_STAGE_OFF").equals(row.getValueString("status")));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Eligibility considers enabled providers and their purge-enabled hosts.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void sendingEligibility() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_NONE_PROVIDER,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
CdnTestSupport.liveProvider(CdnProviderType.NONE, null);
|
||||
Integer live = CdnTestSupport.liveProvider(CdnProviderType.CLOUDFRONT, null);
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.PENDING,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.HOSTNAME)
|
||||
.withRecord(new QRecord().withValue("id", 2).withValue("purgeEnabled", false)));
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_STAGE_OFF,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(live));
|
||||
new com.kingsrook.qqq.backend.core.actions.tables.UpdateAction().execute(
|
||||
new com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput(CdnTables.PROVIDER)
|
||||
.withRecord(new QRecord().withValue("id", live).withValue("enabled", false)));
|
||||
assertEquals(com.kingsrook.qbits.cdn.model.CdnPurgeStatus.SKIPPED_NONE_PROVIDER,
|
||||
com.kingsrook.qbits.cdn.purge.CdnPurgeService.sendingStatus(null));
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without an enabled provider, do not claim a request row was recorded.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void absentProvider() throws Exception
|
||||
{
|
||||
CdnTestSupport.setUp();
|
||||
CdnTestSupport.postRule();
|
||||
CdnTestSupport.insert("post", new QRecord().withValue("slug", "one"));
|
||||
List<List<String>> lines = run(CdnProcesses.PURGE_PREFIX + "post", "1");
|
||||
assertThat(lines.get(0)).contains("1 No CDN provider configured: purges are recorded only, not sent.");
|
||||
assertThat(lines.get(1)).containsExactly("1 record had no enabled CDN provider (no purge request recorded)");
|
||||
assertThat(purges()).isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -17,12 +17,11 @@
|
||||
package com.kingsrook.qbits.cdn.purge;
|
||||
|
||||
|
||||
import java.net.http.HttpClient;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport.FakePathProvider;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport.FakeTagProvider;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnException;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
@@ -46,7 +45,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
*******************************************************************************/
|
||||
class PurgePipelineTest
|
||||
{
|
||||
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of()));
|
||||
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(name -> null, host -> List.of()));
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -18,15 +18,19 @@ package com.kingsrook.qbits.cdn.purge;
|
||||
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.http.HttpClient;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerificationException;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerifier;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -46,6 +50,7 @@ class RevalidationHookRunnerTest
|
||||
private final List<String> signatures = new CopyOnWriteArrayList<>();
|
||||
private final AtomicInteger status = new AtomicInteger(200);
|
||||
private HttpServer server;
|
||||
private final List<Map<String, String>> headers = new CopyOnWriteArrayList<>();
|
||||
|
||||
|
||||
|
||||
@@ -59,6 +64,9 @@ class RevalidationHookRunnerTest
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/api/revalidate", exchange ->
|
||||
{
|
||||
Map<String, String> received = new LinkedHashMap<>();
|
||||
exchange.getRequestHeaders().forEach((key, values) -> received.put(key.toLowerCase(java.util.Locale.ROOT), values.get(0)));
|
||||
headers.add(received);
|
||||
bodies.add(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
|
||||
String signature = exchange.getRequestHeaders().getFirst(RevalidationHookRunner.SIGNATURE_HEADER);
|
||||
signatures.add(signature == null ? "none" : signature);
|
||||
@@ -88,13 +96,21 @@ class RevalidationHookRunnerTest
|
||||
void signsAndFansOut() throws Exception
|
||||
{
|
||||
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true).withValue("fanOutDns", true)
|
||||
.withValue("secretRef", "SECRET").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> "SECRET".equals(name) ? "s3cret" : null,
|
||||
.withValue("secretRef", "SECRET").withValue("allowPlainHttp", true).withValue("internalAllowlist", "frontend.invalid").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "SECRET".equals(name) ? "test-secret-at-least-16-bytes" : null,
|
||||
host -> List.of("127.0.0.1", "127.0.0.1"));
|
||||
assertEquals(2, runner.run(List.of("post:1")));
|
||||
assertThat(bodies).hasSize(2).allMatch("{\"tags\":[\"post:1\"]}"::equals);
|
||||
String expected = "sha256=" + RevalidationHookRunner.sign("s3cret", bodies.get(0).getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(signatures).allMatch(expected::equals);
|
||||
WebhookVerifier verifier = new WebhookVerifier(List.of(WebhookSecret.parse("test-secret-at-least-16-bytes")));
|
||||
for(int i = 0; i < bodies.size(); i++)
|
||||
{
|
||||
verifier.verify(headers.get(i), bodies.get(i).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
assertThat(signatures).containsExactly("none", "none");
|
||||
assertThat(headers.get(0).get("webhook-id")).isEqualTo(headers.get(1).get("webhook-id"));
|
||||
assertThat(headers.get(0).get("host")).startsWith("frontend.invalid:");
|
||||
assertThatThrownBy(() -> verifier.verify(headers.get(0), "tampered".getBytes(StandardCharsets.UTF_8)))
|
||||
.isInstanceOf(WebhookVerificationException.class);
|
||||
assertEquals(0, runner.run(List.of()));
|
||||
}
|
||||
|
||||
@@ -108,12 +124,94 @@ class RevalidationHookRunnerTest
|
||||
{
|
||||
status.set(500);
|
||||
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true)
|
||||
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of());
|
||||
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1").withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.run(List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("500");
|
||||
assertThat(signatures).containsExactly("none");
|
||||
|
||||
QRecord fanOut = new QRecord().withValue("name", "empty").withValue("fanOutDns", true).withValue("url", "http://x.invalid/api/revalidate");
|
||||
assertThatThrownBy(() -> runner.call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
|
||||
assertThatThrownBy(() -> new RevalidationHookRunner(name -> null, host -> List.of()).call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Legacy mode preserves fixed signatures, including whitespace and whsec_ keys.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void legacyGoldenVectors() throws Exception
|
||||
{
|
||||
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
|
||||
assertThat(signatures.get(0)).isEqualTo("sha256=540457336a45ef962b165f8732f71ebf327ffa67c0cb1047c446df0231fa0db2");
|
||||
new RevalidationHookRunner(name -> " whsec_test-key-with-spaces ", host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
|
||||
assertThat(signatures.get(1)).isEqualTo("sha256=a4dd022bffe6f799633102c415fdc18e22f078bfc98357f909b5df58d5318b05");
|
||||
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
|
||||
.call(hook(), List.of("post:1"));
|
||||
assertThat(signatures.get(2)).as("a null format (existing rows) keeps the legacy signature").isEqualTo(signatures.get(0));
|
||||
assertThat(bodies).containsExactly("{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}");
|
||||
assertThat(headers).allSatisfy(value -> assertThat(value).doesNotContainKeys("webhook-id", "webhook-timestamp", "webhook-signature"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An explicit standard format accepts encoded secrets.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void explicitStandardFormat() throws Exception
|
||||
{
|
||||
WebhookSecret secret = WebhookSecret.generate();
|
||||
new RevalidationHookRunner(name -> secret.reveal(), host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"));
|
||||
new WebhookVerifier(List.of(secret)).verify(headers.get(0), bodies.get(0).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalid format and short standard secrets fail before sending.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void invalidSigningConfiguration()
|
||||
{
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "short", host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("at least 16 bytes");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "UNKNOWN"), List.of("post:1"))).isInstanceOf(QException.class);
|
||||
assertThat(bodies).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Safe defaults reject HTTP, private addresses and mixed fan-out before sending.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void safeHttpPolicy()
|
||||
{
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("allowPlainHttp", null), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Only https");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("internalAllowlist", null), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
|
||||
RevalidationHookRunner mixed = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1", "169.254.169.254"));
|
||||
assertThatThrownBy(() -> mixed.call(hook().withValue("fanOutDns", true), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("timeoutMs", 0), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("positive");
|
||||
assertThat(bodies).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A local test hook explicitly opts into HTTP and loopback.
|
||||
*******************************************************************************/
|
||||
private QRecord hook()
|
||||
{
|
||||
return (new QRecord().withValue("name", "test").withValue("secretRef", "SECRET")
|
||||
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1")
|
||||
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user