Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
32e6146e08
|
||
|
|
18b3b64839
|
||
|
|
4993c0f837
|
||
|
|
710f480549
|
||
|
|
5f87b54ae3 | ||
|
|
2d24bf8bf2 | ||
|
|
ba23ccebbe | ||
|
|
29a39a491c |
No files matched your search
+11
-57
@@ -1,60 +1,14 @@
|
||||
version: 2.1
|
||||
|
||||
setup: true
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.8
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
workflows:
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release\/.*)/
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only: [develop]
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only: [/release\/.*/]
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [/^v\d+\.\d+\.\d+$/]
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
@@ -0,0 +1,25 @@
|
||||
# Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository.
|
||||
name: qrun-release
|
||||
on:
|
||||
push: {}
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
jobs:
|
||||
release:
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/java-docker.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
docker-runner: container-build
|
||||
browsers: true
|
||||
browser-runner: browser-build
|
||||
postpublish:
|
||||
if: ${{ needs.release.outputs.capture == 'true' }}
|
||||
needs: release
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
uses: self:KofTwentyTwo/platform-ci/.gitea/workflows/qrun-source-version.yaml@qrun-release-v1
|
||||
with:
|
||||
runner: linux-small
|
||||
@@ -0,0 +1,23 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
|
||||
@@ -1,25 +0,0 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release-no-hotfix
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
hotfix: true
|
||||
asciidoc: false
|
||||
@@ -35,7 +35,7 @@ This file provides guidance to Codex when working with code in this repository.
|
||||
## Dependencies
|
||||
|
||||
- Java 21.
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.5` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
|
||||
- Parent `com.kingsrook:qbit-build-parent:2.0.0` (qqq 4.0.0). The default-active `qqq-snapshot` profile imports `qqq-bom-pom:4.1.0-RC.4` ahead of it, as in qbit-quick-search (ADR-0007). Remove that activation when the parent moves to 4.1.
|
||||
- Tests: see pom.xml.
|
||||
|
||||
## Release
|
||||
|
||||
@@ -4,11 +4,6 @@ All notable changes to this project are documented here. The format follows Keep
|
||||
|
||||
## [0.1.1] - Unreleased
|
||||
|
||||
### Changed
|
||||
- Require released QQQ BOM 4.1.0-RC.5; revalidation hooks default to core Standard Webhooks signing. Per-hook `signatureFormat=LEGACY` preserves the prior header and signature bytes.
|
||||
- Use core `SafeHttpClient` for revalidation requests: HTTPS/public targets by default, explicit `allowPlainHttp`/`internalAllowlist` exceptions, bounded responses and checked DNS pinning. Fan-out preserves Host/TLS identity. Host schema migration and receiver migration are described in README and `docs/revalidation-hook-migration.md`.
|
||||
|
||||
|
||||
### Fixed
|
||||
- Tag rule `{prefix.field}` placeholders now resolve the joined table from the `<prefix>Id` field's possible-value source, so prefixed tables (`cms_blog`) work; a missing joined table is logged instead of silently skipped.
|
||||
|
||||
|
||||
@@ -58,36 +58,6 @@ CloudFront credentials: `<secretRef>_ACCESS_KEY_ID` / `<secretRef>_SECRET_ACCESS
|
||||
Drift: the OpenTofu `expected_config` JSON in `CDN_EXPECTED_CONFIG` (or the
|
||||
variable named by the provider setting `expectedConfigEnv`).
|
||||
|
||||
## Revalidation hooks
|
||||
|
||||
Requires QQQ **4.1.0-RC.5** (the default `qqq-snapshot` profile imports the released BOM;
|
||||
its historical property name does not select a core SNAPSHOT).
|
||||
|
||||
New `cdn_revalidation_hook` rows default `signatureFormat` to `STANDARD_WEBHOOKS`;
|
||||
existing rows with a null format keep the `LEGACY` signature. Signed requests use core `WebhookSigner` and
|
||||
`WebhookSecret`: `webhook-id`, `webhook-timestamp`, and `webhook-signature`.
|
||||
Rows on `LEGACY` (or null) keep sending
|
||||
`X-Revalidate-Signature: sha256=<lowercase hex HMAC-SHA256 of the body>`.
|
||||
Legacy keys remain the exact UTF-8 environment value, with no trimming or decoding.
|
||||
Standard keys must contain at least 16 bytes after core parsing (trimmed raw text,
|
||||
or base64-decoded `whsec_` text). `secretRef` names the environment variable;
|
||||
as before, no resolved secret means no signature headers.
|
||||
|
||||
Core `SafeHttpClient` now enforces HTTPS and rejects internal addresses by default.
|
||||
For an internal HTTP pod endpoint explicitly set `allowPlainHttp=true` and
|
||||
`internalAllowlist` to its exact hostname, a `.suffix`, IP, or CIDR (comma/whitespace
|
||||
separated). Metadata and unspecified addresses remain blocked. Every DNS address
|
||||
is checked before fan-out; the original hostname is retained for Host/TLS, and
|
||||
connections use the checked addresses without resolving again. Redirects fail,
|
||||
responses are bounded to 64 KiB, and `timeoutMs` is a positive per-request total
|
||||
limit (default 5000). These transport rules apply to both signature formats.
|
||||
|
||||
Host apps must migrate `cdn_revalidation_hook` to add `signatureFormat` (string),
|
||||
`allowPlainHttp` (boolean), and `internalAllowlist` (text). Backfill `LEGACY` for
|
||||
old receivers; configure internal network exceptions explicitly before dispatch.
|
||||
See [receiver migration](docs/revalidation-hook-migration.md) for the inspected
|
||||
KoF22 receiver and exact changes required.
|
||||
|
||||
## Build
|
||||
|
||||
```sh
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
# Revalidation hook migration (F11 / qbit-cdn #15)
|
||||
|
||||
Reference: core source read directly at tag `v4.1.0-RC.5`, including
|
||||
`WebhookSigner`, `WebhookSecret`, `WebhookVerifier`, `SafeHttpClient`,
|
||||
`SafeHttpPolicy`, and `AddressRules`. No temporary core bridge is needed.
|
||||
|
||||
## Receiver found locally
|
||||
|
||||
The plain `Website-Frontend` and `Website-Publisher` checkouts have no match for
|
||||
`X-Revalidate-Signature`. The implementation is in
|
||||
`R:/Git.Local/KofTwentyTwo/Website-Frontend-wt-isr`, branch `feature/cdn-isr`,
|
||||
HEAD `476318b` at inspection (working files inspected):
|
||||
|
||||
- `src/lib/revalidate.ts:5`: names `x-revalidate-signature`.
|
||||
- `src/lib/revalidate.ts:9`: `signBody` returns `sha256=` plus hex HMAC-SHA256
|
||||
over the body using the raw string key (the calculation is line 10).
|
||||
- `src/lib/revalidate.ts:14`: `verifySignature` compares this value in constant
|
||||
time; line 21 lowercases the incoming legacy header.
|
||||
- `src/app/api/revalidate/route.ts:30`: reads `CDN_REVALIDATE_SECRET`.
|
||||
- `src/app/api/revalidate/route.ts:36`: reads the body before JSON parsing.
|
||||
- `src/app/api/revalidate/route.ts:40`: reads only the legacy signature header
|
||||
and calls the verifier; standard requests currently return 401.
|
||||
- `src/__tests__/revalidate-route.test.ts:12`: test requests send only the legacy header.
|
||||
|
||||
These receiver files were inspected, not modified.
|
||||
|
||||
## Deployment and receiver changes
|
||||
|
||||
1. Add the three hook columns documented in README. Existing rows (null
|
||||
`signatureFormat`) keep the LEGACY signature; switch a hook to `STANDARD_WEBHOOKS`
|
||||
after its receiver verifies Standard Webhooks. New rows default to `STANDARD_WEBHOOKS`.
|
||||
For internal HTTP endpoints also explicitly allow HTTP and the intended host.
|
||||
2. Update `src/lib/revalidate.ts:14` to verify Standard Webhooks: require all
|
||||
three headers, validate an integer Unix timestamp within 300 seconds of the
|
||||
receiver clock (reject malformed/non-finite values), then HMAC-SHA256 the
|
||||
bytes `<webhook-id>.<webhook-timestamp>.<raw body>` and compare decoded
|
||||
signatures in constant time. Accept any matching `v1,<base64>` entry in the
|
||||
space-separated signature list. Do not lowercase base64 signatures.
|
||||
Match core secret parsing: trim text; decode `whsec_` base64 keys, otherwise
|
||||
UTF-8; use at least 16 key bytes. Rotate short existing keys on both sides.
|
||||
3. At `src/app/api/revalidate/route.ts:36`, retain raw body bytes for verification;
|
||||
at line 40 pass all three headers to the new verifier before parsing JSON.
|
||||
Update its header import/documentation (lines 5 and 22). During migration the
|
||||
receiver may explicitly support both formats; do not fall back to legacy
|
||||
when standard headers are present but invalid.
|
||||
4. Update `src/__tests__/revalidate-route.test.ts:12` and helper tests to cover
|
||||
valid standard requests, altered body/id/timestamp, missing headers, stale
|
||||
or future timestamps, malformed timestamps, encoded keys, and legacy mode.
|
||||
5. Deploy the receiver, then switch its hook to `STANDARD_WEBHOOKS`.
|
||||
|
||||
The JSON payload remains `{"tags":[...]}`. One UUID identifies a hook invocation,
|
||||
shared across its fan-out requests; each request gets its own current timestamp.
|
||||
A later dispatcher invocation generates a new UUID. Revalidation remains
|
||||
idempotent, so receivers must not rely on this ID to deduplicate purge retries;
|
||||
any optional deduplication must remain local to each pod, not suppress other pods.
|
||||
|
||||
## Compatibility and verification
|
||||
|
||||
The legacy HMAC helper remains solely for the explicitly supported legacy format,
|
||||
not as a temporary replacement for core. Golden tests use fixed digests calculated
|
||||
independently of that helper, including a short key and a whitespace/prefix key.
|
||||
Standard requests are checked using RC.5 `WebhookVerifier`; transport tests cover
|
||||
HTTP/private-address refusal, explicit local exceptions and mixed-address fan-out.
|
||||
|
||||
The test/collaborator constructor now takes `(secrets, resolver)`; callers no longer
|
||||
inject `java.net.http.HttpClient`. The production no-argument constructor remains.
|
||||
The only existing test adaptation outside the runner tests is `PurgePipelineTest`
|
||||
using this constructor. The RC.5 bump required no unrelated code fixes.
|
||||
@@ -18,11 +18,11 @@
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
|
||||
<properties>
|
||||
<revision>0.1.0-RC.1</revision>
|
||||
<revision>0.1.0</revision>
|
||||
<maven.compiler.source>21</maven.compiler.source>
|
||||
<maven.compiler.target>21</maven.compiler.target>
|
||||
<jacoco.version>0.8.15</jacoco.version>
|
||||
<aws.sdk.version>2.41.34</aws.sdk.version>
|
||||
<aws.sdk.version>2.41.10</aws.sdk.version>
|
||||
</properties>
|
||||
|
||||
<parent>
|
||||
@@ -192,7 +192,7 @@
|
||||
<activeByDefault>true</activeByDefault>
|
||||
</activation>
|
||||
<properties>
|
||||
<qqq.snapshot.version>4.1.0-RC.5</qqq.snapshot.version>
|
||||
<qqq.snapshot.version>4.1.0-RC.4</qqq.snapshot.version>
|
||||
</properties>
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.model;
|
||||
|
||||
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Signature format selected on each revalidation hook.
|
||||
*******************************************************************************/
|
||||
public enum CdnSignatureFormat implements PossibleValueEnum<String>
|
||||
{
|
||||
STANDARD_WEBHOOKS("STANDARD_WEBHOOKS", "Standard Webhooks"),
|
||||
LEGACY("LEGACY", "Legacy X-Revalidate-Signature");
|
||||
|
||||
private final String id;
|
||||
private final String label;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Constructor.
|
||||
*******************************************************************************/
|
||||
CdnSignatureFormat(String id, String label)
|
||||
{
|
||||
this.id = id;
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for possibleValueId.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getPossibleValueId()
|
||||
{
|
||||
return (id);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Getter for possibleValueLabel.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public String getPossibleValueLabel()
|
||||
{
|
||||
return (label);
|
||||
}
|
||||
}
|
||||
@@ -54,12 +54,11 @@ public final class CdnTables
|
||||
public static final List<String> ALL = List.of(PROVIDER, HOSTNAME, CACHE_POLICY, TABLE_TAG_RULE, PURGE_REQUEST, REVALIDATION_HOOK,
|
||||
EDGE_STAT_DAILY, ORIGIN_CHECK, DRIFT_FINDING);
|
||||
|
||||
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
|
||||
public static final String PVS_STAGE = "cdn_hostname_stage";
|
||||
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
|
||||
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
|
||||
public static final String PVS_SIGNATURE_FORMAT = "cdn_signature_format";
|
||||
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
|
||||
public static final String PVS_PROVIDER_TYPE = "cdn_provider_type";
|
||||
public static final String PVS_STAGE = "cdn_hostname_stage";
|
||||
public static final String PVS_PURGE_STATUS = "cdn_purge_status";
|
||||
public static final String PVS_PURGE_SOURCE = "cdn_purge_source";
|
||||
public static final String PVS_POLICY_CLASS = "cdn_policy_class";
|
||||
|
||||
|
||||
|
||||
@@ -96,7 +95,6 @@ public final class CdnTables
|
||||
QPossibleValueSource.newForEnum(PVS_PURGE_STATUS, CdnPurgeStatus.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_PURGE_SOURCE, CdnPurgeSource.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_POLICY_CLASS, CdnPolicyClass.values()),
|
||||
QPossibleValueSource.newForEnum(PVS_SIGNATURE_FORMAT, CdnSignatureFormat.values()),
|
||||
QPossibleValueSource.newForTable(PROVIDER),
|
||||
QPossibleValueSource.newForTable(HOSTNAME)));
|
||||
}
|
||||
@@ -320,9 +318,6 @@ public final class CdnTables
|
||||
.withField(new QFieldMetaData("url", QFieldType.STRING).withIsRequired(true).withMaxLength(500))
|
||||
.withField(new QFieldMetaData("fanOutDns", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData("secretRef", QFieldType.STRING).withMaxLength(100))
|
||||
.withField(new QFieldMetaData("signatureFormat", QFieldType.STRING).withPossibleValueSourceName(PVS_SIGNATURE_FORMAT).withDefaultValue(CdnSignatureFormat.STANDARD_WEBHOOKS.name()))
|
||||
.withField(new QFieldMetaData("allowPlainHttp", QFieldType.BOOLEAN).withDefaultValue(false))
|
||||
.withField(new QFieldMetaData("internalAllowlist", QFieldType.TEXT))
|
||||
.withField(new QFieldMetaData("timeoutMs", QFieldType.INTEGER).withDefaultValue(5000))
|
||||
.withField(new QFieldMetaData("enabled", QFieldType.BOOLEAN).withDefaultValue(true)));
|
||||
}
|
||||
|
||||
@@ -21,18 +21,16 @@ import javax.crypto.Mac;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.net.InetAddress;
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.HttpRequest;
|
||||
import java.net.http.HttpResponse;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HexFormat;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.function.Function;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.kingsrook.qbits.cdn.model.CdnSignatureFormat;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
@@ -43,20 +41,17 @@ import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.StringUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
|
||||
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpClient;
|
||||
import com.kingsrook.qqq.backend.core.utils.http.SafeHttpPolicy;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSigner;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Calls every enabled cdn_revalidation_hook before the CDN is purged, so the
|
||||
** edge never refetches a stale page (design spec section 4.5 step 3).
|
||||
**
|
||||
** Request: POST {"tags":[...]} signed with core Standard Webhooks headers.
|
||||
** LEGACY keeps X-Revalidate-Signature with a hex HMAC of the body. The
|
||||
** secretRef names an environment variable. SafeHttpClient checks and pins
|
||||
** every address; fan-out preserves the original Host and TLS server name.
|
||||
** Request: POST {"tags":[...]} with header
|
||||
** "X-Revalidate-Signature: sha256=<hex HMAC-SHA256 of the body>", keyed by
|
||||
** the secret in the environment variable the hook's secretRef names. With
|
||||
** fanOutDns, the hook's host is resolved and every address is called (one
|
||||
** per pod behind a headless Service). Every call must answer 2xx.
|
||||
*******************************************************************************/
|
||||
public class RevalidationHookRunner
|
||||
{
|
||||
@@ -64,8 +59,9 @@ public class RevalidationHookRunner
|
||||
|
||||
private static final ObjectMapper JSON = new ObjectMapper();
|
||||
|
||||
private final Function<String, String> secrets;
|
||||
private final Resolver resolver;
|
||||
private final HttpClient http;
|
||||
private final Function<String, String> secrets;
|
||||
private final Resolver resolver;
|
||||
|
||||
|
||||
|
||||
@@ -87,7 +83,7 @@ public class RevalidationHookRunner
|
||||
*******************************************************************************/
|
||||
public RevalidationHookRunner()
|
||||
{
|
||||
this(System::getenv,
|
||||
this(HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(), System::getenv,
|
||||
host -> java.util.Arrays.stream(InetAddress.getAllByName(host)).map(InetAddress::getHostAddress).toList());
|
||||
}
|
||||
|
||||
@@ -96,8 +92,9 @@ public class RevalidationHookRunner
|
||||
/*******************************************************************************
|
||||
** Constructor with collaborators.
|
||||
*******************************************************************************/
|
||||
public RevalidationHookRunner(Function<String, String> secrets, Resolver resolver)
|
||||
public RevalidationHookRunner(HttpClient http, Function<String, String> secrets, Resolver resolver)
|
||||
{
|
||||
this.http = http;
|
||||
this.secrets = secrets;
|
||||
this.resolver = resolver;
|
||||
}
|
||||
@@ -136,63 +133,42 @@ public class RevalidationHookRunner
|
||||
byte[] body = JSON.writeValueAsBytes(java.util.Map.of("tags", tags));
|
||||
String secretRef = hook.getValueString("secretRef");
|
||||
String secret = StringUtils.hasContent(secretRef) ? secrets.apply(secretRef) : null;
|
||||
String formatValue = hook.getValueString("signatureFormat");
|
||||
CdnSignatureFormat format = formatValue == null ? CdnSignatureFormat.LEGACY : CdnSignatureFormat.valueOf(formatValue);
|
||||
WebhookSecret signingSecret = secret != null && format == CdnSignatureFormat.STANDARD_WEBHOOKS ? WebhookSecret.parse(secret) : null;
|
||||
String messageId = UUID.randomUUID().toString();
|
||||
URI uri = URI.create(hook.getValueString("url"));
|
||||
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
|
||||
if(timeout != null && timeout <= 0)
|
||||
List<URI> targets = new ArrayList<>();
|
||||
if(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns"))))
|
||||
{
|
||||
throw (new QException("Revalidation hook timeoutMs must be positive"));
|
||||
for(String address : resolver.resolve(uri.getHost()))
|
||||
{
|
||||
String host = address.contains(":") ? "[" + address + "]" : address;
|
||||
targets.add(new URI(uri.getScheme(), null, host, uri.getPort(), uri.getPath(), uri.getQuery(), null));
|
||||
}
|
||||
}
|
||||
String allowlist = hook.getValueString("internalAllowlist");
|
||||
SafeHttpPolicy policy = SafeHttpPolicy.defaults()
|
||||
.withAllowPlainHttp(Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("allowPlainHttp"))))
|
||||
.withInternalAllowlist(allowlist == null ? List.of() : Arrays.asList(allowlist.split("[,\\s]+")))
|
||||
.withTotalTimeout(Duration.ofMillis(timeout == null ? 5000 : timeout));
|
||||
List<String> resolved = resolver.resolve(uri.getHost());
|
||||
InetAddress[] addresses = new InetAddress[resolved.size()];
|
||||
for(int i = 0; i < resolved.size(); i++)
|
||||
else
|
||||
{
|
||||
addresses[i] = InetAddress.getByName(resolved.get(i));
|
||||
targets.add(uri);
|
||||
}
|
||||
if(addresses.length == 0)
|
||||
if(targets.isEmpty())
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " resolved to no addresses"));
|
||||
}
|
||||
try(SafeHttpClient validator = new SafeHttpClient(policy, host -> addresses))
|
||||
Integer timeout = ValueUtils.getValueAsInteger(hook.getValue("timeoutMs"));
|
||||
for(URI target : targets)
|
||||
{
|
||||
validator.checkUrl(uri.toString());
|
||||
}
|
||||
boolean fanOut = Boolean.TRUE.equals(ValueUtils.getValueAsBoolean(hook.getValue("fanOutDns")));
|
||||
int calls = fanOut ? addresses.length : 1;
|
||||
for(int i = 0; i < calls; i++)
|
||||
{
|
||||
InetAddress[] targetAddresses = fanOut ? new InetAddress[] {addresses[i]} : addresses;
|
||||
Map<String, String> headers = new LinkedHashMap<>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
HttpRequest.Builder request = HttpRequest.newBuilder(target)
|
||||
.timeout(Duration.ofMillis(timeout == null ? 5000 : timeout))
|
||||
.header("Content-Type", "application/json")
|
||||
.POST(HttpRequest.BodyPublishers.ofByteArray(body));
|
||||
if(secret != null)
|
||||
{
|
||||
if(format == CdnSignatureFormat.LEGACY)
|
||||
{
|
||||
headers.put(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
|
||||
}
|
||||
else
|
||||
{
|
||||
headers.putAll(WebhookSigner.headers(messageId, Instant.now(), body, List.of(signingSecret)));
|
||||
}
|
||||
request.header(SIGNATURE_HEADER, "sha256=" + sign(secret, body));
|
||||
}
|
||||
try(SafeHttpClient client = new SafeHttpClient(policy, host -> targetAddresses))
|
||||
HttpResponse<String> response = http.send(request.build(), HttpResponse.BodyHandlers.ofString());
|
||||
if(response.statusCode() / 100 != 2)
|
||||
{
|
||||
int status = client.post(uri.toString(), headers, body).statusCode();
|
||||
if(status / 100 != 2)
|
||||
{
|
||||
throw (new QException("Revalidation hook " + name + " answered " + status + " from " + uri.getHost()));
|
||||
}
|
||||
throw (new QException("Revalidation hook " + name + " answered " + response.statusCode() + " from " + target.getHost()));
|
||||
}
|
||||
}
|
||||
return (calls);
|
||||
return (targets.size());
|
||||
}
|
||||
catch(QException e)
|
||||
{
|
||||
@@ -212,7 +188,7 @@ public class RevalidationHookRunner
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Legacy compatibility: hex HMAC-SHA256 over the body with raw UTF-8 key bytes.
|
||||
** Hex HMAC-SHA256.
|
||||
*******************************************************************************/
|
||||
public static String sign(String secret, byte[] body) throws Exception
|
||||
{
|
||||
|
||||
@@ -17,11 +17,12 @@
|
||||
package com.kingsrook.qbits.cdn.purge;
|
||||
|
||||
|
||||
import java.net.http.HttpClient;
|
||||
import java.util.List;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport.FakePathProvider;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport.FakeTagProvider;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnException;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
@@ -45,7 +46,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
*******************************************************************************/
|
||||
class PurgePipelineTest
|
||||
{
|
||||
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(name -> null, host -> List.of()));
|
||||
private final CdnPurgeDispatcher dispatcher = new CdnPurgeDispatcher(new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of()));
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -18,19 +18,15 @@ package com.kingsrook.qbits.cdn.purge;
|
||||
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.http.HttpClient;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CopyOnWriteArrayList;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookSecret;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerificationException;
|
||||
import com.kingsrook.qqq.backend.core.utils.webhooks.WebhookVerifier;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
@@ -50,7 +46,6 @@ class RevalidationHookRunnerTest
|
||||
private final List<String> signatures = new CopyOnWriteArrayList<>();
|
||||
private final AtomicInteger status = new AtomicInteger(200);
|
||||
private HttpServer server;
|
||||
private final List<Map<String, String>> headers = new CopyOnWriteArrayList<>();
|
||||
|
||||
|
||||
|
||||
@@ -64,9 +59,6 @@ class RevalidationHookRunnerTest
|
||||
server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
|
||||
server.createContext("/api/revalidate", exchange ->
|
||||
{
|
||||
Map<String, String> received = new LinkedHashMap<>();
|
||||
exchange.getRequestHeaders().forEach((key, values) -> received.put(key.toLowerCase(java.util.Locale.ROOT), values.get(0)));
|
||||
headers.add(received);
|
||||
bodies.add(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));
|
||||
String signature = exchange.getRequestHeaders().getFirst(RevalidationHookRunner.SIGNATURE_HEADER);
|
||||
signatures.add(signature == null ? "none" : signature);
|
||||
@@ -96,21 +88,13 @@ class RevalidationHookRunnerTest
|
||||
void signsAndFansOut() throws Exception
|
||||
{
|
||||
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true).withValue("fanOutDns", true)
|
||||
.withValue("secretRef", "SECRET").withValue("allowPlainHttp", true).withValue("internalAllowlist", "frontend.invalid").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "SECRET".equals(name) ? "test-secret-at-least-16-bytes" : null,
|
||||
.withValue("secretRef", "SECRET").withValue("url", "http://frontend.invalid:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> "SECRET".equals(name) ? "s3cret" : null,
|
||||
host -> List.of("127.0.0.1", "127.0.0.1"));
|
||||
assertEquals(2, runner.run(List.of("post:1")));
|
||||
assertThat(bodies).hasSize(2).allMatch("{\"tags\":[\"post:1\"]}"::equals);
|
||||
WebhookVerifier verifier = new WebhookVerifier(List.of(WebhookSecret.parse("test-secret-at-least-16-bytes")));
|
||||
for(int i = 0; i < bodies.size(); i++)
|
||||
{
|
||||
verifier.verify(headers.get(i), bodies.get(i).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
assertThat(signatures).containsExactly("none", "none");
|
||||
assertThat(headers.get(0).get("webhook-id")).isEqualTo(headers.get(1).get("webhook-id"));
|
||||
assertThat(headers.get(0).get("host")).startsWith("frontend.invalid:");
|
||||
assertThatThrownBy(() -> verifier.verify(headers.get(0), "tampered".getBytes(StandardCharsets.UTF_8)))
|
||||
.isInstanceOf(WebhookVerificationException.class);
|
||||
String expected = "sha256=" + RevalidationHookRunner.sign("s3cret", bodies.get(0).getBytes(StandardCharsets.UTF_8));
|
||||
assertThat(signatures).allMatch(expected::equals);
|
||||
assertEquals(0, runner.run(List.of()));
|
||||
}
|
||||
|
||||
@@ -124,94 +108,12 @@ class RevalidationHookRunnerTest
|
||||
{
|
||||
status.set(500);
|
||||
CdnTestSupport.insert(CdnTables.REVALIDATION_HOOK, new QRecord().withValue("name", "next").withValue("enabled", true)
|
||||
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1").withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
|
||||
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(HttpClient.newHttpClient(), name -> null, host -> List.of());
|
||||
assertThatThrownBy(() -> runner.run(List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("500");
|
||||
assertThat(signatures).containsExactly("none");
|
||||
|
||||
QRecord fanOut = new QRecord().withValue("name", "empty").withValue("fanOutDns", true).withValue("url", "http://x.invalid/api/revalidate");
|
||||
assertThatThrownBy(() -> new RevalidationHookRunner(name -> null, host -> List.of()).call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Legacy mode preserves fixed signatures, including whitespace and whsec_ keys.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void legacyGoldenVectors() throws Exception
|
||||
{
|
||||
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
|
||||
assertThat(signatures.get(0)).isEqualTo("sha256=540457336a45ef962b165f8732f71ebf327ffa67c0cb1047c446df0231fa0db2");
|
||||
new RevalidationHookRunner(name -> " whsec_test-key-with-spaces ", host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "LEGACY"), List.of("post:1"));
|
||||
assertThat(signatures.get(1)).isEqualTo("sha256=a4dd022bffe6f799633102c415fdc18e22f078bfc98357f909b5df58d5318b05");
|
||||
new RevalidationHookRunner(name -> "s3cret", host -> List.of("127.0.0.1"))
|
||||
.call(hook(), List.of("post:1"));
|
||||
assertThat(signatures.get(2)).as("a null format (existing rows) keeps the legacy signature").isEqualTo(signatures.get(0));
|
||||
assertThat(bodies).containsExactly("{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}", "{\"tags\":[\"post:1\"]}");
|
||||
assertThat(headers).allSatisfy(value -> assertThat(value).doesNotContainKeys("webhook-id", "webhook-timestamp", "webhook-signature"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An explicit standard format accepts encoded secrets.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void explicitStandardFormat() throws Exception
|
||||
{
|
||||
WebhookSecret secret = WebhookSecret.generate();
|
||||
new RevalidationHookRunner(name -> secret.reveal(), host -> List.of("127.0.0.1"))
|
||||
.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"));
|
||||
new WebhookVerifier(List.of(secret)).verify(headers.get(0), bodies.get(0).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Invalid format and short standard secrets fail before sending.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void invalidSigningConfiguration()
|
||||
{
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> "short", host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "STANDARD_WEBHOOKS"), List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("at least 16 bytes");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("signatureFormat", "UNKNOWN"), List.of("post:1"))).isInstanceOf(QException.class);
|
||||
assertThat(bodies).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Safe defaults reject HTTP, private addresses and mixed fan-out before sending.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void safeHttpPolicy()
|
||||
{
|
||||
RevalidationHookRunner runner = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1"));
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("allowPlainHttp", null), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Only https");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("internalAllowlist", null), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
|
||||
RevalidationHookRunner mixed = new RevalidationHookRunner(name -> null, host -> List.of("127.0.0.1", "169.254.169.254"));
|
||||
assertThatThrownBy(() -> mixed.call(hook().withValue("fanOutDns", true), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("Blocked");
|
||||
assertThatThrownBy(() -> runner.call(hook().withValue("timeoutMs", 0), List.of("post:1")))
|
||||
.isInstanceOf(QException.class).hasMessageContaining("positive");
|
||||
assertThat(bodies).isEmpty();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A local test hook explicitly opts into HTTP and loopback.
|
||||
*******************************************************************************/
|
||||
private QRecord hook()
|
||||
{
|
||||
return (new QRecord().withValue("name", "test").withValue("secretRef", "SECRET")
|
||||
.withValue("allowPlainHttp", true).withValue("internalAllowlist", "127.0.0.1")
|
||||
.withValue("url", "http://127.0.0.1:" + server.getAddress().getPort() + "/api/revalidate"));
|
||||
assertThatThrownBy(() -> runner.call(fanOut, List.of("post:1"))).isInstanceOf(QException.class).hasMessageContaining("no addresses");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user