From 3980e0571911ea438dc08722b21ee06c7e5781e0 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 16:57:57 -0500 Subject: [PATCH 1/3] feat: origin header check, setup card, guided empty states, create-default actions Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + README.md | 4 + UPGRADE.md | 7 + .../kingsrook/qbits/cdn/CdnQBitProducer.java | 3 + .../kingsrook/qbits/cdn/app/CdnHomeData.java | 4 +- .../kingsrook/qbits/cdn/app/CdnHomeView.java | 47 +- .../qbits/cdn/app/CdnSetupStatus.java | 200 ++++++++ .../qbits/cdn/checks/CdnOriginCheck.java | 301 ++++++++++++ .../kingsrook/qbits/cdn/model/CdnHelp.java | 126 +++++ .../kingsrook/qbits/cdn/model/CdnTables.java | 3 +- .../cdn/processes/CdnDefaultsProcesses.java | 431 ++++++++++++++++++ .../processes/CdnOriginCheckProcesses.java | 205 +++++++++ src/main/resources/help/qbit-cdn.yaml | 58 +++ .../kingsrook/qbits/cdn/app/CdnHomeTest.java | 4 +- .../tables/CdnDefaultsAndOriginCheckTest.java | 311 +++++++++++++ .../qbits/cdn/tables/CdnSetupStatusTest.java | 101 ++++ .../cdn/tables/CdnTablesGuardrailTest.java | 189 ++++++++ 17 files changed, 1994 insertions(+), 6 deletions(-) create mode 100644 UPGRADE.md create mode 100644 src/main/java/com/kingsrook/qbits/cdn/app/CdnSetupStatus.java create mode 100644 src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java create mode 100644 src/main/java/com/kingsrook/qbits/cdn/model/CdnHelp.java create mode 100644 src/main/java/com/kingsrook/qbits/cdn/processes/CdnDefaultsProcesses.java create mode 100644 src/main/java/com/kingsrook/qbits/cdn/processes/CdnOriginCheckProcesses.java create mode 100644 src/main/resources/help/qbit-cdn.yaml create mode 100644 src/test/java/com/kingsrook/qbits/cdn/tables/CdnDefaultsAndOriginCheckTest.java create mode 100644 src/test/java/com/kingsrook/qbits/cdn/tables/CdnSetupStatusTest.java create mode 100644 src/test/java/com/kingsrook/qbits/cdn/tables/CdnTablesGuardrailTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index a3daa8f..3f15348 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project are documented here. The format follows Keep ## [0.1.1] - Unreleased +### Added +- Origin header check: Check Origin Headers on Hostnames (and a nightly run over hostnames that are not Off) asks the origin for sample pages and writes `cdn_origin_check` rows. The table had no writer before. +- Guided empty states: help bundle `help/qbit-cdn.yaml` (app and all nine tables, with "How rows get created") applied by `CdnHelp`, and a Setup card on the CDN home. +- "Create default provider" (No CDN), "Create default policies" (two turned-off examples) and "Create default rules" (a tags-only rule per purgeable table) table actions; all insert-only. +- `hostnameRole` is documented as a label that changes nothing; cache policies are documented as applying only where the application calls `CdnResponseHeaderPolicy`. + ### Changed - Require released QQQ BOM 4.1.0-RC.5; revalidation hooks default to core Standard Webhooks signing. Per-hook `signatureFormat=LEGACY` preserves the prior header and signature bytes. - Use core `SafeHttpClient` for revalidation requests: HTTPS/public targets by default, explicit `allowPlainHttp`/`internalAllowlist` exceptions, bounded responses and checked DNS pinning. Fan-out preserves Host/TLS identity. Host schema migration and receiver migration are described in README and `docs/revalidation-hook-migration.md`. diff --git a/README.md b/README.md index f267167..fef5438 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,10 @@ Design: QRun-IO/qqq `docs/specs/2026-10-12-qbit-cdn-design.md`. NONE. See `docs/cloudfront-expected-config.md`. - **Permissions:** `cdn.view`, `cdn.purge`, `cdn.purgeAll`, `cdn.admin`. +## Setup + +This qbit ships no DDL and seeds nothing from a changelog (a host chooses its own column names). Open the CDN app: the Setup card lists each part with the next step. On a fresh install run **Create default provider** (No CDN: purges are recorded, nothing is sent), **Create default rules** (tags for every purgeable table), and optionally **Create default policies** (two turned-off examples). Add your CDN and hostnames when you have them. Cache policies change a response only where the application calls `CdnResponseHeaderPolicy.decide`. **Check Origin Headers** on a hostname (and the nightly run) asks the origin, the provider's `originHost` when set, for the home page, prefix policy paths and literal tag-rule paths, and records how many have the right Cache-Control. + ## Tag rules `cdn_table_tag_rule.onlyWhenFieldsChange` is a comma-separated list of QQQ field diff --git a/UPGRADE.md b/UPGRADE.md new file mode 100644 index 0000000..f86de9c --- /dev/null +++ b/UPGRADE.md @@ -0,0 +1,7 @@ +# Upgrading + +## Unreleased + +- No schema change. Hosts get the help text, the Setup card and the new actions automatically. +- The nightly origin check runs on hosts with a scheduler (02:45 UTC) over hostnames that are not Off. It makes HTTPS requests to the provider's origin host (or the hostname) for a few pages; allow that egress or leave hostnames Off. +- Run Create default provider on hosts that have no provider if you want purges recorded as would-be purges. diff --git a/src/main/java/com/kingsrook/qbits/cdn/CdnQBitProducer.java b/src/main/java/com/kingsrook/qbits/cdn/CdnQBitProducer.java index e10f0c6..fcba30a 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/CdnQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/cdn/CdnQBitProducer.java @@ -149,6 +149,8 @@ public class CdnQBitProducer implements QBitMetaDataProducer } List processes = new ArrayList<>(CdnProcesses.processes(qBitConfig)); processes.addAll(com.kingsrook.qbits.cdn.processes.CdnProviderProcesses.processes(qBitConfig)); + processes.addAll(com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses.processes(qBitConfig)); + processes.addAll(com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses.processes()); for(QProcessMetaData process : processes) { process.setSourceQBitName(qBit.getName()); @@ -160,6 +162,7 @@ public class CdnQBitProducer implements QBitMetaDataProducer } output.add(CdnWidgets.app(qBitConfig, tables)); output.addSelfToInstance(qInstance); + com.kingsrook.qbits.cdn.model.CdnHelp.apply(qInstance, qBitConfig.getAppName()); if(!qBitConfig.getPurgeableTables().isEmpty()) { diff --git a/src/main/java/com/kingsrook/qbits/cdn/app/CdnHomeData.java b/src/main/java/com/kingsrook/qbits/cdn/app/CdnHomeData.java index 787ca9d..8353365 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/app/CdnHomeData.java +++ b/src/main/java/com/kingsrook/qbits/cdn/app/CdnHomeData.java @@ -50,7 +50,7 @@ import com.kingsrook.qqq.backend.core.utils.ValueUtils; *******************************************************************************/ public record CdnHomeData(List providers, List hostnames, List stats, List recentPurges, int failed24h, int dead24h, int failed7d, int dead7d, List budgets, List originChecks, - Map openDriftByArea, Instant lastDriftCheck, Instant now) + Map openDriftByArea, Instant lastDriftCheck, Instant now, List setup) { public static final int ROTATION_WARNING_DAYS = 75; public static final int ROTATION_DAYS = 90; @@ -237,7 +237,7 @@ public record CdnHomeData(List providers, List hostn } return (new CdnHomeData(providers, hostnames, stats, recent, failed24, dead24, failed7, dead7, budgets, new ArrayList<>(latestOrigin.values()), - drift, lastDrift, now)); + drift, lastDrift, now, CdnSetupStatus.load())); } diff --git a/src/main/java/com/kingsrook/qbits/cdn/app/CdnHomeView.java b/src/main/java/com/kingsrook/qbits/cdn/app/CdnHomeView.java index 83cd0af..6da31c2 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/app/CdnHomeView.java +++ b/src/main/java/com/kingsrook/qbits/cdn/app/CdnHomeView.java @@ -61,6 +61,7 @@ public final class CdnHomeView { StringBuilder html = new StringBuilder("
"); connection(html, data, appPath); + setup(html, data, appPath); performance(html, data); purges(html, data, appPath); budget(html, data); @@ -217,6 +218,50 @@ public final class CdnHomeView + /******************************************************************************* + ** The Setup card: each configuration table, whether it is set up, and the next + ** step, linked to its table. + *******************************************************************************/ + private static void setup(StringBuilder html, CdnHomeData data, String appPath) + { + if(data.setup() == null || data.setup().isEmpty()) + { + return; + } + open(html, "Setup"); + long todo = data.setup().stream().filter(i -> i.state() == CdnSetupStatus.State.TODO).count(); + html.append("

").append(todo == 0 ? "Everything the CDN needs is set up." : esc(todo + (todo == 1 ? " item needs" : " items need") + " attention.")).append("

    "); + for(CdnSetupStatus.Item item : data.setup()) + { + String color = switch(item.state()) + { + case OK -> "#16a34a"; + case TODO -> "#dc2626"; + case DEFAULT -> "#d97706"; + case OPTIONAL -> "#9ca3af"; + }; + String label = switch(item.state()) + { + case OK -> "Set up"; + case TODO -> "Needs setup"; + case DEFAULT -> "Using defaults"; + case OPTIONAL -> "Optional"; + }; + html.append("
  • ").append(esc(item.label())).append(" ") + .append(" ").append(esc(label)).append("
    ").append(esc(item.detail())).append("
    "); + if(item.nextStep() != null) + { + html.append("
    Next: ").append(esc(item.nextStep())).append("
    "); + } + html.append("
    • "); + } + html.append("
"); + close(html); + } + + + /******************************************************************************* ** Origin header card. *******************************************************************************/ @@ -225,7 +270,7 @@ public final class CdnHomeView open(html, "Origin headers"); if(data.originChecks().isEmpty()) { - empty(html, "Origin header checks are not run by this version; nothing has been checked."); + empty(html, "No origin check yet. Run Check Origin Headers on Hostnames; hostnames that are not Off are also checked every night."); } for(QRecord check : data.originChecks()) { diff --git a/src/main/java/com/kingsrook/qbits/cdn/app/CdnSetupStatus.java b/src/main/java/com/kingsrook/qbits/cdn/app/CdnSetupStatus.java new file mode 100644 index 0000000..b76cad2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/app/CdnSetupStatus.java @@ -0,0 +1,200 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.app; + + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import com.kingsrook.qbits.cdn.CdnQBitConfig; +import com.kingsrook.qbits.cdn.CdnRuntime; +import com.kingsrook.qbits.cdn.model.CdnHostnameStage; +import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qbits.cdn.spi.CdnProviderType; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** The Setup card on the CDN home: each configuration table, whether it is set up, + ** and the next step. Reads the small configuration tables the same way the runtime + ** does (enabled rows only where the runtime filters on enabled, the application's + ** purgeable tables from the qbit config). + *******************************************************************************/ +public final class CdnSetupStatus +{ + /******************************************************************************* + ** OK: in place. TODO: needs doing. DEFAULT: working on defaults. OPTIONAL: off + ** until set up. + *******************************************************************************/ + public enum State + { + OK, TODO, DEFAULT, OPTIONAL + } + + + + /******************************************************************************* + ** One line of the card. + *******************************************************************************/ + public record Item(String label, String tableName, State state, String detail, String nextStep) + { + } + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private CdnSetupStatus() + { + } + + + + /******************************************************************************* + ** Build the items from the tables. + *******************************************************************************/ + public static List load() throws QException + { + CdnQBitConfig config = CdnRuntime.getConfig(); + List providers = all(CdnTables.PROVIDER); + List hostnames = all(CdnTables.HOSTNAME); + List policies = all(CdnTables.CACHE_POLICY); + List rules = all(CdnTables.TABLE_TAG_RULE); + List hooks = all(CdnTables.REVALIDATION_HOOK); + List checks = all(CdnTables.ORIGIN_CHECK); + return (build(providers, hostnames, policies, rules, hooks, checks, config == null ? List.of() : config.getPurgeableTables())); + } + + + + /******************************************************************************* + ** The items for given rows (no queries; for tests and reuse). + *******************************************************************************/ + public static List build(List providers, List hostnames, List policies, List rules, List hooks, + List checks, List purgeableTables) + { + List items = new ArrayList<>(); + + ////////////// + // provider // + ////////////// + List enabledProviders = providers.stream().filter(p -> !Boolean.FALSE.equals(p.getValueBoolean("enabled"))).toList(); + List real = enabledProviders.stream().filter(p -> !CdnProviderType.NONE.name().equals(p.getValueString("providerType"))).toList(); + if(providers.isEmpty()) + { + items.add(new Item("Providers", CdnTables.PROVIDER, State.TODO, "No provider: changes are not even recorded as purges.", + "Run Create default provider (No CDN: records purges, sends nothing), or add your CDN with New.")); + } + else if(enabledProviders.isEmpty()) + { + items.add(new Item("Providers", CdnTables.PROVIDER, State.TODO, "Every provider is turned off.", "Turn one on (Enabled).")); + } + else if(real.isEmpty()) + { + items.add(new Item("Providers", CdnTables.PROVIDER, State.DEFAULT, "Only No CDN: purges are recorded and nothing is sent.", + "Add your CDN with New (type, distribution id, and the environment variable that holds its key), then Test Connection.")); + } + else + { + boolean failing = real.stream().anyMatch(p -> "FAILED".equals(p.getValueString("lastHealthStatus"))); + items.add(failing + ? new Item("Providers", CdnTables.PROVIDER, State.TODO, "A provider's last connection test failed.", "Check its key variable and run Test Connection.") + : new Item("Providers", CdnTables.PROVIDER, State.OK, real.size() + " CDN provider" + (real.size() == 1 ? "" : "s") + " turned on.", null)); + } + + /////////////// + // hostnames // + /////////////// + if(hostnames.isEmpty()) + { + items.add(real.isEmpty() + ? new Item("Hostnames", CdnTables.HOSTNAME, State.OPTIONAL, "None: nothing sits behind a CDN yet.", "Add each site hostname once a CDN provider exists.") + : new Item("Hostnames", CdnTables.HOSTNAME, State.TODO, "A CDN is set up but no hostname uses it, so nothing is purged.", "Add the hostnames behind it, at stage Off, and move them to Headers only, then Live.")); + } + else + { + long live = hostnames.stream().filter(h -> CdnHostnameStage.LIVE.name().equals(h.getValueString("stage")) || CdnHostnameStage.LOCKED.name().equals(h.getValueString("stage"))).count(); + items.add(live == 0 + ? new Item("Hostnames", CdnTables.HOSTNAME, State.DEFAULT, hostnames.size() + " hostname" + (hostnames.size() == 1 ? "" : "s") + ", none Live: purges are recorded, not sent.", + "When the CDN serves a hostname, move it to Live.") + : new Item("Hostnames", CdnTables.HOSTNAME, State.OK, live + " of " + hostnames.size() + " hostnames Live.", null)); + } + + //////////////////////// + // cache policies // + //////////////////////// + long enabledPolicies = policies.stream().filter(p -> !Boolean.FALSE.equals(p.getValueBoolean("enabled"))).count(); + items.add(policies.isEmpty() + ? new Item("Cache policies", CdnTables.CACHE_POLICY, State.OPTIONAL, "None: responses keep the headers your application sets.", + "Run Create default policies for examples. Policies only change responses when the application calls CdnResponseHeaderPolicy.") + : enabledPolicies == 0 + ? new Item("Cache policies", CdnTables.CACHE_POLICY, State.DEFAULT, policies.size() + " polic" + (policies.size() == 1 ? "y" : "ies") + ", all turned off.", + "Edit one and turn it on. It takes effect only where the application calls CdnResponseHeaderPolicy.") + : new Item("Cache policies", CdnTables.CACHE_POLICY, State.OK, enabledPolicies + " polic" + (enabledPolicies == 1 ? "y" : "ies") + " turned on (applied where the application calls CdnResponseHeaderPolicy).", null)); + + //////////////////////////////////////// + // tag rules: every purgeable table // + //////////////////////////////////////// + Set covered = new HashSet<>(); + rules.stream().filter(r -> !Boolean.FALSE.equals(r.getValueBoolean("enabled"))).forEach(r -> covered.add(r.getValueString("tableName"))); + List uncovered = purgeableTables.stream().filter(t -> !covered.contains(t)).toList(); + if(purgeableTables.isEmpty()) + { + items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.OPTIONAL, "The application lists no purgeable tables, so record changes purge nothing.", "List them in the qbit config (purgeableTables).")); + } + else if(uncovered.isEmpty()) + { + items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.OK, "Every purgeable table has a rule (" + String.join(", ", purgeableTables) + ").", null)); + } + else + { + items.add(new Item("Tag rules", CdnTables.TABLE_TAG_RULE, State.TODO, "No rule for " + String.join(", ", uncovered) + ": changes to them purge nothing.", + "Run Create default rules (tags only), then add invalidation paths where your CDN purges by path.")); + } + + /////////////////////// + // revalidation hooks // + /////////////////////// + long enabledHooks = hooks.stream().filter(h -> !Boolean.FALSE.equals(h.getValueBoolean("enabled"))).count(); + items.add(enabledHooks == 0 + ? new Item("Revalidation hooks", CdnTables.REVALIDATION_HOOK, State.OPTIONAL, "None.", "Add one if a front end keeps its own cache (for example Next.js) that must refresh before the CDN is purged.") + : new Item("Revalidation hooks", CdnTables.REVALIDATION_HOOK, State.OK, enabledHooks + " hook" + (enabledHooks == 1 ? "" : "s") + " turned on.", null)); + + //////////////////// + // origin checks // + //////////////////// + items.add(checks.isEmpty() + ? new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OPTIONAL, "Never run.", "Run Check Origin Headers on Hostnames. Hostnames that are not Off are also checked every night.") + : new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OK, checks.size() + " check" + (checks.size() == 1 ? "" : "s") + " recorded.", null)); + return (items); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List all(String table) throws QException + { + return (new QueryAction().execute(new QueryInput(table)).getRecords()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java b/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java new file mode 100644 index 0000000..6c5cc83 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java @@ -0,0 +1,301 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.checks; + + +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.cdn.CdnRuntime; +import com.kingsrook.qbits.cdn.headers.CdnResponseHeaderPolicy; +import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.utils.StringUtils; + + +/******************************************************************************* + ** The origin header check (design spec section 3.2): requests a few sample paths + ** of a hostname from its origin and records whether each answers with the + ** Cache-Control the cache policies say it should. One cdn_origin_check row is + ** written per hostname and run: objects checked, correct, missing or wrong, and + ** unreadable (the request failed, or the origin answered 4xx or 5xx). + ** + ** The sample paths are "/", the path of every enabled PREFIX policy that applies + ** to the hostname, and the literal (placeholder-free) invalidation paths of the + ** enabled tag rules, up to MAX_PATHS. The origin is the provider's originHost when + ** it has one, otherwise the hostname itself. Requests are HTTPS GETs, never follow + ** redirects, and refuse hosts that resolve to private, loopback or link-local + ** addresses. + ** + ** A response is correct when it has a Cache-Control header and, when an enabled + ** policy matches the path, the header equals what the policy would send (compared + ** ignoring case and spaces). With no enabled policy for the path, any Cache-Control + ** header is correct: the check proves the origin sets one at all. + *******************************************************************************/ +public final class CdnOriginCheck +{ + public static final int MAX_PATHS = 20; + + private static volatile Fetcher fetcher = new HttpFetcher(); + + + + /******************************************************************************* + ** One origin answer: status and lower-cased header names. + *******************************************************************************/ + public record Response(int status, Map headers) + { + } + + + + /******************************************************************************* + ** How the origin is asked (replaceable in tests). + *******************************************************************************/ + public interface Fetcher + { + /******************************************************************************* + ** GET https://host/path; throws when the origin cannot be read at all. + *******************************************************************************/ + Response fetch(String host, String path) throws IOException; + } + + + + /******************************************************************************* + ** The outcome of checking one hostname. + *******************************************************************************/ + public record Result(String scope, int checked, int correct, int wrong, int unreadable, List problems) + { + } + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private CdnOriginCheck() + { + } + + + + /******************************************************************************* + ** Replace how origins are asked; null restores the real HTTP fetcher. + *******************************************************************************/ + public static void setFetcher(Fetcher newFetcher) + { + fetcher = newFetcher == null ? new HttpFetcher() : newFetcher; + } + + + + /******************************************************************************* + ** The sample paths of a hostname. + *******************************************************************************/ + public static List samplePaths(String hostname) throws QException + { + Set paths = new LinkedHashSet<>(); + paths.add("/"); + for(CdnResponseHeaderPolicy.Policy policy : CdnResponseHeaderPolicy.current()) + { + if("PREFIX".equals(policy.matchType()) && policy.matchValue().startsWith("/") && (policy.hostname() == null || policy.hostname().equalsIgnoreCase(hostname))) + { + paths.add(policy.matchValue()); + } + } + for(QRecord rule : new QueryAction().execute(new QueryInput(CdnTables.TABLE_TAG_RULE) + .withFilter(new QQueryFilter(new QFilterCriteria("enabled", QCriteriaOperator.EQUALS, true)))).getRecords()) + { + String text = rule.getValueString("invalidationPaths"); + for(String line : text == null ? new String[0] : text.split("\\R")) + { + String path = line.strip(); + if(path.startsWith("/") && !path.contains("{") && !path.contains("*")) + { + paths.add(path); + } + } + } + return (new ArrayList<>(paths).subList(0, Math.min(MAX_PATHS, paths.size()))); + } + + + + /******************************************************************************* + ** Check one hostname row (and record nothing; see record()). + *******************************************************************************/ + public static Result check(QRecord hostname) throws QException + { + String name = hostname.getValueString("hostname"); + String origin = originOf(hostname, name); + int correct = 0; + int wrong = 0; + int unread = 0; + List problems = new ArrayList<>(); + List paths = samplePaths(name); + for(String path : paths) + { + Response response; + try + { + response = fetcher.fetch(origin, path); + } + catch(IOException | RuntimeException e) + { + unread++; + problems.add(path + ": could not be read (" + e.getMessage() + ")"); + continue; + } + if(response.status() >= 400) + { + unread++; + problems.add(path + ": the origin answered " + response.status()); + continue; + } + String actual = response.headers().get("cache-control"); + if(!StringUtils.hasContent(actual)) + { + wrong++; + problems.add(path + ": no Cache-Control header"); + continue; + } + CdnResponseHeaderPolicy.Decision decision = CdnResponseHeaderPolicy.decide( + new CdnResponseHeaderPolicy.Request(name, path, null, null, false, false, List.of(), List.of()), null); + if(decision.policyName() != null && !normalize(decision.cacheControl()).equals(normalize(actual))) + { + wrong++; + problems.add(path + ": Cache-Control is \"" + actual + "\" but the policy " + decision.policyName() + " says \"" + decision.cacheControl() + "\""); + } + else + { + correct++; + } + } + return (new Result(name, paths.size(), correct, wrong, unread, problems)); + } + + + + /******************************************************************************* + ** Check a hostname and write its cdn_origin_check row; returns the result. + *******************************************************************************/ + public static Result checkAndRecord(QRecord hostname) throws QException + { + Result result = check(hostname); + new InsertAction().execute(new InsertInput(CdnTables.ORIGIN_CHECK).withRecord(new QRecord() + .withValue("checkDate", CdnRuntime.clock().instant()) + .withValue("scope", result.scope()) + .withValue("objectsChecked", result.checked()) + .withValue("correct", result.correct()) + .withValue("missingOrWrong", result.wrong()) + .withValue("unreadable", result.unreadable()))); + return (result); + } + + + + /******************************************************************************* + ** The host to ask: the provider's origin host, else the hostname. + *******************************************************************************/ + private static String originOf(QRecord hostname, String fallback) throws QException + { + java.io.Serializable providerId = hostname.getValue("cdnProviderId"); + if(providerId != null) + { + for(QRecord provider : new QueryAction().execute(new QueryInput(CdnTables.PROVIDER) + .withFilter(new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, providerId)))).getRecords()) + { + String originHost = provider.getValueString("originHost"); + if(StringUtils.hasContent(originHost)) + { + return (originHost.strip().replaceFirst(":\\d+$", "")); + } + } + } + return (fallback); + } + + + + /******************************************************************************* + ** Lower case, no spaces. + *******************************************************************************/ + private static String normalize(String cacheControl) + { + return (cacheControl == null ? "" : cacheControl.toLowerCase(Locale.ROOT).replace(" ", "")); + } + + + + /******************************************************************************* + ** The real fetcher: HTTPS GET, no redirects, public addresses only. + *******************************************************************************/ + static final class HttpFetcher implements Fetcher + { + private final HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).connectTimeout(Duration.ofSeconds(5)).build(); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public Response fetch(String host, String path) throws IOException + { + for(InetAddress address : InetAddress.getAllByName(host)) + { + if(address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress() || address.isSiteLocalAddress() || address.isMulticastAddress()) + { + throw (new IOException("the host resolves to a non-public address")); + } + } + try + { + HttpResponse response = client.send(HttpRequest.newBuilder(URI.create("https://" + host + path)).timeout(Duration.ofSeconds(10)).GET().build(), + HttpResponse.BodyHandlers.discarding()); + Map headers = new LinkedHashMap<>(); + response.headers().map().forEach((key, values) -> headers.put(key.toLowerCase(Locale.ROOT), String.join(", ", values))); + return (new Response(response.statusCode(), headers)); + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + throw (new IOException("interrupted", e)); + } + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/cdn/model/CdnHelp.java b/src/main/java/com/kingsrook/qbits/cdn/model/CdnHelp.java new file mode 100644 index 0000000..d4dacb0 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/model/CdnHelp.java @@ -0,0 +1,126 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.model; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Map; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Applies help/qbit-cdn.yaml (the QQQ help-bundle format) to the CDN app + ** and its cdn_* tables: what each table is for and how its rows get created, + ** shown on the table screen and when it is empty. + ** STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.8): replace with HelpBundles.register once the pinned QQQ + ** release includes it; the bundle file stays the same. + *******************************************************************************/ +public final class CdnHelp +{ + public static final String RESOURCE = "/help/qbit-cdn.yaml"; + public static final String SLOT = "help"; + public static final String APP_KEY = "app:cdnAdmin;slot:help"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private CdnHelp() + { + } + + + + /******************************************************************************* + ** The bundle's entries by key, validated (fails startup if malformed). + *******************************************************************************/ + public static Map entries() throws QException + { + try(InputStream stream = CdnHelp.class.getResourceAsStream(RESOURCE)) + { + if(stream == null) + { + throw (new IOException("missing resource")); + } + JsonNode bundle = new ObjectMapper(new YAMLFactory()).readTree(stream); + if(!"qbit-cdn".equals(bundle.path("bundle").asText()) || !"QBIT".equals(bundle.path("layer").asText())) + { + throw (new IOException("invalid bundle header")); + } + Map entries = new LinkedHashMap<>(); + for(JsonNode entry : bundle.path("entries")) + { + String key = entry.path("key").asText(); + String summary = entry.path("summary").asText(); + if(key.isBlank() || summary.isBlank() || summary.length() > 120 || entry.path("content").asText().isBlank() || entries.put(key, entry) != null) + { + throw (new IOException("invalid or duplicate entry: " + key)); + } + } + return (entries); + } + catch(IOException | RuntimeException e) + { + throw (new QException("Could not load " + RESOURCE, e)); + } + } + + + + /******************************************************************************* + ** Apply the help to the app and every cdn_* table in the instance. A host's + ** own help in the slot is kept (the bundle only fills it). + *******************************************************************************/ + public static void apply(QInstance qInstance, String appName) throws QException + { + Map entries = entries(); + for(QTableMetaData table : qInstance.getTables().values()) + { + JsonNode entry = entries.get("table:" + table.getName() + ";slot:" + SLOT); + if(entry != null && (table.getHelpContent() == null || !table.getHelpContent().containsKey(SLOT))) + { + table.withHelpContent(SLOT, content(entry)); + } + } + QAppMetaData app = qInstance.getApp(appName); + JsonNode appEntry = entries.get(APP_KEY); + if(app != null && appEntry != null && (app.getHelpContent() == null || !app.getHelpContent().containsKey(SLOT))) + { + app.withHelpContent(SLOT, content(appEntry)); + } + } + + + + /******************************************************************************* + ** Summary as the first paragraph, then the content, as markdown. + *******************************************************************************/ + static QHelpContent content(JsonNode entry) + { + return (new QHelpContent().withContentAsMarkdown(entry.path("summary").asText().strip() + "\n\n" + entry.path("content").asText().strip())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/cdn/model/CdnTables.java b/src/main/java/com/kingsrook/qbits/cdn/model/CdnTables.java index 20aa283..ef8f5fd 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/model/CdnTables.java +++ b/src/main/java/com/kingsrook/qbits/cdn/model/CdnTables.java @@ -231,7 +231,8 @@ public final class CdnTables .withRecordLabelFormatAndFields("%s", "hostname") .withField(new QFieldMetaData("hostname", QFieldType.STRING).withIsRequired(true).withMaxLength(255)) .withField(new QFieldMetaData("cdnProviderId", QFieldType.INTEGER).withIsRequired(true).withPossibleValueSourceName(PROVIDER)) - .withField(new QFieldMetaData("hostnameRole", QFieldType.STRING).withMaxLength(20)) + .withField(new QFieldMetaData("hostnameRole", QFieldType.STRING).withMaxLength(20) + .withHelpContent(new QHelpContent("A label for your own reference (for example WEB, API or ASSETS). It is not used to change any behavior."))) .withField(new QFieldMetaData("stage", QFieldType.STRING).withIsRequired(true).withPossibleValueSourceName(PVS_STAGE).withDefaultValue(CdnHostnameStage.OFF.name())) .withField(new QFieldMetaData("purgeEnabled", QFieldType.BOOLEAN).withDefaultValue(true)) .withField(new QFieldMetaData("notes", QFieldType.TEXT))); diff --git a/src/main/java/com/kingsrook/qbits/cdn/processes/CdnDefaultsProcesses.java b/src/main/java/com/kingsrook/qbits/cdn/processes/CdnDefaultsProcesses.java new file mode 100644 index 0000000..7fa5f3c --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/processes/CdnDefaultsProcesses.java @@ -0,0 +1,431 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.processes; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.kingsrook.qbits.cdn.CdnQBitConfig; +import com.kingsrook.qbits.cdn.CdnRuntime; +import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qbits.cdn.permissions.CdnPermissions; +import com.kingsrook.qbits.cdn.spi.CdnProviderType; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import static com.kingsrook.qbits.cdn.processes.CdnProcesses.line; + + +/******************************************************************************* + ** The "Create defaults" table actions that give an empty CDN setup something to + ** start from. Each is a standard streamed ETL (Review lists the rows that will + ** be added, Result the rows that were), needs no selected records, needs + ** cdn.admin, and is insert-only: a row (or the whole kind of row) that exists is + ** never changed or duplicated. + **
    + **
  • Create default provider (cdn_provider): a "No CDN" provider, so changes + ** are recorded as would-be purges and nothing is sent, until a real CDN is + ** added;
    • + **
  • Create default policies (cdn_cache_policy): two disabled examples + ** (immutable assets, never cache) to edit and turn on;
    • + **
  • Create default rules (cdn_table_tag_rule): one tag rule per purgeable table + ** that has none (tags only; paths are the application's to add).
    • + **
+ ** These are actions, not a database seed, because the qbit ships no DDL and a + ** host chooses its own column names; they work on whatever the backend is. + *******************************************************************************/ +public final class CdnDefaultsProcesses +{ + public static final String CREATE_PROVIDER = "cdnCreateDefaultProvider"; + public static final String CREATE_POLICIES = "cdnCreateDefaultPolicies"; + public static final String CREATE_RULES = "cdnCreateDefaultRules"; + + public static final String NONE_PROVIDER_NAME = "No CDN (record only)"; + public static final String ASSETS_POLICY_NAME = "Static assets (immutable)"; + public static final String NEVER_POLICY_NAME = "Never cache (default)"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private CdnDefaultsProcesses() + { + } + + + + /******************************************************************************* + ** The three actions. + *******************************************************************************/ + public static List processes() + { + return (List.of( + define(CREATE_PROVIDER, "Create default provider", CdnTables.PROVIDER, ProviderExtract.class, + "This provider will be added. It records purges and sends nothing, until you add a real CDN."), + define(CREATE_POLICIES, "Create default policies", CdnTables.CACHE_POLICY, PolicyExtract.class, + "These policies will be added turned off, as examples to edit and turn on."), + define(CREATE_RULES, "Create default rules", CdnTables.TABLE_TAG_RULE, RuleExtract.class, + "These tag rules will be added, one for each purgeable table that has none."))); + } + + + + /******************************************************************************* + ** One table action. + *******************************************************************************/ + private static QProcessMetaData define(String name, String label, String table, Class extract, String preview) + { + Map values = new HashMap<>(); + values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, table); + values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, table); + values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_DO_FULL_VALIDATION, true); + values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE, preview + " Existing rows are never changed."); + QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(extract, DefaultsTransform.class, DefaultsLoad.class, values); + process.setName(name); + process.setLabel(label); + process.setIcon(new QIcon("playlist_add")); + process.setTableName(table); + process.setMinInputRecords(0); + process.setPermissionRules(CdnTables.permissionRules()); + return (CdnProcesses.permission(process, CdnPermissions.ADMIN)); + } + + + + /******************************************************************************* + ** The rows the provider action would add: the No CDN provider, when the table + ** has no NONE provider at all. + *******************************************************************************/ + public static List> missingProviders() throws QException + { + for(QRecord row : all(CdnTables.PROVIDER)) + { + if(CdnProviderType.NONE.name().equals(row.getValueString("providerType"))) + { + return (List.of()); + } + } + Map row = new LinkedHashMap<>(); + row.put("name", NONE_PROVIDER_NAME); + row.put("providerType", CdnProviderType.NONE.name()); + row.put("enabled", true); + return (List.of(row)); + } + + + + /******************************************************************************* + ** The policies the action would add, by name: both disabled, so adding them + ** changes no response until an administrator turns one on. + *******************************************************************************/ + public static List> missingPolicies() throws QException + { + Set existing = new HashSet<>(); + all(CdnTables.CACHE_POLICY).forEach(row -> existing.add(row.getValueString("name"))); + List> rows = new ArrayList<>(); + if(!existing.contains(ASSETS_POLICY_NAME)) + { + Map row = new LinkedHashMap<>(); + row.put("name", ASSETS_POLICY_NAME); + row.put("matchType", "GLOB"); + row.put("matchValue", "/assets/*"); + row.put("policyClass", "IMMUTABLE"); + row.put("browserMaxAge", 31536000); + row.put("edgeMaxAge", 31536000); + row.put("immutable", true); + row.put("noStore", false); + row.put("priority", 10); + row.put("enabled", false); + row.put("seeded", true); + rows.add(row); + } + if(!existing.contains(NEVER_POLICY_NAME)) + { + Map row = new LinkedHashMap<>(); + row.put("name", NEVER_POLICY_NAME); + row.put("matchType", "PREFIX"); + row.put("matchValue", "/"); + row.put("policyClass", "NEVER"); + row.put("immutable", false); + row.put("noStore", true); + row.put("priority", 1000); + row.put("enabled", false); + row.put("seeded", true); + rows.add(row); + } + return (rows); + } + + + + /******************************************************************************* + ** The rules the action would add: tags "table" and "table:id" for each purgeable + ** table without a rule (any rule, enabled or not). + *******************************************************************************/ + public static List> missingRules() throws QException + { + CdnQBitConfig config = CdnRuntime.getConfig(); + Set have = new HashSet<>(); + all(CdnTables.TABLE_TAG_RULE).forEach(row -> have.add(row.getValueString("tableName"))); + List> rows = new ArrayList<>(); + for(String table : config == null ? List.of() : config.getPurgeableTables()) + { + if(!have.contains(table)) + { + Map row = new LinkedHashMap<>(); + row.put("tableName", table); + row.put("tagTemplates", table + ":{id}\n" + table); + row.put("enabled", true); + rows.add(row); + } + } + return (rows); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List all(String table) throws QException + { + return (new QueryAction().execute(new QueryInput(table)).getRecords()); + } + + + + /******************************************************************************* + ** Extract the rows to add (computed; nothing is read from the selected records). + *******************************************************************************/ + public abstract static class DefaultsExtract extends AbstractExtractStep + { + /******************************************************************************* + ** The table the rows go in. + *******************************************************************************/ + protected abstract String table(); + + + + /******************************************************************************* + ** The rows that are missing. + *******************************************************************************/ + protected abstract List> missing() throws QException; + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(Map values : missing()) + { + QRecord record = new QRecord().withTableName(table()); + record.setValues(new LinkedHashMap<>(values)); + getRecordPipe().addRecord(record); + } + } + } + + + + /******************************************************************************* + ** The provider rows. + *******************************************************************************/ + public static class ProviderExtract extends DefaultsExtract + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + protected String table() + { + return (CdnTables.PROVIDER); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + protected List> missing() throws QException + { + return (missingProviders()); + } + } + + + + /******************************************************************************* + ** The cache policy rows. + *******************************************************************************/ + public static class PolicyExtract extends DefaultsExtract + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + protected String table() + { + return (CdnTables.CACHE_POLICY); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + protected List> missing() throws QException + { + return (missingPolicies()); + } + } + + + + /******************************************************************************* + ** The tag rule rows. + *******************************************************************************/ + public static class RuleExtract extends DefaultsExtract + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + protected String table() + { + return (CdnTables.TABLE_TAG_RULE); + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + protected List> missing() throws QException + { + return (missingRules()); + } + } + + + + /******************************************************************************* + ** Review: the rows that will be added. + *******************************************************************************/ + public static class DefaultsTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine added = line(Status.OK, "default row will be added", "default rows will be added", "default row was added", "default rows were added"); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord record : input.getRecords()) + { + added.incrementCount(); + output.getRecords().add(record); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + added.addSelfToListIfAnyCount(lines); + if(lines.isEmpty()) + { + lines.add(new ProcessSummaryLine(Status.INFO, null, "Nothing to add: the defaults are already there.")); + } + return (lines); + } + } + + + + /******************************************************************************* + ** Result: insert through the normal insert, so the user's own permissions apply. + *******************************************************************************/ + public static class DefaultsLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine added = line(Status.OK, "default row will be added", "default rows will be added", "default row was added", "default rows were added"); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord inserted : new InsertAction().execute(new InsertInput(input.getTableName()).withRecords(input.getRecords())).getRecords()) + { + added.incrementCountAndAddPrimaryKey(inserted.getValue("id")); + output.getRecords().add(inserted); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + added.addSelfToListIfAnyCount(lines); + return (lines); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/cdn/processes/CdnOriginCheckProcesses.java b/src/main/java/com/kingsrook/qbits/cdn/processes/CdnOriginCheckProcesses.java new file mode 100644 index 0000000..8631722 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/processes/CdnOriginCheckProcesses.java @@ -0,0 +1,205 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.processes; + + +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.cdn.CdnQBitConfig; +import com.kingsrook.qbits.cdn.checks.CdnOriginCheck; +import com.kingsrook.qbits.cdn.model.CdnHostnameStage; +import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qbits.cdn.permissions.CdnPermissions; +import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; +import static com.kingsrook.qbits.cdn.processes.CdnProcesses.line; + + +/******************************************************************************* + ** Check Origin Headers: a record action on cdn_hostname (standard streamed ETL: + ** Review lists the hostnames, Result says how many sample objects had the right + ** Cache-Control) and a daily scheduled run over every hostname that is not Off. + ** The work is CdnOriginCheck; each run writes one cdn_origin_check row per hostname. + *******************************************************************************/ +public final class CdnOriginCheckProcesses +{ + public static final String CHECK_ORIGIN = "cdnCheckOrigin"; + public static final String SCHEDULED_ORIGIN = "cdnScheduledCheckOrigin"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private CdnOriginCheckProcesses() + { + } + + + + /******************************************************************************* + ** The processes. + *******************************************************************************/ + public static List processes(CdnQBitConfig config) + { + List processes = new ArrayList<>(); + QProcessMetaData action = CdnProcesses.etl(CHECK_ORIGIN, "Check Origin Headers", CdnTables.HOSTNAME, CheckTransform.class, CheckLoad.class, + "This is a preview of the hostnames whose origin will be asked for a few sample pages, to check their Cache-Control headers."); + action.setIcon(new QIcon("fact_check")); + processes.add(CdnProcesses.permission(action, CdnPermissions.ADMIN)); + + QProcessMetaData scheduled = new QProcessMetaData().withName(SCHEDULED_ORIGIN).withLabel("Check CDN Origin Headers").withIcon(new QIcon("schedule")).withIsHidden(true) + .withPermissionRules(CdnTables.permissionRules()) + .withStep(new QBackendStepMetaData().withName("run").withCode(new QCodeReference(ScheduledStep.class))); + if(config.getSchedulerName() != null) + { + scheduled.withSchedule(new QScheduleMetaData().withSchedulerName(config.getSchedulerName()).withCronExpression("0 45 2 * * ?").withCronTimeZoneId("UTC")); + } + processes.add(CdnProcesses.permission(scheduled, CdnPermissions.ADMIN)); + return (processes); + } + + + + /******************************************************************************* + ** Review: the hostnames that will be checked. + *******************************************************************************/ + public static class CheckTransform extends AbstractTransformStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine hosts = line(Status.OK, "hostname will be checked", "hostnames will be checked", "hostname was checked", "hostnames were checked"); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord hostname : input.getRecords()) + { + hosts.incrementCountAndAddPrimaryKey(hostname.getValue("id")); + output.getRecords().add(hostname); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + hosts.addSelfToListIfAnyCount(lines); + return (lines); + } + } + + + + /******************************************************************************* + ** Result: check each hostname and record the row. + *******************************************************************************/ + public static class CheckLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface + { + private final ProcessSummaryLine clean = line(Status.OK, "hostname will be checked", "hostnames will be checked", "hostname has the right headers", "hostnames have the right headers"); + private final ProcessSummaryLine wrong = line(Status.WARNING, "page has a missing or wrong Cache-Control", "pages have a missing or wrong Cache-Control", + "page has a missing or wrong Cache-Control", "pages have a missing or wrong Cache-Control"); + private final ProcessSummaryLine unreadable = line(Status.WARNING, "page could not be read", "pages could not be read", "page could not be read", "pages could not be read"); + private final List problems = new ArrayList<>(); + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord hostname : input.getRecords()) + { + CdnOriginCheck.Result result = CdnOriginCheck.checkAndRecord(hostname); + if(result.wrong() == 0 && result.unreadable() == 0) + { + clean.incrementCountAndAddPrimaryKey(hostname.getValue("id")); + } + wrong.incrementCount(result.wrong()); + unreadable.incrementCount(result.unreadable()); + result.problems().forEach(problem -> problems.add(result.scope() + problem)); + output.getRecords().add(hostname); + } + } + + + + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen) + { + ArrayList lines = new ArrayList<>(); + clean.addSelfToListIfAnyCount(lines); + wrong.setBulletsOfText(new ArrayList<>(problems)); + wrong.addSelfToListIfAnyCount(lines); + unreadable.addSelfToListIfAnyCount(lines); + return (lines); + } + } + + + + /******************************************************************************* + ** Scheduled: every hostname that is not Off. + *******************************************************************************/ + public static class ScheduledStep implements BackendStep + { + /******************************************************************************* + ** {@inheritDoc} + *******************************************************************************/ + @Override + public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException + { + for(QRecord hostname : new QueryAction().execute(new QueryInput(CdnTables.HOSTNAME) + .withFilter(new QQueryFilter(new QFilterCriteria("stage", QCriteriaOperator.NOT_EQUALS, CdnHostnameStage.OFF.name())))).getRecords()) + { + CdnOriginCheck.checkAndRecord(hostname); + } + } + } +} diff --git a/src/main/resources/help/qbit-cdn.yaml b/src/main/resources/help/qbit-cdn.yaml new file mode 100644 index 0000000..e6d9cf4 --- /dev/null +++ b/src/main/resources/help/qbit-cdn.yaml @@ -0,0 +1,58 @@ +bundle: qbit-cdn +version: 1 +layer: QBIT +# Help for the CDN app and every cdn_* table, in the QQQ help-bundle format +# (qqq#1198). Each table entry says what the table is for and how its rows get +# created, so an empty table explains itself. CdnHelp applies it until the core +# HelpBundles registry ships in a QQQ release (STAND-IN(core=#1198)). +# Summaries stay under 120 characters (CdnTablesGuardrailTest). +entries: + - key: app:cdnAdmin;slot:help + summary: "Put a CDN in front of the application: cache headers, change-driven purges, edge statistics and checks." + content: | + The Setup card on the home lists each part (provider, hostnames, cache policies, tag rules, revalidation hooks, origin checks) with its status and the next step. A new installation has nothing: run Create default provider on Providers (No CDN records purges and sends nothing), Create default rules on Tag rules, and add your CDN when you have one. + + - key: table:cdn_provider;slot:help + summary: "A CDN the application purges: its type, distribution id and the variable that holds its key." + content: | + **How rows get created:** an administrator adds one with New (pick the type, the distribution or zone id, and name the environment variable that holds the API key; the key itself is never stored), or runs Create default provider, which adds "No CDN (record only)": every record change is then recorded as a would-be purge and nothing is sent. Test Connection, Import Edge Stats and Check Config Drift are actions on a provider. + + - key: table:cdn_hostname;slot:help + summary: "A hostname behind a provider and its rollout stage: Off, Headers only, Live or Locked." + content: | + **How rows get created:** an administrator adds one with New for each hostname the CDN serves, at stage Off, then moves it to Headers only and Live as the rollout proceeds. Only Live and Locked hostnames with purging enabled are sent purges. Check Origin Headers is an action on a hostname, and hostnames that are not Off are checked every night. The role field is a label for your own reference and changes nothing. + + - key: table:cdn_cache_policy;slot:help + summary: "Cache-Control rules by path, route class or table; the lowest priority number that matches wins." + content: | + **How rows get created:** an administrator adds one with New, or runs Create default policies, which adds two turned-off examples (immutable static assets, never cache) to edit and turn on. A policy changes a response only where the application calls CdnResponseHeaderPolicy.decide when it answers; the qbit does not rewrite responses on its own. A response that sets a cookie or answers a signed-in user is always private, whatever the policies say. + + - key: table:cdn_table_tag_rule;slot:help + summary: "How a table's record changes become cache tags and invalidation paths for purges." + content: | + **How rows get created:** an administrator adds one with New for each purgeable table the application lists, or runs Create default rules, which adds a tags-only rule for every purgeable table that has none (tags "table:id" and "table"). Add invalidation paths where your CDN purges by path, and a published field so draft edits purge nothing. + + - key: table:cdn_purge_request;slot:help + summary: "Every purge the qbit recorded or sent, with its status, paths and tags." + content: | + **How rows get created:** only the qbit writes it: a change to a purgeable record, Purge From CDN on a record, or Purge Everything on a hostname. Without a provider nothing is recorded, so run Create default provider first. Retry Purge is an action here. + + - key: table:cdn_revalidation_hook;slot:help + summary: "A signed webhook called before a CDN purge, so a front-end cache (for example Next.js) refreshes first." + content: | + **How rows get created:** an administrator adds one with New: the URL, the signature format and the environment variable that holds the signing secret (never the secret). It is optional; add one only when a front end keeps its own cache. + + - key: table:cdn_edge_stat_daily;slot:help + summary: "Daily edge statistics imported from the CDN: requests, bytes, hit rate and error rates." + content: | + **How rows get created:** Import Edge Stats on a provider writes it, and a daily scheduled run does the same for every provider that supports it. A No CDN provider has no statistics. + + - key: table:cdn_origin_check;slot:help + summary: "Results of asking a hostname's origin for sample pages and comparing their Cache-Control headers." + content: | + **How rows get created:** Check Origin Headers on a hostname writes one row per hostname, and a nightly scheduled run does the same for every hostname that is not Off. It requests the home page, the paths of prefix cache policies and the literal paths of tag rules over HTTPS and counts the right, missing or wrong, and unreadable ones. + + - key: table:cdn_drift_finding;slot:help + summary: "Differences between the CDN's expected configuration and what the provider reports." + content: | + **How rows get created:** Check Config Drift on a provider writes it, and a nightly scheduled run does the same. It needs the expected configuration in the environment variable the provider names (CDN_EXPECTED_CONFIG by default); a finding is resolved when it no longer appears. diff --git a/src/test/java/com/kingsrook/qbits/cdn/app/CdnHomeTest.java b/src/test/java/com/kingsrook/qbits/cdn/app/CdnHomeTest.java index e7ade8a..929ecec 100644 --- a/src/test/java/com/kingsrook/qbits/cdn/app/CdnHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/cdn/app/CdnHomeTest.java @@ -59,7 +59,7 @@ class CdnHomeTest CdnTestSupport.setUp(); String html = render(CdnWidgets.HOME_WIDGET); assertThat(html).contains("No CDN provider configured: purges are recorded only.").contains("No edge statistics yet.").contains("No purges recorded yet.") - .contains("No path-billed provider connected yet.").contains("Origin header checks are not run by this version; nothing has been checked.").contains("Nothing to compare yet.") + .contains("No path-billed provider connected yet.").contains("No origin check yet.").contains("Nothing to compare yet.") .contains("minmax(min(100%,22rem),1fr)"); assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN provider configured: purges are recorded only.").contains("href=\"/cdnAdmin\""); } @@ -141,7 +141,7 @@ class CdnHomeTest CdnTestSupport.insert(CdnTables.PURGE_REQUEST, new QRecord().withValue("cdnProviderId", provider).withValue("status", "SKIPPED_NONE_PROVIDER")); assertThat(render(CdnWidgets.HOME_WIDGET)).contains("No CDN provider configured: purges are recorded only.") .contains("The CloudFront distribution is created in the cutover window.").contains("recorded only (not sent)") - .contains("Origin header checks are not run by this version; nothing has been checked."); + .contains("No origin check yet."); assertThat(render(CdnWidgets.HEALTH_CARD_WIDGET)).contains("No CDN provider configured: purges are recorded only."); } } diff --git a/src/test/java/com/kingsrook/qbits/cdn/tables/CdnDefaultsAndOriginCheckTest.java b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnDefaultsAndOriginCheckTest.java new file mode 100644 index 0000000..206af3b --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnDefaultsAndOriginCheckTest.java @@ -0,0 +1,311 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.tables; + + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.cdn.CdnRuntime; +import com.kingsrook.qbits.cdn.CdnTestSupport; +import com.kingsrook.qbits.cdn.app.CdnSetupStatus; +import com.kingsrook.qbits.cdn.checks.CdnOriginCheck; +import com.kingsrook.qbits.cdn.headers.CdnResponseHeaderPolicy; +import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses; +import com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** The "Create default ..." table actions (insert-only, change nothing that is + ** there) and the origin header check (the writer of cdn_origin_check, which had + ** none): what it counts, what it records, and the action and nightly entry points. + *******************************************************************************/ +class CdnDefaultsAndOriginCheckTest +{ + private final Map origin = new LinkedHashMap<>(); + private final List asked = new java.util.ArrayList<>(); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + CdnTestSupport.setUp(); + CdnOriginCheck.setFetcher((host, path) -> + { + asked.add(host + path); + CdnOriginCheck.Response response = origin.get(path); + if(response == null) + { + throw (new IOException("connection refused")); + } + return (response); + }); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void tearDown() + { + CdnOriginCheck.setFetcher(null); + CdnResponseHeaderPolicy.clearCache(); + } + + + + /******************************************************************************* + ** Run a table action through Review and then Result; returns the result lines. + *******************************************************************************/ + @SuppressWarnings("unchecked") + private static List run(String process, String recordIds) throws Exception + { + RunProcessInput input = new RunProcessInput(); + input.setProcessName(process); + if(recordIds != null) + { + input.addValue("recordsParam", "recordIds"); + input.addValue("recordIds", recordIds); + } + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + RunProcessOutput review = new RunProcessAction().execute(input); + RunProcessInput go = new RunProcessInput(); + go.setProcessName(process); + go.setProcessUUID(review.getProcessUUID()); + go.setStartAfterStep(StreamedETLWithFrontendProcess.STEP_NAME_REVIEW); + go.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + RunProcessOutput result = new RunProcessAction().execute(go); + List lines = (List) result.getValues().get(StreamedETLWithFrontendProcess.FIELD_PROCESS_SUMMARY); + return (lines == null ? List.of() : lines.stream().map(line -> + { + ProcessSummaryLine summary = (ProcessSummaryLine) line; + summary.pickMessage(true); + return (summary.getCount() + " " + summary.getMessage()); + }).toList()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static List rows(String table) throws Exception + { + return (new QueryAction().execute(new QueryInput(table)).getRecords()); + } + + + + /******************************************************************************* + ** Create default provider adds No CDN once; and with it a record change is + ** recorded as a would-be purge (with no provider it is not recorded at all). + *******************************************************************************/ + @Test + void defaultProviderMakesPurgesRecorded() throws Exception + { + CdnTestSupport.postRule(); + CdnTestSupport.insert("post", new QRecord().withValue("slug", "a").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true)); + assertThat(rows(CdnTables.PURGE_REQUEST)).as("no provider: nothing is recorded").isEmpty(); + + assertThat(run(CdnDefaultsProcesses.CREATE_PROVIDER, null)).containsExactly("1 default row was added"); + assertThat(rows(CdnTables.PROVIDER)).singleElement().satisfies(p -> + { + assertThat(p.getValueString("name")).isEqualTo(CdnDefaultsProcesses.NONE_PROVIDER_NAME); + assertThat(p.getValueString("providerType")).isEqualTo("NONE"); + assertThat(p.getValueBoolean("enabled")).isTrue(); + }); + + CdnTestSupport.insert("post", new QRecord().withValue("slug", "b").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true)); + assertThat(rows(CdnTables.PURGE_REQUEST)).singleElement().satisfies(r -> assertThat(r.getValueString("status")).isEqualTo("SKIPPED_NONE_PROVIDER")); + + /////////////////////////////////////////////////////////////////////// + // insert-only: again changes nothing, and an edited row is left alone // + /////////////////////////////////////////////////////////////////////// + new UpdateAction().execute(new UpdateInput(CdnTables.PROVIDER).withRecord(new QRecord().withValue("id", rows(CdnTables.PROVIDER).get(0).getValue("id")).withValue("name", "Ours"))); + assertThat(run(CdnDefaultsProcesses.CREATE_PROVIDER, null)).containsExactly("null Nothing to add: the defaults are already there."); + assertThat(rows(CdnTables.PROVIDER)).singleElement().satisfies(p -> assertThat(p.getValueString("name")).isEqualTo("Ours")); + } + + + + /******************************************************************************* + ** Create default policies adds two turned-off policies (so nothing changes until + ** one is turned on), by name; an edited policy survives a second run; turning one + ** on makes the header decision change. + *******************************************************************************/ + @Test + void defaultPoliciesAreOffUntilTurnedOn() throws Exception + { + assertThat(run(CdnDefaultsProcesses.CREATE_POLICIES, null)).containsExactly("2 default rows were added"); + List policies = rows(CdnTables.CACHE_POLICY); + assertThat(policies).hasSize(2).allSatisfy(p -> assertThat(p.getValueBoolean("enabled")).isFalse()); + CdnResponseHeaderPolicy.clearCache(); + assertThat(CdnResponseHeaderPolicy.current()).as("turned-off policies change no response").isEmpty(); + + QRecord assets = policies.stream().filter(p -> CdnDefaultsProcesses.ASSETS_POLICY_NAME.equals(p.getValueString("name"))).findFirst().orElseThrow(); + new UpdateAction().execute(new UpdateInput(CdnTables.CACHE_POLICY).withRecord(new QRecord().withValue("id", assets.getValue("id")).withValue("enabled", true).withValue("browserMaxAge", 600))); + CdnResponseHeaderPolicy.clearCache(); + CdnResponseHeaderPolicy.Decision decision = CdnResponseHeaderPolicy.decide(new CdnResponseHeaderPolicy.Request("www.example.com", "/assets/app.js", null, null, false, false, List.of(), List.of()), null); + assertThat(decision.policyName()).isEqualTo(CdnDefaultsProcesses.ASSETS_POLICY_NAME); + assertThat(decision.cacheControl()).startsWith("public, max-age=600").endsWith("immutable"); + + assertThat(run(CdnDefaultsProcesses.CREATE_POLICIES, null)).containsExactly("null Nothing to add: the defaults are already there."); + assertThat(rows(CdnTables.CACHE_POLICY)).hasSize(2).anySatisfy(p -> assertThat(p.getValueInteger("browserMaxAge")).isEqualTo(600)); + } + + + + /******************************************************************************* + ** Create default rules adds a tags-only rule for each purgeable table without + ** one, and the rule makes a change record tags. + *******************************************************************************/ + @Test + void defaultRulesCoverPurgeableTables() throws Exception + { + assertThat(run(CdnDefaultsProcesses.CREATE_RULES, null)).containsExactly("1 default row was added"); + assertThat(rows(CdnTables.TABLE_TAG_RULE)).singleElement().satisfies(r -> + { + assertThat(r.getValueString("tableName")).isEqualTo("post"); + assertThat(r.getValueString("tagTemplates")).isEqualTo("post:{id}\npost"); + assertThat(r.getValueBoolean("enabled")).isTrue(); + }); + CdnTestSupport.insert(CdnTables.PROVIDER, new QRecord().withValue("name", "none").withValue("providerType", "NONE").withValue("enabled", true)); + QRecord post = CdnTestSupport.insert("post", new QRecord().withValue("slug", "a").withValue("title", "T").withValue("blogId", 1).withValue("isPublished", true)).get(0); + assertThat(rows(CdnTables.PURGE_REQUEST)).singleElement().satisfies(r -> assertThat(r.getValueString("tags")).contains("post:" + post.getValue("id"))); + + assertThat(run(CdnDefaultsProcesses.CREATE_RULES, null)).containsExactly("null Nothing to add: the defaults are already there."); + assertThat(rows(CdnTables.TABLE_TAG_RULE)).hasSize(1); + } + + + + /******************************************************************************* + ** The origin check: a good page, a page with no Cache-Control, a page whose + ** Cache-Control differs from its policy, a 404 and an unreachable page. + *******************************************************************************/ + @Test + void originCheckCountsRightWrongAndUnreadable() throws Exception + { + Integer provider = CdnTestSupport.liveProvider(com.kingsrook.qbits.cdn.spi.CdnProviderType.CLOUDFRONT, null); + new UpdateAction().execute(new UpdateInput(CdnTables.PROVIDER).withRecord(new QRecord().withValue("id", provider).withValue("originHost", "origin.example.com:443"))); + CdnTestSupport.insert(CdnTables.CACHE_POLICY, + new QRecord().withValue("name", "blog").withValue("matchType", "PREFIX").withValue("matchValue", "/blog").withValue("policyClass", "LEAF").withValue("browserMaxAge", 60).withValue("enabled", true), + new QRecord().withValue("name", "news").withValue("matchType", "PREFIX").withValue("matchValue", "/news").withValue("policyClass", "LEAF").withValue("browserMaxAge", 60).withValue("enabled", true)); + CdnTestSupport.insert(CdnTables.TABLE_TAG_RULE, new QRecord().withValue("tableName", "post").withValue("enabled", true) + .withValue("invalidationPaths", "/about\n/blog/{slug}*\nhub:/blog")); + CdnResponseHeaderPolicy.clearCache(); + + origin.put("/", new CdnOriginCheck.Response(200, Map.of("cache-control", "max-age=5"))); + origin.put("/blog", new CdnOriginCheck.Response(200, Map.of("cache-control", "Public, max-age=60"))); + origin.put("/news", new CdnOriginCheck.Response(200, Map.of("cache-control", "public, max-age=999"))); + origin.put("/about", new CdnOriginCheck.Response(200, Map.of())); + CdnTestSupport.insert(CdnTables.CACHE_POLICY, new QRecord().withValue("name", "gone").withValue("matchType", "PREFIX").withValue("matchValue", "/gone").withValue("policyClass", "LEAF").withValue("enabled", true), + new QRecord().withValue("name", "down").withValue("matchType", "PREFIX").withValue("matchValue", "/down").withValue("policyClass", "LEAF").withValue("enabled", true)); + CdnResponseHeaderPolicy.clearCache(); + origin.put("/gone", new CdnOriginCheck.Response(404, Map.of())); + + QRecord hostname = rows(CdnTables.HOSTNAME).get(0); + CdnOriginCheck.Result result = CdnOriginCheck.checkAndRecord(hostname); + + assertThat(asked).as("the provider's origin host (without a port) is asked, not the public name").allMatch(url -> url.startsWith("origin.example.com/")); + assertThat(result.checked()).isEqualTo(6); + assertThat(result.correct()).as("/ (any Cache-Control, no policy) and /blog (equals its policy, ignoring case)").isEqualTo(2); + assertThat(result.wrong()).as("/about has no header; /news differs from its policy").isEqualTo(2); + assertThat(result.unreadable()).as("/gone answered 404; /down is unreachable").isEqualTo(2); + assertThat(result.problems()).anyMatch(p -> p.startsWith("/about: no Cache-Control")).anyMatch(p -> p.startsWith("/news") && p.contains("public, max-age=60")) + .anyMatch(p -> p.startsWith("/gone: the origin answered 404")).anyMatch(p -> p.startsWith("/down: could not be read")); + + assertThat(rows(CdnTables.ORIGIN_CHECK)).singleElement().satisfies(row -> + { + assertThat(row.getValueString("scope")).isEqualTo("www.example.com"); + assertThat(row.getValueInteger("objectsChecked")).isEqualTo(6); + assertThat(row.getValueInteger("correct")).isEqualTo(2); + assertThat(row.getValueInteger("missingOrWrong")).isEqualTo(2); + assertThat(row.getValueInteger("unreadable")).isEqualTo(2); + assertThat(row.getValue("checkDate")).isNotNull(); + }); + } + + + + /******************************************************************************* + ** Sample paths are bounded and only literal paths are used. + *******************************************************************************/ + @Test + void samplePathsAreLiteralAndBounded() throws Exception + { + StringBuilder paths = new StringBuilder("/blog/{slug}*"); + for(int i = 0; i < 40; i++) + { + paths.append("\n/page-").append(i); + } + CdnTestSupport.insert(CdnTables.TABLE_TAG_RULE, new QRecord().withValue("tableName", "post").withValue("enabled", true).withValue("invalidationPaths", paths.toString())); + List sample = CdnOriginCheck.samplePaths("www.example.com"); + assertThat(sample).hasSize(CdnOriginCheck.MAX_PATHS).startsWith("/", "/page-0").noneMatch(p -> p.contains("{") || p.contains("*")); + } + + + + /******************************************************************************* + ** The record action writes a row per selected hostname; the nightly entry point + ** skips hostnames that are Off. + *******************************************************************************/ + @Test + void actionAndNightlyRun() throws Exception + { + CdnTestSupport.liveProvider(com.kingsrook.qbits.cdn.spi.CdnProviderType.CLOUDFRONT, null); + CdnTestSupport.insert(CdnTables.HOSTNAME, new QRecord().withValue("hostname", "off.example.com").withValue("cdnProviderId", 1).withValue("stage", "OFF")); + origin.put("/", new CdnOriginCheck.Response(200, Map.of("cache-control", "max-age=5"))); + + List lines = run(CdnOriginCheckProcesses.CHECK_ORIGIN, "1"); + assertThat(lines).containsExactly("1 hostname has the right headers"); + assertThat(rows(CdnTables.ORIGIN_CHECK)).hasSize(1); + + new CdnOriginCheckProcesses.ScheduledStep().run(new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput(), new com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput()); + assertThat(rows(CdnTables.ORIGIN_CHECK)).extracting(r -> r.getValueString("scope")).containsExactly("www.example.com", "www.example.com"); + + CdnRuntime.setClock(null); + assertThat(CdnSetupStatus.load()).anySatisfy(item -> + { + assertThat(item.tableName()).isEqualTo(CdnTables.ORIGIN_CHECK); + assertThat(item.state()).isEqualTo(CdnSetupStatus.State.OK); + }); + } +} diff --git a/src/test/java/com/kingsrook/qbits/cdn/tables/CdnSetupStatusTest.java b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnSetupStatusTest.java new file mode 100644 index 0000000..0502501 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnSetupStatusTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.tables; + + +import java.util.List; +import com.kingsrook.qbits.cdn.CdnTestSupport; +import com.kingsrook.qbits.cdn.app.CdnSetupStatus; +import com.kingsrook.qbits.cdn.app.CdnSetupStatus.Item; +import com.kingsrook.qbits.cdn.app.CdnSetupStatus.State; +import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** The Setup card of the CDN home: empty, on the No CDN default, and fully set up. + *******************************************************************************/ +class CdnSetupStatusTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + private static Item item(List items, String label) + { + return (items.stream().filter(i -> i.label().equals(label)).findFirst().orElseThrow()); + } + + + + /******************************************************************************* + ** Nothing set up: the next steps name the Create default actions, and the home shows them. + *******************************************************************************/ + @Test + void emptySetupPointsAtTheDefaults() throws Exception + { + CdnTestSupport.setUp(); + List items = CdnSetupStatus.load(); + assertThat(item(items, "Providers").state()).isEqualTo(State.TODO); + assertThat(item(items, "Providers").nextStep()).contains("Create default provider"); + assertThat(item(items, "Tag rules").state()).isEqualTo(State.TODO); + assertThat(item(items, "Tag rules").nextStep()).contains("Create default rules"); + assertThat(item(items, "Cache policies").nextStep()).contains("Create default policies"); + assertThat(items).extracting(Item::tableName).contains(CdnTables.PROVIDER, CdnTables.HOSTNAME, CdnTables.CACHE_POLICY, CdnTables.TABLE_TAG_RULE, + CdnTables.REVALIDATION_HOOK, CdnTables.ORIGIN_CHECK); + } + + + + /******************************************************************************* + ** A real CDN, a Live hostname, a policy that is on, a rule for each table and a hook. + *******************************************************************************/ + @Test + void aCompleteSetupIsOk() + { + List providers = List.of(new QRecord().withValue("providerType", "CLOUDFRONT").withValue("enabled", true).withValue("lastHealthStatus", "OK")); + List hostnames = List.of(new QRecord().withValue("stage", "LIVE")); + List policies = List.of(new QRecord().withValue("enabled", true)); + List rules = List.of(new QRecord().withValue("tableName", "post").withValue("enabled", true)); + List hooks = List.of(new QRecord().withValue("enabled", true)); + List checks = List.of(new QRecord()); + List items = CdnSetupStatus.build(providers, hostnames, policies, rules, hooks, checks, List.of("post")); + assertThat(items).extracting(Item::state).containsOnly(State.OK); + } + + + + /******************************************************************************* + ** The softer states: No CDN only, hostnames not Live, a failing provider, a table without a rule. + *******************************************************************************/ + @Test + void softerStatesAreNamed() + { + QRecord none = new QRecord().withValue("providerType", "NONE").withValue("enabled", true); + List items = CdnSetupStatus.build(List.of(none), List.of(new QRecord().withValue("stage", "OFF")), List.of(new QRecord().withValue("enabled", false)), + List.of(), List.of(), List.of(), List.of("post", "page")); + assertThat(item(items, "Providers").state()).isEqualTo(State.DEFAULT); + assertThat(item(items, "Hostnames").state()).isEqualTo(State.DEFAULT); + assertThat(item(items, "Cache policies").state()).isEqualTo(State.DEFAULT); + assertThat(item(items, "Tag rules").detail()).contains("post, page"); + + QRecord failing = new QRecord().withValue("providerType", "FASTLY").withValue("enabled", true).withValue("lastHealthStatus", "FAILED"); + assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), List.of(), List.of()), "Providers").state()).isEqualTo(State.TODO); + assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), List.of(), List.of()), "Hostnames").state()).isEqualTo(State.TODO); + } +} diff --git a/src/test/java/com/kingsrook/qbits/cdn/tables/CdnTablesGuardrailTest.java b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnTablesGuardrailTest.java new file mode 100644 index 0000000..0b838c7 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnTablesGuardrailTest.java @@ -0,0 +1,189 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.tables; + + +import java.lang.reflect.Field; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import com.kingsrook.qbits.cdn.CdnTestSupport; +import com.kingsrook.qbits.cdn.model.CdnHelp; +import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qbits.cdn.processes.CdnDefaultsProcesses; +import com.kingsrook.qbits.cdn.processes.CdnOriginCheckProcesses; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Guardrail (owner requirement 2026-10-14: "all tables should be populated and + ** the qbit should be fully using them"). For every cdn_* table: + ** - runtime code (not just its model, metadata or home screen) reads or writes it; + ** - it is listed here with why it starts empty and, where there is one, the table + ** action that sets it up (this qbit ships no DDL and a host chooses its column + ** names, so defaults are actions, not a database seed); + ** - it has bundle help that says what it is for and how its rows get created + ** (shown on the table screen and when it is empty). + ** A new table fails this test until it is wired and explained. + *******************************************************************************/ +class CdnTablesGuardrailTest +{ + private static final Path MAIN = Path.of("src", "main", "java", "com", "kingsrook", "qbits", "cdn"); + + /******************************************************************************* + ** Why each table starts empty, and where its rows come from. + *******************************************************************************/ + static final Map STARTS_EMPTY = Map.ofEntries( + Map.entry(CdnTables.PROVIDER, "host connection config: the Create default provider action adds No CDN; a real CDN needs the host's account and key variable"), + Map.entry(CdnTables.HOSTNAME, "host-specific: the hostnames the host serves"), + Map.entry(CdnTables.CACHE_POLICY, "the Create default policies action adds two turned-off examples; turning one on is the host's decision"), + Map.entry(CdnTables.TABLE_TAG_RULE, "the Create default rules action adds a tags-only rule per purgeable table; paths are the host's"), + Map.entry(CdnTables.PURGE_REQUEST, "runtime: a record change or a purge action"), + Map.entry(CdnTables.REVALIDATION_HOOK, "host-specific and optional: a front end with its own cache"), + Map.entry(CdnTables.EDGE_STAT_DAILY, "runtime: Import Edge Stats and the daily import"), + Map.entry(CdnTables.ORIGIN_CHECK, "runtime: Check Origin Headers and the nightly check"), + Map.entry(CdnTables.DRIFT_FINDING, "runtime: Check Config Drift and the nightly check")); + + /******************************************************************************* + ** The table action that sets a table up, by table. + *******************************************************************************/ + static final Map SETUP_ACTIONS = Map.of( + CdnTables.PROVIDER, CdnDefaultsProcesses.CREATE_PROVIDER, + CdnTables.CACHE_POLICY, CdnDefaultsProcesses.CREATE_POLICIES, + CdnTables.TABLE_TAG_RULE, CdnDefaultsProcesses.CREATE_RULES); + + private QInstance qInstance; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + qInstance = CdnTestSupport.setUp(); + } + + + + /******************************************************************************* + ** Every table is used by runtime code. + *******************************************************************************/ + @Test + void everyTableIsUsedAtRuntime() throws Exception + { + List runtimeSource = new ArrayList<>(); + try(Stream files = Files.walk(MAIN)) + { + for(Path file : files.filter(p -> p.toString().endsWith(".java")).filter(p -> !p.startsWith(MAIN.resolve("model")) && !p.startsWith(MAIN.resolve("app"))).toList()) + { + runtimeSource.add(Files.readString(file, StandardCharsets.UTF_8)); + } + } + for(String table : cdnTables()) + { + String constant = constantFor(table); + Pattern use = Pattern.compile("\\bCdnTables\\." + constant + "\\b"); + assertThat(runtimeSource.stream().anyMatch(source -> use.matcher(source).find())) + .as(table + " (CdnTables." + constant + ") must be read or written by runtime code, not only declared").isTrue(); + } + } + + + + /******************************************************************************* + ** Every table is explained, and the setup actions it names exist. + *******************************************************************************/ + @Test + void everyTableIsExplained() + { + assertThat(cdnTables()).containsExactlyInAnyOrderElementsOf(STARTS_EMPTY.keySet()); + assertThat(CdnTables.ALL).containsExactlyInAnyOrderElementsOf(STARTS_EMPTY.keySet()); + SETUP_ACTIONS.forEach((table, action) -> + { + assertThat(qInstance.getProcess(action)).as(action).isNotNull(); + assertThat(qInstance.getProcess(action).getTableName()).isEqualTo(table); + assertThat(qInstance.getProcess(action).getMinInputRecords()).as(action + " needs no selected records").isEqualTo(0); + }); + assertThat(qInstance.getProcess(CdnOriginCheckProcesses.CHECK_ORIGIN)).isNotNull(); + } + + + + /******************************************************************************* + ** Every table and the app have bundle help that says how rows get created, and + ** the help of a table with a setup action names it. + *******************************************************************************/ + @Test + void everyTableHasEmptyStateHelp() throws Exception + { + assertThat(CdnHelp.entries()).allSatisfy((key, entry) -> assertThat(entry.path("summary").asText().length()).as(key).isLessThanOrEqualTo(120)); + for(String table : cdnTables()) + { + List help = qInstance.getTable(table).getHelpContent() == null ? null : qInstance.getTable(table).getHelpContent().get(CdnHelp.SLOT); + assertThat(help).as(table + " table help").isNotEmpty(); + assertThat(help.get(0).getContent()).as(table + " help says how rows get created").contains("How rows get created"); + if(SETUP_ACTIONS.containsKey(table)) + { + assertThat(help.get(0).getContent()).as(table + " help names its setup action").contains(qInstance.getProcess(SETUP_ACTIONS.get(table)).getLabel()); + } + } + assertThat(qInstance.getApp("cdnAdmin").getHelpContent()).containsKey(CdnHelp.SLOT); + assertThat(CdnHelp.entries().keySet()).allMatch(key -> !key.startsWith("table:") || qInstance.getTable(key.substring(6, key.indexOf(';'))) != null, + "help names only real tables"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private List cdnTables() + { + List tables = qInstance.getTables().values().stream().map(QTableMetaData::getName).filter(n -> n.startsWith("cdn_")).sorted().toList(); + assertThat(tables).hasSize(9); + return (tables); + } + + + + /******************************************************************************* + ** The CdnTables constant whose value is the table name. + *******************************************************************************/ + private static String constantFor(String table) throws Exception + { + for(Field field : CdnTables.class.getFields()) + { + if(field.getType() == String.class && table.equals(field.get(null)) && !field.getName().startsWith("PVS_")) + { + return (field.getName()); + } + } + throw (new IllegalStateException("no constant for " + table)); + } +} -- 2.54.0 From 422b34988d5897295b36a8238ae7692fb0931101 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 17:06:20 -0500 Subject: [PATCH 2/3] fix: pin origin check to the checked address, full address blocklist, header-only read; setup and defaults review fixes Co-Authored-By: Claude Opus 5.5 --- .../qbits/cdn/app/CdnSetupStatus.java | 29 ++- .../qbits/cdn/checks/CdnOriginCheck.java | 234 ++++++++++++++++-- .../cdn/processes/CdnDefaultsProcesses.java | 18 +- .../processes/CdnOriginCheckProcesses.java | 13 +- .../cdn/checks/CdnOriginCheckSafetyTest.java | 142 +++++++++++ .../qbits/cdn/tables/CdnSetupStatusTest.java | 21 +- .../cdn/tables/CdnTablesGuardrailTest.java | 17 ++ 7 files changed, 425 insertions(+), 49 deletions(-) create mode 100644 src/test/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheckSafetyTest.java diff --git a/src/main/java/com/kingsrook/qbits/cdn/app/CdnSetupStatus.java b/src/main/java/com/kingsrook/qbits/cdn/app/CdnSetupStatus.java index b76cad2..c08e703 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/app/CdnSetupStatus.java +++ b/src/main/java/com/kingsrook/qbits/cdn/app/CdnSetupStatus.java @@ -26,8 +26,10 @@ import com.kingsrook.qbits.cdn.CdnRuntime; import com.kingsrook.qbits.cdn.model.CdnHostnameStage; import com.kingsrook.qbits.cdn.model.CdnTables; import com.kingsrook.qbits.cdn.spi.CdnProviderType; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -80,8 +82,9 @@ public final class CdnSetupStatus List policies = all(CdnTables.CACHE_POLICY); List rules = all(CdnTables.TABLE_TAG_RULE); List hooks = all(CdnTables.REVALIDATION_HOOK); - List checks = all(CdnTables.ORIGIN_CHECK); - return (build(providers, hostnames, policies, rules, hooks, checks, config == null ? List.of() : config.getPurgeableTables())); + long checks = new CountAction().execute(new CountInput(CdnTables.ORIGIN_CHECK)).getCount(); + boolean scheduled = config != null && config.getSchedulerName() != null; + return (build(providers, hostnames, policies, rules, hooks, checks, config == null ? List.of() : config.getPurgeableTables(), scheduled)); } @@ -90,14 +93,14 @@ public final class CdnSetupStatus ** The items for given rows (no queries; for tests and reuse). *******************************************************************************/ public static List build(List providers, List hostnames, List policies, List rules, List hooks, - List checks, List purgeableTables) + long checks, List purgeableTables, boolean scheduled) { List items = new ArrayList<>(); ////////////// // provider // ////////////// - List enabledProviders = providers.stream().filter(p -> !Boolean.FALSE.equals(p.getValueBoolean("enabled"))).toList(); + List enabledProviders = providers.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("enabled"))).toList(); List real = enabledProviders.stream().filter(p -> !CdnProviderType.NONE.name().equals(p.getValueString("providerType"))).toList(); if(providers.isEmpty()) { @@ -132,17 +135,18 @@ public final class CdnSetupStatus } else { - long live = hostnames.stream().filter(h -> CdnHostnameStage.LIVE.name().equals(h.getValueString("stage")) || CdnHostnameStage.LOCKED.name().equals(h.getValueString("stage"))).count(); + long live = hostnames.stream().filter(h -> (CdnHostnameStage.LIVE.name().equals(h.getValueString("stage")) || CdnHostnameStage.LOCKED.name().equals(h.getValueString("stage"))) + && Boolean.TRUE.equals(h.getValueBoolean("purgeEnabled"))).count(); items.add(live == 0 ? new Item("Hostnames", CdnTables.HOSTNAME, State.DEFAULT, hostnames.size() + " hostname" + (hostnames.size() == 1 ? "" : "s") + ", none Live: purges are recorded, not sent.", - "When the CDN serves a hostname, move it to Live.") + "When the CDN serves a hostname, move it to Live (and leave Purge enabled on).") : new Item("Hostnames", CdnTables.HOSTNAME, State.OK, live + " of " + hostnames.size() + " hostnames Live.", null)); } //////////////////////// // cache policies // //////////////////////// - long enabledPolicies = policies.stream().filter(p -> !Boolean.FALSE.equals(p.getValueBoolean("enabled"))).count(); + long enabledPolicies = policies.stream().filter(p -> Boolean.TRUE.equals(p.getValueBoolean("enabled"))).count(); items.add(policies.isEmpty() ? new Item("Cache policies", CdnTables.CACHE_POLICY, State.OPTIONAL, "None: responses keep the headers your application sets.", "Run Create default policies for examples. Policies only change responses when the application calls CdnResponseHeaderPolicy.") @@ -155,7 +159,7 @@ public final class CdnSetupStatus // tag rules: every purgeable table // //////////////////////////////////////// Set covered = new HashSet<>(); - rules.stream().filter(r -> !Boolean.FALSE.equals(r.getValueBoolean("enabled"))).forEach(r -> covered.add(r.getValueString("tableName"))); + rules.stream().filter(r -> Boolean.TRUE.equals(r.getValueBoolean("enabled"))).forEach(r -> covered.add(r.getValueString("tableName"))); List uncovered = purgeableTables.stream().filter(t -> !covered.contains(t)).toList(); if(purgeableTables.isEmpty()) { @@ -174,7 +178,7 @@ public final class CdnSetupStatus /////////////////////// // revalidation hooks // /////////////////////// - long enabledHooks = hooks.stream().filter(h -> !Boolean.FALSE.equals(h.getValueBoolean("enabled"))).count(); + long enabledHooks = hooks.stream().filter(h -> Boolean.TRUE.equals(h.getValueBoolean("enabled"))).count(); items.add(enabledHooks == 0 ? new Item("Revalidation hooks", CdnTables.REVALIDATION_HOOK, State.OPTIONAL, "None.", "Add one if a front end keeps its own cache (for example Next.js) that must refresh before the CDN is purged.") : new Item("Revalidation hooks", CdnTables.REVALIDATION_HOOK, State.OK, enabledHooks + " hook" + (enabledHooks == 1 ? "" : "s") + " turned on.", null)); @@ -182,9 +186,10 @@ public final class CdnSetupStatus //////////////////// // origin checks // //////////////////// - items.add(checks.isEmpty() - ? new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OPTIONAL, "Never run.", "Run Check Origin Headers on Hostnames. Hostnames that are not Off are also checked every night.") - : new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OK, checks.size() + " check" + (checks.size() == 1 ? "" : "s") + " recorded.", null)); + items.add(checks == 0 + ? new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OPTIONAL, "Never run.", + "Run Check Origin Headers on Hostnames." + (scheduled ? " Hostnames that are not Off are also checked every night." : " No scheduler is configured, so nothing runs it automatically.")) + : new Item("Origin checks", CdnTables.ORIGIN_CHECK, State.OK, checks + " check" + (checks == 1 ? "" : "s") + " recorded.", null)); return (items); } diff --git a/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java b/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java index 6c5cc83..08b0565 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java +++ b/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java @@ -17,12 +17,17 @@ package com.kingsrook.qbits.cdn.checks; +import javax.net.ssl.SNIHostName; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import java.io.BufferedInputStream; import java.io.IOException; +import java.io.InputStream; import java.net.InetAddress; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -262,39 +267,214 @@ public final class CdnOriginCheck /******************************************************************************* - ** The real fetcher: HTTPS GET, no redirects, public addresses only. + ** Whether an address is one an origin check may connect to: public unicast only. + ** Refuses loopback, private, link-local, carrier-grade NAT (100.64.0.0/10), + ** 0.0.0.0/8, benchmarking (198.18.0.0/15), multicast and reserved IPv4; IPv6 + ** unique-local (fc00::/7), link-local, multicast; and IPv6 forms that embed an + ** IPv4 address (v4-mapped and -compatible, NAT64 64:ff9b::/96, 6to4), which are + ** unwrapped and checked as IPv4. + *******************************************************************************/ + public static boolean isPublic(InetAddress address) + { + byte[] b = address.getAddress(); + if(b.length == 4) + { + return (isPublicV4(b[0] & 0xff, b[1] & 0xff, b[2] & 0xff)); + } + boolean zeros12 = true; + for(int i = 0; i < 12; i++) + { + zeros12 &= b[i] == 0; + } + boolean mapped = true; + for(int i = 0; i < 10; i++) + { + mapped &= b[i] == 0; + } + mapped &= (b[10] & 0xff) == 0xff && (b[11] & 0xff) == 0xff; + boolean nat64 = (b[0] & 0xff) == 0x00 && (b[1] & 0xff) == 0x64 && (b[2] & 0xff) == 0xff && (b[3] & 0xff) == 0x9b; + for(int i = 4; i < 12 && nat64; i++) + { + nat64 &= b[i] == 0; + } + if(zeros12 || mapped || nat64) + { + return (isPublicV4(b[12] & 0xff, b[13] & 0xff, b[14] & 0xff)); + } + if((b[0] & 0xff) == 0x20 && (b[1] & 0xff) == 0x02 && !isPublicV4(b[2] & 0xff, b[3] & 0xff, b[4] & 0xff)) + { + return (false); + } + int first = b[0] & 0xff; + boolean ula = (first & 0xfe) == 0xfc; + boolean linkLocal = first == 0xfe && (b[1] & 0xc0) == 0x80; + boolean siteLocal = first == 0xfe && (b[1] & 0xc0) == 0xc0; + return (!ula && !linkLocal && !siteLocal && first != 0xff); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean isPublicV4(int a, int b, int c) + { + if(a == 0 || a == 10 || a == 127 || a >= 224) + { + return (false); + } + if(a == 100 && b >= 64 && b <= 127) + { + return (false); + } + if(a == 169 && b == 254) + { + return (false); + } + if(a == 172 && b >= 16 && b <= 31) + { + return (false); + } + if(a == 192 && (b == 168 || (b == 0 && (c == 0 || c == 2)) || (b == 88 && c == 99))) + { + return (false); + } + if(a == 198 && (b == 18 || b == 19 || (b == 51 && c == 100))) + { + return (false); + } + return (!(a == 203 && b == 0 && c == 113)); + } + + + + /******************************************************************************* + ** The address to connect to: every address the name resolves to must be public, + ** and the first one is used, so the connection goes to what was checked. + *******************************************************************************/ + static InetAddress pick(InetAddress[] resolved) throws IOException + { + if(resolved == null || resolved.length == 0) + { + throw (new IOException("the host did not resolve")); + } + for(InetAddress address : resolved) + { + if(!isPublic(address)) + { + throw (new IOException("the host resolves to a non-public address")); + } + } + return (resolved[0]); + } + + + + /******************************************************************************* + ** Read a status line and headers (and nothing else) from an HTTP/1.1 response, + ** within a total deadline and a size limit. + *******************************************************************************/ + static Response readResponse(InputStream in, long deadlineNanos) throws IOException + { + String status = readLine(in, deadlineNanos); + int total = status.length(); + String[] parts = status.split(" ", 3); + if(parts.length < 2 || !parts[0].startsWith("HTTP/")) + { + throw (new IOException("not an HTTP response")); + } + int code; + try + { + code = Integer.parseInt(parts[1]); + } + catch(NumberFormatException e) + { + throw (new IOException("bad status line")); + } + Map headers = new LinkedHashMap<>(); + while(true) + { + String line = readLine(in, deadlineNanos); + total += line.length(); + if(line.isEmpty()) + { + return (new Response(code, headers)); + } + if(total > 16_384) + { + throw (new IOException("the headers are too large")); + } + int colon = line.indexOf(':'); + if(colon > 0) + { + headers.merge(line.substring(0, colon).strip().toLowerCase(Locale.ROOT), line.substring(colon + 1).strip(), (x, y) -> x + ", " + y); + } + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String readLine(InputStream in, long deadlineNanos) throws IOException + { + StringBuilder line = new StringBuilder(); + while(true) + { + if(System.nanoTime() > deadlineNanos) + { + throw (new IOException("the origin was too slow")); + } + int c = in.read(); + if(c < 0) + { + throw (new IOException("the connection closed early")); + } + if(c == '\n') + { + return (line.toString().stripTrailing()); + } + line.append((char) c); + if(line.length() > 8192) + { + throw (new IOException("a header line is too long")); + } + } + } + + + + /******************************************************************************* + ** The real fetcher: resolve once, refuse non-public addresses, connect to the + ** address that was checked (TLS with the name as SNI and for verification), send + ** one GET, read the status line and headers only, and close. No redirects, no body. *******************************************************************************/ static final class HttpFetcher implements Fetcher { - private final HttpClient client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NEVER).connectTimeout(Duration.ofSeconds(5)).build(); - - - /******************************************************************************* ** {@inheritDoc} *******************************************************************************/ @Override public Response fetch(String host, String path) throws IOException { - for(InetAddress address : InetAddress.getAllByName(host)) + InetAddress address = pick(InetAddress.getAllByName(host)); + long deadline = System.nanoTime() + Duration.ofSeconds(10).toNanos(); + try(Socket plain = new Socket()) { - if(address.isAnyLocalAddress() || address.isLoopbackAddress() || address.isLinkLocalAddress() || address.isSiteLocalAddress() || address.isMulticastAddress()) - { - throw (new IOException("the host resolves to a non-public address")); - } - } - try - { - HttpResponse response = client.send(HttpRequest.newBuilder(URI.create("https://" + host + path)).timeout(Duration.ofSeconds(10)).GET().build(), - HttpResponse.BodyHandlers.discarding()); - Map headers = new LinkedHashMap<>(); - response.headers().map().forEach((key, values) -> headers.put(key.toLowerCase(Locale.ROOT), String.join(", ", values))); - return (new Response(response.statusCode(), headers)); - } - catch(InterruptedException e) - { - Thread.currentThread().interrupt(); - throw (new IOException("interrupted", e)); + plain.connect(new InetSocketAddress(address, 443), 5000); + plain.setSoTimeout(5000); + SSLSocket socket = (SSLSocket) ((SSLSocketFactory) SSLSocketFactory.getDefault()).createSocket(plain, host, 443, true); + SSLParameters parameters = socket.getSSLParameters(); + parameters.setServerNames(List.of(new SNIHostName(host))); + parameters.setEndpointIdentificationAlgorithm("HTTPS"); + socket.setSSLParameters(parameters); + socket.startHandshake(); + socket.getOutputStream().write(("GET " + path + " HTTP/1.1\r\nHost: " + host + "\r\nUser-Agent: qbit-cdn-origin-check\r\nAccept: */*\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.US_ASCII)); + socket.getOutputStream().flush(); + return (readResponse(new BufferedInputStream(socket.getInputStream()), deadline)); } } } diff --git a/src/main/java/com/kingsrook/qbits/cdn/processes/CdnDefaultsProcesses.java b/src/main/java/com/kingsrook/qbits/cdn/processes/CdnDefaultsProcesses.java index 7fa5f3c..a2bd151 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/processes/CdnDefaultsProcesses.java +++ b/src/main/java/com/kingsrook/qbits/cdn/processes/CdnDefaultsProcesses.java @@ -398,7 +398,9 @@ public final class CdnDefaultsProcesses *******************************************************************************/ public static class DefaultsLoad extends AbstractLoadStep implements ProcessSummaryProviderInterface { - private final ProcessSummaryLine added = line(Status.OK, "default row will be added", "default rows will be added", "default row was added", "default rows were added"); + private final ProcessSummaryLine added = line(Status.OK, "default row will be added", "default rows will be added", "default row was added", "default rows were added"); + private final ProcessSummaryLine failed = line(Status.ERROR, "default row could not be added", "default rows could not be added", "default row could not be added", "default rows could not be added"); + private final List problems = new ArrayList<>(); @@ -410,8 +412,16 @@ public final class CdnDefaultsProcesses { for(QRecord inserted : new InsertAction().execute(new InsertInput(input.getTableName()).withRecords(input.getRecords())).getRecords()) { - added.incrementCountAndAddPrimaryKey(inserted.getValue("id")); - output.getRecords().add(inserted); + if(inserted.getErrors() == null || inserted.getErrors().isEmpty()) + { + added.incrementCountAndAddPrimaryKey(inserted.getValue("id")); + output.getRecords().add(inserted); + } + else + { + failed.incrementCount(); + inserted.getErrors().forEach(error -> problems.add(error.getMessage())); + } } } @@ -425,6 +435,8 @@ public final class CdnDefaultsProcesses { ArrayList lines = new ArrayList<>(); added.addSelfToListIfAnyCount(lines); + failed.setBulletsOfText(new ArrayList<>(problems)); + failed.addSelfToListIfAnyCount(lines); return (lines); } } diff --git a/src/main/java/com/kingsrook/qbits/cdn/processes/CdnOriginCheckProcesses.java b/src/main/java/com/kingsrook/qbits/cdn/processes/CdnOriginCheckProcesses.java index 8631722..91668fa 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/processes/CdnOriginCheckProcesses.java +++ b/src/main/java/com/kingsrook/qbits/cdn/processes/CdnOriginCheckProcesses.java @@ -27,6 +27,7 @@ import com.kingsrook.qbits.cdn.permissions.CdnPermissions; import com.kingsrook.qqq.backend.core.actions.processes.BackendStep; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; @@ -46,6 +47,7 @@ import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwith import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface; import static com.kingsrook.qbits.cdn.processes.CdnProcesses.line; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; /******************************************************************************* @@ -56,6 +58,8 @@ import static com.kingsrook.qbits.cdn.processes.CdnProcesses.line; *******************************************************************************/ public final class CdnOriginCheckProcesses { + private static final QLogger LOG = QLogger.getLogger(CdnOriginCheckProcesses.class); + public static final String CHECK_ORIGIN = "cdnCheckOrigin"; public static final String SCHEDULED_ORIGIN = "cdnScheduledCheckOrigin"; @@ -198,7 +202,14 @@ public final class CdnOriginCheckProcesses for(QRecord hostname : new QueryAction().execute(new QueryInput(CdnTables.HOSTNAME) .withFilter(new QQueryFilter(new QFilterCriteria("stage", QCriteriaOperator.NOT_EQUALS, CdnHostnameStage.OFF.name())))).getRecords()) { - CdnOriginCheck.checkAndRecord(hostname); + try + { + CdnOriginCheck.checkAndRecord(hostname); + } + catch(RuntimeException | QException e) + { + LOG.warn("An origin check failed", e, logPair("hostname", hostname.getValueString("hostname"))); + } } } } diff --git a/src/test/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheckSafetyTest.java b/src/test/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheckSafetyTest.java new file mode 100644 index 0000000..9a95e51 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheckSafetyTest.java @@ -0,0 +1,142 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.checks; + + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** The safety of the origin check's own HTTP code: which addresses it refuses + ** (every literal is parsed without DNS), that it connects to the address it + ** checked, and that it reads only a bounded status line and headers. + *******************************************************************************/ +class CdnOriginCheckSafetyTest +{ + /******************************************************************************* + ** + *******************************************************************************/ + private static boolean isPublic(String literal) throws Exception + { + return (CdnOriginCheck.isPublic(InetAddress.getByName(literal))); + } + + + + /******************************************************************************* + ** Public unicast is allowed. + *******************************************************************************/ + @Test + void publicAddressesAreAllowed() throws Exception + { + for(String literal : List.of("93.184.216.34", "8.8.8.8", "100.63.255.255", "100.128.0.1", "172.15.0.1", "172.32.0.1", "198.17.0.1", "198.20.0.1", "2606:4700:4700::1111", "2001:4860:4860::8888")) + { + assertThat(isPublic(literal)).as(literal).isTrue(); + } + } + + + + /******************************************************************************* + ** Everything non-public is refused, including cloud metadata addresses and the + ** IPv6 forms that carry an IPv4 address. + *******************************************************************************/ + @Test + void nonPublicAddressesAreRefused() throws Exception + { + for(String literal : List.of("127.0.0.1", "10.1.2.3", "172.16.0.1", "172.31.255.255", "192.168.1.1", "169.254.169.254", "100.64.0.1", "100.100.100.200", "100.127.255.255", + "0.0.0.0", "0.1.2.3", "198.18.0.1", "198.19.255.255", "224.0.0.1", "255.255.255.255", "192.0.2.1", "203.0.113.9", + "::1", "::", "fe80::1", "fc00::1", "fd00:ec2::254", "fdff::1", "ff02::1", + "::ffff:127.0.0.1", "::ffff:169.254.169.254", "::ffff:10.0.0.1", "::127.0.0.1", "64:ff9b::7f00:1", "64:ff9b::a9fe:a9fe", "2002:7f00:1::1", "2002:a9fe:a9fe::1")) + { + assertThat(isPublic(literal)).as(literal).isFalse(); + } + assertThat(isPublic("::ffff:8.8.8.8")).as("a mapped public address is judged by its IPv4 part").isTrue(); + } + + + + /******************************************************************************* + ** One bad address in a name's answer refuses the whole name, and the address + ** used is one that was checked (the first). + *******************************************************************************/ + @Test + void pickRefusesAnyBadAddressAndReturnsACheckedOne() throws Exception + { + InetAddress good = InetAddress.getByName("93.184.216.34"); + assertThat(CdnOriginCheck.pick(new InetAddress[] {good, InetAddress.getByName("8.8.8.8")})).isSameAs(good); + assertThatThrownBy(() -> CdnOriginCheck.pick(new InetAddress[] {good, InetAddress.getByName("169.254.169.254")})).isInstanceOf(IOException.class).hasMessageContaining("non-public"); + assertThatThrownBy(() -> CdnOriginCheck.pick(new InetAddress[0])).isInstanceOf(IOException.class); + assertThatThrownBy(() -> new CdnOriginCheck.HttpFetcher().fetch("localhost", "/")).isInstanceOf(IOException.class).hasMessageContaining("non-public"); + } + + + + /******************************************************************************* + ** Only the status line and headers are read, names are lower-cased, repeats are + ** joined, and the body is never touched. + *******************************************************************************/ + @Test + void readsOnlyStatusAndHeaders() throws Exception + { + InputStream in = new ByteArrayInputStream("HTTP/1.1 200 OK\r\nCache-Control: public, max-age=60\r\nVary: a\r\nVary: b\r\n\r\nBODY".getBytes(StandardCharsets.US_ASCII)); + CdnOriginCheck.Response response = CdnOriginCheck.readResponse(in, System.nanoTime() + 1_000_000_000L); + assertThat(response.status()).isEqualTo(200); + assertThat(response.headers()).containsEntry("cache-control", "public, max-age=60").containsEntry("vary", "a, b"); + assertThat(in.read()).as("the body is left unread").isEqualTo('B'); + } + + + + /******************************************************************************* + ** Malformed, oversized, truncated and too-slow responses are refused. + *******************************************************************************/ + @Test + void refusesBadResponses() throws Exception + { + long deadline = System.nanoTime() + 1_000_000_000L; + assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("garbage\r\n\r\n"), deadline)).isInstanceOf(IOException.class); + assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 abc OK\r\n\r\n"), deadline)).isInstanceOf(IOException.class); + assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\nA: b\r\n"), deadline)).hasMessageContaining("closed early"); + assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\nA: " + "x".repeat(9000) + "\r\n\r\n"), deadline)).hasMessageContaining("too long"); + StringBuilder many = new StringBuilder("HTTP/1.1 200 OK\r\n"); + for(int i = 0; i < 400; i++) + { + many.append("H").append(i).append(": ").append("v".repeat(60)).append("\r\n"); + } + assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream(many.append("\r\n").toString()), deadline)).hasMessageContaining("too large"); + assertThatThrownBy(() -> CdnOriginCheck.readResponse(stream("HTTP/1.1 200 OK\r\n\r\n"), System.nanoTime() - 1)).hasMessageContaining("too slow"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static InputStream stream(String text) + { + return (new ByteArrayInputStream(text.getBytes(StandardCharsets.US_ASCII))); + } +} diff --git a/src/test/java/com/kingsrook/qbits/cdn/tables/CdnSetupStatusTest.java b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnSetupStatusTest.java index 0502501..68217ba 100644 --- a/src/test/java/com/kingsrook/qbits/cdn/tables/CdnSetupStatusTest.java +++ b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnSetupStatusTest.java @@ -69,12 +69,11 @@ class CdnSetupStatusTest void aCompleteSetupIsOk() { List providers = List.of(new QRecord().withValue("providerType", "CLOUDFRONT").withValue("enabled", true).withValue("lastHealthStatus", "OK")); - List hostnames = List.of(new QRecord().withValue("stage", "LIVE")); + List hostnames = List.of(new QRecord().withValue("stage", "LIVE").withValue("purgeEnabled", true)); List policies = List.of(new QRecord().withValue("enabled", true)); List rules = List.of(new QRecord().withValue("tableName", "post").withValue("enabled", true)); List hooks = List.of(new QRecord().withValue("enabled", true)); - List checks = List.of(new QRecord()); - List items = CdnSetupStatus.build(providers, hostnames, policies, rules, hooks, checks, List.of("post")); + List items = CdnSetupStatus.build(providers, hostnames, policies, rules, hooks, 1, List.of("post"), true); assertThat(items).extracting(Item::state).containsOnly(State.OK); } @@ -88,14 +87,24 @@ class CdnSetupStatusTest { QRecord none = new QRecord().withValue("providerType", "NONE").withValue("enabled", true); List items = CdnSetupStatus.build(List.of(none), List.of(new QRecord().withValue("stage", "OFF")), List.of(new QRecord().withValue("enabled", false)), - List.of(), List.of(), List.of(), List.of("post", "page")); + List.of(), List.of(), 0, List.of("post", "page"), false); assertThat(item(items, "Providers").state()).isEqualTo(State.DEFAULT); assertThat(item(items, "Hostnames").state()).isEqualTo(State.DEFAULT); assertThat(item(items, "Cache policies").state()).isEqualTo(State.DEFAULT); assertThat(item(items, "Tag rules").detail()).contains("post, page"); + assertThat(item(items, "Origin checks").nextStep()).contains("No scheduler is configured"); + + /////////////////////////////////////////////////////////////////////////////////// + // like the runtime: Live needs purge enabled, and a null enabled counts as off // + /////////////////////////////////////////////////////////////////////////////////// + List unpurgeable = CdnSetupStatus.build(List.of(new QRecord().withValue("providerType", "CLOUDFRONT")), List.of(new QRecord().withValue("stage", "LIVE").withValue("purgeEnabled", false)), + List.of(), List.of(), List.of(), 0, List.of(), true); + assertThat(item(unpurgeable, "Providers").state()).isEqualTo(State.TODO); + assertThat(item(unpurgeable, "Hostnames").state()).isEqualTo(State.DEFAULT); + assertThat(item(unpurgeable, "Origin checks").nextStep()).contains("every night"); QRecord failing = new QRecord().withValue("providerType", "FASTLY").withValue("enabled", true).withValue("lastHealthStatus", "FAILED"); - assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), List.of(), List.of()), "Providers").state()).isEqualTo(State.TODO); - assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), List.of(), List.of()), "Hostnames").state()).isEqualTo(State.TODO); + assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), 0, List.of(), false), "Providers").state()).isEqualTo(State.TODO); + assertThat(item(CdnSetupStatus.build(List.of(failing), List.of(), List.of(), List.of(), List.of(), 0, List.of(), false), "Hostnames").state()).isEqualTo(State.TODO); } } diff --git a/src/test/java/com/kingsrook/qbits/cdn/tables/CdnTablesGuardrailTest.java b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnTablesGuardrailTest.java index 0b838c7..2f9fbf4 100644 --- a/src/test/java/com/kingsrook/qbits/cdn/tables/CdnTablesGuardrailTest.java +++ b/src/test/java/com/kingsrook/qbits/cdn/tables/CdnTablesGuardrailTest.java @@ -160,6 +160,23 @@ class CdnTablesGuardrailTest + /******************************************************************************* + ** A host's own help in the slot is never replaced by the bundle. + *******************************************************************************/ + @Test + void hostHelpIsKept() throws Exception + { + QTableMetaData table = qInstance.getTable(CdnTables.PROVIDER); + table.withHelpContent(CdnHelp.SLOT, new QHelpContent("Host help for providers.")); + qInstance.getApp("cdnAdmin").withHelpContent(CdnHelp.SLOT, new QHelpContent("Host help for the app.")); + CdnHelp.apply(qInstance, "cdnAdmin"); + assertThat(table.getHelpContent().get(CdnHelp.SLOT)).singleElement().satisfies(h -> assertThat(h.getContent()).isEqualTo("Host help for providers.")); + assertThat(qInstance.getApp("cdnAdmin").getHelpContent().get(CdnHelp.SLOT)).singleElement().satisfies(h -> assertThat(h.getContent()).isEqualTo("Host help for the app.")); + assertThat(qInstance.getTable(CdnTables.HOSTNAME).getHelpContent().get(CdnHelp.SLOT).get(0).getContent()).contains("How rows get created"); + } + + + /******************************************************************************* ** *******************************************************************************/ -- 2.54.0 From 5a0d3e824a7ecdf943214e76967481e3ad7c0572 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 17:12:41 -0500 Subject: [PATCH 3/3] fix: hard socket deadline, IPv6 allowlist, port stripping for the origin check Co-Authored-By: Claude Opus 5.5 --- .../qbits/cdn/checks/CdnOriginCheck.java | 180 +++++++++++++----- .../cdn/checks/CdnOriginCheckSafetyTest.java | 92 +++++++++ 2 files changed, 229 insertions(+), 43 deletions(-) diff --git a/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java b/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java index 08b0565..c0f95fc 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java +++ b/src/main/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheck.java @@ -36,6 +36,10 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import com.kingsrook.qbits.cdn.CdnRuntime; import com.kingsrook.qbits.cdn.headers.CdnResponseHeaderPolicy; import com.kingsrook.qbits.cdn.model.CdnTables; @@ -247,7 +251,7 @@ public final class CdnOriginCheck String originHost = provider.getValueString("originHost"); if(StringUtils.hasContent(originHost)) { - return (originHost.strip().replaceFirst(":\\d+$", "")); + return (stripPort(originHost.strip())); } } } @@ -256,6 +260,26 @@ public final class CdnOriginCheck + /******************************************************************************* + ** The host without a port: "[v6]:443" gives v6, "name:443" gives name, and a bare + ** IPv6 literal or a plain name is left alone. + *******************************************************************************/ + static String stripPort(String hostAndPort) + { + if(hostAndPort.startsWith("[") && hostAndPort.contains("]")) + { + return (hostAndPort.substring(1, hostAndPort.indexOf(']'))); + } + int colon = hostAndPort.indexOf(':'); + if(colon > 0 && colon == hostAndPort.lastIndexOf(':') && hostAndPort.substring(colon + 1).matches("\\d+")) + { + return (hostAndPort.substring(0, colon)); + } + return (hostAndPort); + } + + + /******************************************************************************* ** Lower case, no spaces. *******************************************************************************/ @@ -268,11 +292,13 @@ public final class CdnOriginCheck /******************************************************************************* ** Whether an address is one an origin check may connect to: public unicast only. - ** Refuses loopback, private, link-local, carrier-grade NAT (100.64.0.0/10), - ** 0.0.0.0/8, benchmarking (198.18.0.0/15), multicast and reserved IPv4; IPv6 - ** unique-local (fc00::/7), link-local, multicast; and IPv6 forms that embed an - ** IPv4 address (v4-mapped and -compatible, NAT64 64:ff9b::/96, 6to4), which are - ** unwrapped and checked as IPv4. + ** IPv4 refuses loopback, private, link-local, carrier-grade NAT (100.64.0.0/10), + ** 0.0.0.0/8, benchmarking (198.18.0.0/15), documentation, multicast and reserved. + ** IPv6 is an allowlist: only global unicast 2000::/3 passes, minus Teredo + ** (2001::/32) and documentation (2001:db8::/32); 6to4 (2002::/16) passes only + ** when its embedded IPv4 is public. The forms that embed an IPv4 address (::/96, + ** v4-mapped, IPv4-translated, NAT64 64:ff9b::/96) are unwrapped and judged as + ** IPv4; the local-use NAT64 prefix 64:ff9b:1::/48 is refused. *******************************************************************************/ public static boolean isPublic(InetAddress address) { @@ -281,35 +307,64 @@ public final class CdnOriginCheck { return (isPublicV4(b[0] & 0xff, b[1] & 0xff, b[2] & 0xff)); } - boolean zeros12 = true; - for(int i = 0; i < 12; i++) - { - zeros12 &= b[i] == 0; - } - boolean mapped = true; - for(int i = 0; i < 10; i++) - { - mapped &= b[i] == 0; - } - mapped &= (b[10] & 0xff) == 0xff && (b[11] & 0xff) == 0xff; - boolean nat64 = (b[0] & 0xff) == 0x00 && (b[1] & 0xff) == 0x64 && (b[2] & 0xff) == 0xff && (b[3] & 0xff) == 0x9b; - for(int i = 4; i < 12 && nat64; i++) - { - nat64 &= b[i] == 0; - } - if(zeros12 || mapped || nat64) + return (isPublicV6(b)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + static boolean isPublicV6(byte[] b) + { + if(zero(b, 0, 12) || (zero(b, 0, 10) && ff(b, 10)) || (zero(b, 0, 8) && ff(b, 8) && b[10] == 0 && b[11] == 0)) { return (isPublicV4(b[12] & 0xff, b[13] & 0xff, b[14] & 0xff)); } - if((b[0] & 0xff) == 0x20 && (b[1] & 0xff) == 0x02 && !isPublicV4(b[2] & 0xff, b[3] & 0xff, b[4] & 0xff)) + if((b[0] & 0xff) == 0x00 && (b[1] & 0xff) == 0x64 && (b[2] & 0xff) == 0xff && (b[3] & 0xff) == 0x9b) + { + return (zero(b, 4, 12) && isPublicV4(b[12] & 0xff, b[13] & 0xff, b[14] & 0xff)); + } + if((b[0] & 0xe0) != 0x20) { return (false); } - int first = b[0] & 0xff; - boolean ula = (first & 0xfe) == 0xfc; - boolean linkLocal = first == 0xfe && (b[1] & 0xc0) == 0x80; - boolean siteLocal = first == 0xfe && (b[1] & 0xc0) == 0xc0; - return (!ula && !linkLocal && !siteLocal && first != 0xff); + if((b[0] & 0xff) == 0x20 && (b[1] & 0xff) == 0x01 && ((b[2] == 0 && b[3] == 0) || ((b[2] & 0xff) == 0x0d && (b[3] & 0xff) == 0xb8))) + { + return (false); + } + if((b[0] & 0xff) == 0x20 && (b[1] & 0xff) == 0x02) + { + return (isPublicV4(b[2] & 0xff, b[3] & 0xff, b[4] & 0xff)); + } + return (true); + } + + + + /******************************************************************************* + ** Whether bytes from (inclusive) to (exclusive) are all zero. + *******************************************************************************/ + private static boolean zero(byte[] b, int from, int to) + { + for(int i = from; i < to; i++) + { + if(b[i] != 0) + { + return (false); + } + } + return (true); + } + + + + /******************************************************************************* + ** Whether two bytes at an index are ff ff. + *******************************************************************************/ + private static boolean ff(byte[] b, int at) + { + return ((b[at] & 0xff) == 0xff && (b[at + 1] & 0xff) == 0xff); } @@ -453,28 +508,67 @@ public final class CdnOriginCheck *******************************************************************************/ static final class HttpFetcher implements Fetcher { + private static final ScheduledExecutorService WATCHDOG = Executors.newSingleThreadScheduledExecutor(runnable -> + { + Thread thread = new Thread(runnable, "cdn-origin-check-watchdog"); + thread.setDaemon(true); + return (thread); + }); + + + /******************************************************************************* ** {@inheritDoc} *******************************************************************************/ @Override public Response fetch(String host, String path) throws IOException { - InetAddress address = pick(InetAddress.getAllByName(host)); - long deadline = System.nanoTime() + Duration.ofSeconds(10).toNanos(); + InetAddress address = pick(InetAddress.getAllByName(host)); + return (fetch(address, 443, host, path, Duration.ofSeconds(10))); + } + + + + /******************************************************************************* + ** Fetch from one checked address within one hard deadline that covers connect, + ** the TLS handshake and the reads: at the deadline the socket is closed, which + ** fails whatever read or write is blocked. + *******************************************************************************/ + Response fetch(InetAddress address, int port, String host, String path, Duration limit) throws IOException + { + long deadline = System.nanoTime() + limit.toNanos(); try(Socket plain = new Socket()) { - plain.connect(new InetSocketAddress(address, 443), 5000); - plain.setSoTimeout(5000); - SSLSocket socket = (SSLSocket) ((SSLSocketFactory) SSLSocketFactory.getDefault()).createSocket(plain, host, 443, true); - SSLParameters parameters = socket.getSSLParameters(); - parameters.setServerNames(List.of(new SNIHostName(host))); - parameters.setEndpointIdentificationAlgorithm("HTTPS"); - socket.setSSLParameters(parameters); - socket.startHandshake(); - socket.getOutputStream().write(("GET " + path + " HTTP/1.1\r\nHost: " + host + "\r\nUser-Agent: qbit-cdn-origin-check\r\nAccept: */*\r\nConnection: close\r\n\r\n") - .getBytes(StandardCharsets.US_ASCII)); - socket.getOutputStream().flush(); - return (readResponse(new BufferedInputStream(socket.getInputStream()), deadline)); + ScheduledFuture guard = WATCHDOG.schedule(() -> + { + try + { + plain.close(); + } + catch(IOException e) + { + // already closed + } + }, limit.toMillis(), TimeUnit.MILLISECONDS); + try + { + plain.connect(new InetSocketAddress(address, port), (int) Math.min(5000, limit.toMillis())); + plain.setSoTimeout(5000); + SSLSocket socket = (SSLSocket) ((SSLSocketFactory) SSLSocketFactory.getDefault()).createSocket(plain, host, port, true); + SSLParameters parameters = socket.getSSLParameters(); + parameters.setServerNames(List.of(new SNIHostName(host))); + parameters.setEndpointIdentificationAlgorithm("HTTPS"); + socket.setSSLParameters(parameters); + socket.startHandshake(); + socket.getOutputStream().write(("GET " + path + " HTTP/1.1\r\nHost: " + host + "\r\nUser-Agent: qbit-cdn-origin-check\r\nAccept: */*\r\nConnection: close\r\n\r\n") + .getBytes(StandardCharsets.US_ASCII)); + socket.getOutputStream().flush(); + return (readResponse(new BufferedInputStream(socket.getInputStream()), deadline)); + } + finally + { + guard.cancel(false); + } } } } diff --git a/src/test/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheckSafetyTest.java b/src/test/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheckSafetyTest.java index 9a95e51..0fca0ad 100644 --- a/src/test/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheckSafetyTest.java +++ b/src/test/java/com/kingsrook/qbits/cdn/checks/CdnOriginCheckSafetyTest.java @@ -78,6 +78,98 @@ class CdnOriginCheckSafetyTest + /******************************************************************************* + ** IPv6 is an allowlist, built from raw bytes so the IPv6 path is really used (Java + ** turns ::ffff:a.b.c.d literals into IPv4 addresses). + *******************************************************************************/ + @Test + void ipv6IsAnAllowlistOfGlobalUnicast() throws Exception + { + for(String hex : List.of("00000000000000000000ffff7f000001", "00000000000000000000ffffa9fea9fe", "0000000000000000ffff00007f000001", "0064ff9b0000000000000000a9fea9fe", + "0064ff9b0001000000000000" + "7f000001", "20010db8000000000000000000000001", "01000000000000000000000000000001", "20010000000000000000000000000001", + "2002 7f00 0001 0000 0000 0000 0000 0001".replace(" ", ""), "fc000000000000000000000000000001", "fe800000000000000000000000000001", "ff020000000000000000000000000001", + "00000000000000000000000000000001", "00000000000000000000000000000000", "40000000000000000000000000000001")) + { + assertThat(CdnOriginCheck.isPublicV6(bytes(hex))).as(hex).isFalse(); + } + for(String hex : List.of("2606470047000000000000000000" + "1111", "20014860486000000000000000008888", "0064ff9b000000000000000008080808", "00000000000000000000ffff08080808", + "2002080808080000000000000000" + "0001")) + { + assertThat(CdnOriginCheck.isPublicV6(bytes(hex))).as(hex).isTrue(); + } + assertThat(CdnOriginCheck.isPublic(java.net.Inet6Address.getByAddress(bytes("20014860486000000000000000008888")))).isTrue(); + } + + + + /******************************************************************************* + ** An origin that drips one byte at a time (here, never finishing the TLS handshake) + ** is cut off at the hard deadline, not after hours. + *******************************************************************************/ + @Test + void aDrippingOriginIsCutOffAtTheDeadline() throws Exception + { + try(java.net.ServerSocket server = new java.net.ServerSocket(0, 1, InetAddress.getByName("127.0.0.1"))) + { + Thread dripper = new Thread(() -> + { + try(java.net.Socket client = server.accept()) + { + for(int i = 0; i < 60; i++) + { + client.getOutputStream().write(0x16); + client.getOutputStream().flush(); + Thread.sleep(300); + } + } + catch(Exception e) + { + // the client gave up + } + }); + dripper.setDaemon(true); + dripper.start(); + long start = System.nanoTime(); + assertThatThrownBy(() -> new CdnOriginCheck.HttpFetcher().fetch(InetAddress.getByName("127.0.0.1"), server.getLocalPort(), "localhost", "/", java.time.Duration.ofSeconds(2))) + .isInstanceOf(IOException.class); + long seconds = (System.nanoTime() - start) / 1_000_000_000L; + assertThat(seconds).as("gave up at about the 2 second deadline").isLessThan(6); + } + } + + + + /******************************************************************************* + ** A port is stripped from name:port and [v6]:port, never from a bare IPv6 literal. + *******************************************************************************/ + @Test + void stripsPortsOnlyWhenThereIsOne() + { + assertThat(CdnOriginCheck.stripPort("origin.example.com:443")).isEqualTo("origin.example.com"); + assertThat(CdnOriginCheck.stripPort("origin.example.com")).isEqualTo("origin.example.com"); + assertThat(CdnOriginCheck.stripPort("[2606:4700::1111]:443")).isEqualTo("2606:4700::1111"); + assertThat(CdnOriginCheck.stripPort("2606:4700::1111")).isEqualTo("2606:4700::1111"); + assertThat(CdnOriginCheck.stripPort("2606:4700:4700::1")).isEqualTo("2606:4700:4700::1"); + } + + + + /******************************************************************************* + ** Hex to 16 bytes. + *******************************************************************************/ + private static byte[] bytes(String hex) + { + assertThat(hex).hasSize(32); + byte[] out = new byte[16]; + for(int i = 0; i < 16; i++) + { + out[i] = (byte) Integer.parseInt(hex.substring(i * 2, i * 2 + 2), 16); + } + return (out); + } + + + /******************************************************************************* ** One bad address in a name's answer refuses the whole name, and the address ** used is one that was checked (the first). -- 2.54.0