From 9ff55f4c23894c8645f3dfd0bee2bfe38b4b1557 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 16 Oct 2026 11:38:34 -0500 Subject: [PATCH] fix(health): real CloudFront health probe; UP/not-testable instead of UNKNOWN The cdn.provider. health check returned UNKNOWN ("CDN provider does not support connection tests") for any adapter without CdnOperations, i.e. the NONE placeholder and Souin. The central evaluator never alerts or changes state on UNKNOWN, so such a check is permanently useless. - New optional SPI CdnHealthProbe/CdnHealth; the health check prefers it, then the connection test, then reports UP with testable=false. - CloudFront implements it: GetDistribution -> UP when enabled and Deployed, DEGRADED while InProgress, DOWN when NoSuchDistribution or disabled; throttling/5xx/network errors DEGRADED; rejected or missing credentials DOWN. On AccessDenied it falls back to GetDistributionConfig (the kof22 policy grants that, not GetDistribution) and reports testable=false; if both are denied, UP with testable=false. - CloudWatch AWS/CloudFront 5xxErrorRate (Average, 5-min periods, last 30 minutes) goes into details; above 5% the check is DEGRADED. Best effort. - Messages are fixed text; details hold only booleans, numbers and known states. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + .../cdn/health/CdnProviderHealthChecks.java | 83 +++- .../cloudfront/CloudFrontCdnProvider.java | 211 +++++++++- .../kingsrook/qbits/cdn/spi/CdnHealth.java | 54 +++ .../qbits/cdn/spi/CdnHealthProbe.java | 32 ++ .../health/CdnProviderHealthChecksTest.java | 68 ++- .../cloudfront/CloudFrontHealthTest.java | 394 ++++++++++++++++++ 7 files changed, 825 insertions(+), 18 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealth.java create mode 100644 src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealthProbe.java create mode 100644 src/test/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontHealthTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 080b061..3f34cc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ All notable changes to this project are documented here. The format follows Keep ### Fixed +- CDN health checks no longer sit at UNKNOWN (which the central evaluator never alerts on). CloudFront checks the distribution itself: GetDistribution gives UP (enabled and Deployed), DEGRADED (InProgress, origin header missing, throttled or unreachable, or CloudWatch 5xxErrorRate above 5% over 30 minutes) or DOWN (missing or disabled distribution, rejected or missing credentials). Without `cloudfront:GetDistribution` it falls back to GetDistributionConfig and reports UP with `testable=false`. The NONE provider and adapters without a connection test report UP with `testable=false`. New optional SPI `CdnHealthProbe` / `CdnHealth`. - Derive the CDN qbit metadata version from the Maven artifact version, keeping release metadata and the help bundle in sync. - Tag rules accept ignored fields in `onlyWhenFieldsChange`, such as `!viewCount`, so counter-only updates do not purge rendered pages. Plain allow-lists still work, and ignored fields take precedence in mixed lists. No schema change is needed. - Purge actions warn when providers are NONE or hostnames are OFF, and report requests as recorded but not sent. The home explains recorded-only purges and that this version does not run origin header checks. diff --git a/src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java b/src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java index afea743..0538535 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java +++ b/src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java @@ -23,6 +23,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; @@ -30,9 +31,12 @@ import java.util.concurrent.TimeoutException; import com.kingsrook.qbits.cdn.CdnRuntime; import com.kingsrook.qbits.cdn.model.CdnTables; import com.kingsrook.qbits.cdn.purge.CdnPurgeDispatcher; +import com.kingsrook.qbits.cdn.spi.CdnHealth; +import com.kingsrook.qbits.cdn.spi.CdnHealthProbe; import com.kingsrook.qbits.cdn.spi.CdnOperations; import com.kingsrook.qbits.cdn.spi.CdnProviderConfig; import com.kingsrook.qbits.cdn.spi.CdnProviderInterface; +import com.kingsrook.qbits.cdn.spi.CdnProviderType; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -53,7 +57,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; /******************************************************************************* - ** Central health checks reuse the read-only provider connection test. + ** Central health checks: an adapter's own health probe when it has one + ** (CloudFront), else the read-only connection test, else UP/not testable. *******************************************************************************/ public class CdnProviderHealthChecks implements HealthCheckSupplier { @@ -95,38 +100,86 @@ public class CdnProviderHealthChecks implements HealthCheckSupplier } CdnProviderConfig config = CdnPurgeDispatcher.configOf(provider); CdnProviderInterface adapter = CdnRuntime.provider(config.type()); - if(!(adapter instanceof CdnOperations operations)) + if(adapter instanceof CdnHealthProbe healthProbe) { - return (HealthCheckOutcome.unknown("CDN provider does not support connection tests")); + return (bounded(() -> fromProbe(healthProbe.checkHealth(config), config), input)); } - return (probe(operations, config, input)); + if(adapter instanceof CdnOperations operations) + { + return (bounded(() -> fromConnectionTest(operations.connectionTest(config), config), input)); + } + return (notTestable(config)); } /******************************************************************************* - ** The existing SPI has no timeout parameter. Bound its wait by the remaining + ** An adapter with nothing to probe (the NONE placeholder, or one without a + ** connection test) is UP and says it is not testable. UNKNOWN would sit on + ** the dashboard forever: the central evaluator never alerts or changes + ** state on UNKNOWN. + *******************************************************************************/ + private static HealthCheckOutcome notTestable(CdnProviderConfig config) + { + boolean none = config.type() == CdnProviderType.NONE; + return (HealthCheckOutcome.up(none ? "No CDN configured; purges are recorded only" : "CDN provider configured; not testable (no connection test)") + .withDetails(new HealthDetails().withValue("providerType", config.type().name()).withValue("testable", false) + .withValue("reason", none ? "no CDN provider" : "provider type has no connection test"))); + } + + + + /******************************************************************************* + ** Map an adapter health probe. Its contract (CdnHealth) is fixed messages + ** and safe details only. + *******************************************************************************/ + private static HealthCheckOutcome fromProbe(CdnHealth health, CdnProviderConfig config) + { + HealthCheckOutcome outcome = switch(health.state()) + { + case UP -> HealthCheckOutcome.up(health.message()); + case DEGRADED -> HealthCheckOutcome.degraded(health.message()); + case DOWN -> HealthCheckOutcome.down(health.message()); + }; + HealthDetails details = new HealthDetails().withValue("providerType", config.type().name()); + health.details().forEach(details::withValue); + return (outcome.withDetails(details)); + } + + + + /******************************************************************************* + ** Map a provider-neutral connection test. + *******************************************************************************/ + private static HealthCheckOutcome fromConnectionTest(CdnOperations.ConnectionTest test, CdnProviderConfig config) + { + HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed") + : !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured") + : HealthCheckOutcome.up("CDN connection test passed"); + // Only known provider states are safe; arbitrary status text may contain secrets. + String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN"; + return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name()) + .withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured()))); + } + + + + /******************************************************************************* + ** The SPI has no timeout parameter. Bound its wait by the remaining ** deadline and interrupt the probe on timeout or runner cancellation. *******************************************************************************/ - private static HealthCheckOutcome probe(CdnOperations operations, CdnProviderConfig config, HealthCheckInput input) + private static HealthCheckOutcome bounded(Callable probe, HealthCheckInput input) { long remaining = Duration.between(Instant.now(), input.getDeadline()).toNanos(); if(remaining <= 0) { return (HealthCheckOutcome.down("CDN connection test timed out")); } - FutureTask task = new FutureTask<>(() -> operations.connectionTest(config)); + FutureTask task = new FutureTask<>(probe); Thread.startVirtualThread(task); try { - CdnOperations.ConnectionTest test = task.get(remaining, TimeUnit.NANOSECONDS); - HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed") - : !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured") - : HealthCheckOutcome.up("CDN connection test passed"); - // Only known provider states are safe; arbitrary status text may contain secrets. - String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN"; - return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name()) - .withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured()))); + return (task.get(remaining, TimeUnit.NANOSECONDS)); } catch(TimeoutException e) { diff --git a/src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java b/src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java index b92a3df..295254d 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java +++ b/src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java @@ -22,14 +22,18 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.OptionalDouble; import java.util.Set; import java.util.TreeMap; import com.kingsrook.qbits.cdn.spi.CdnCapability; import com.kingsrook.qbits.cdn.spi.CdnEdgeStat; import com.kingsrook.qbits.cdn.spi.CdnException; import com.kingsrook.qbits.cdn.spi.CdnHeaderDialect; +import com.kingsrook.qbits.cdn.spi.CdnHealth; +import com.kingsrook.qbits.cdn.spi.CdnHealthProbe; import com.kingsrook.qbits.cdn.spi.CdnProviderConfig; import com.kingsrook.qbits.cdn.spi.CdnProviderInterface; import com.kingsrook.qbits.cdn.spi.CdnProviderType; @@ -41,6 +45,8 @@ import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.cloudfront.CloudFrontClient; import software.amazon.awssdk.services.cloudfront.model.CreateInvalidationResponse; import software.amazon.awssdk.services.cloudfront.model.Distribution; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigRequest; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionRequest; import software.amazon.awssdk.services.cloudfront.model.GetInvalidationResponse; import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; import software.amazon.awssdk.services.cloudwatch.model.Dimension; @@ -70,7 +76,7 @@ import software.amazon.awssdk.services.cloudwatch.model.ScanBy; ** {@link #testConnection}, {@link #importStats} and {@link #readConfig} ** (compare the result with {@link CloudFrontDriftCheck#compare}).

*******************************************************************************/ -public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsrook.qbits.cdn.spi.CdnOperations +public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsrook.qbits.cdn.spi.CdnOperations, CdnHealthProbe { /** Default origin custom header that proves a request came through CloudFront. */ public static final String DEFAULT_ORIGIN_VERIFY_HEADER = "X-Origin-Verify"; @@ -81,6 +87,22 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo static final String STATUS_COMPLETED = "Completed"; static final int SECONDS_PER_DAY = 86_400; + /** Distribution states CloudFront reports (anything else is reported as UNKNOWN). */ + static final String STATUS_DEPLOYED = "Deployed"; + static final String STATUS_IN_PROGRESS = "InProgress"; + + /** 5xx error rate (percent, CloudWatch 5xxErrorRate) above which the health check is DEGRADED. */ + static final double ERROR_RATE_5XX_DEGRADED_PERCENT = 5.0; + + /** How far back the health check averages the 5xx error rate. */ + static final int ERROR_RATE_WINDOW_MINUTES = 30; + + /** CloudWatch period for the health check's 5xx error rate. */ + static final int ERROR_RATE_PERIOD_SECONDS = 300; + + /** Health message when the credentials cannot read the distribution status. */ + static final String NOT_TESTABLE_MESSAGE = "CloudFront configured; distribution status not testable with current permissions"; + private static final Set RETRYABLE_CODES = Set.of( "TooManyInvalidationsInProgress", "Throttling", "ThrottlingException", "RequestLimitExceeded", "ServiceUnavailable", "InternalError"); @@ -278,6 +300,181 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo + /******************************************************************************* + ** Central health probe. GetDistribution gives existence, Enabled and the + ** deployment Status: UP when enabled and Deployed, DEGRADED while + ** InProgress, DOWN when missing or disabled. The section 7.5 IAM policy + ** grants GetDistributionConfig but not GetDistribution, so on AccessDenied + ** this falls back to GetDistributionConfig (existence and Enabled, no + ** Status) and reports {@code testable=false}. If neither read is permitted + ** it reports UP with {@code testable=false}: UNKNOWN never alerts anyone, + ** and failed invalidations still show up as FAILED/DEAD purge requests. + ** The CloudWatch 5xxErrorRate over the last + ** {@value #ERROR_RATE_WINDOW_MINUTES} minutes is added when readable and + ** makes the check DEGRADED above {@value #ERROR_RATE_5XX_DEGRADED_PERCENT} + ** percent. Messages are fixed text; no AWS message, id or hostname is + ** copied. Never throws. + *******************************************************************************/ + @Override + public CdnHealth checkHealth(CdnProviderConfig config) + { + String headerName = config.settings().getOrDefault(SETTING_ORIGIN_VERIFY_HEADER, DEFAULT_ORIGIN_VERIFY_HEADER); + String distributionId; + try + { + distributionId = distributionId(config); + } + catch(CdnException e) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution id is not configured", null); + } + + Map details = new LinkedHashMap<>(); + CloudFrontConfigSnapshot snapshot; + String status = null; + try(CloudFrontClient client = clientFactory.cloudFront(config)) + { + try + { + Distribution distribution = client.getDistribution(GetDistributionRequest.builder().id(distributionId).build()).distribution(); + status = distribution.status(); + snapshot = CloudFrontConfigSnapshot.from(distribution.distributionConfig()); + } + catch(AwsServiceException e) + { + if(!isAccessDenied(e)) + { + return apiFailure(e); + } + details.put("testable", false); + try + { + snapshot = CloudFrontConfigSnapshot.from(client.getDistributionConfig(GetDistributionConfigRequest.builder().id(distributionId).build()).distributionConfig()); + details.put("reason", "credentials lack cloudfront:GetDistribution"); + } + catch(AwsServiceException e2) + { + if(!isAccessDenied(e2)) + { + return apiFailure(e2); + } + details.put("reason", "credentials lack cloudfront:GetDistribution and cloudfront:GetDistributionConfig"); + return new CdnHealth(CdnHealth.State.UP, NOT_TESTABLE_MESSAGE, details); + } + } + } + catch(CdnException e) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront credentials are not configured", null); + } + catch(SdkClientException e) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront API unreachable; distribution status not checked", Map.of("transient", true)); + } + catch(SdkException e) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront status check failed", null); + } + + boolean statusTestable = !details.containsKey("testable"); + boolean headerConfigured = snapshot.hasOriginHeader(headerName); + details.put("enabled", snapshot.enabled()); + details.put("status", STATUS_DEPLOYED.equals(status) || STATUS_IN_PROGRESS.equals(status) ? status : "UNKNOWN"); + details.put("originHeaderConfigured", headerConfigured); + if(!snapshot.enabled()) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution is disabled", details); + } + + Double errorRate = errorRate5xx(config, distributionId); + details.put("errorRateWindowMinutes", ERROR_RATE_WINDOW_MINUTES); + if(errorRate != null) + { + details.put("errorRate5xxPercent", Math.round(errorRate * 100.0) / 100.0); + } + + if(STATUS_IN_PROGRESS.equals(status)) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront distribution is deploying a change (InProgress)", details); + } + if(!headerConfigured) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CDN origin header is not configured", details); + } + if(errorRate != null && errorRate > ERROR_RATE_5XX_DEGRADED_PERCENT) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront 5xx error rate is above 5% over the last 30 minutes", details); + } + if(!statusTestable) + { + return new CdnHealth(CdnHealth.State.UP, NOT_TESTABLE_MESSAGE, details); + } + return new CdnHealth(CdnHealth.State.UP, "CloudFront distribution is enabled and deployed", details); + } + + + + /******************************************************************************* + ** Whether an AWS error is a permissions refusal. + *******************************************************************************/ + private static boolean isAccessDenied(AwsServiceException e) + { + String code = e.awsErrorDetails() == null ? null : e.awsErrorDetails().errorCode(); + return "AccessDenied".equals(code) || "AccessDeniedException".equals(code) || (code == null && e.statusCode() == 403); + } + + + + /******************************************************************************* + ** Health for an AWS error other than AccessDenied: a missing distribution is + ** DOWN, throttling and AWS-side errors are DEGRADED (transient), anything + ** else (rejected credentials, bad id) is DOWN. + *******************************************************************************/ + private static CdnHealth apiFailure(AwsServiceException e) + { + String code = e.awsErrorDetails() == null ? null : e.awsErrorDetails().errorCode(); + if("NoSuchDistribution".equals(code)) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution not found", null); + } + if(toCdnException("GetDistribution", e).isRetryable()) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront API throttled or unavailable; distribution status not checked", Map.of("transient", true)); + } + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront rejected the status check", null); + } + + + + /******************************************************************************* + ** Average CloudWatch 5xxErrorRate (percent) over the health window, or null + ** when there is no traffic or the metric cannot be read. Best effort: it + ** never fails the health check. + *******************************************************************************/ + private Double errorRate5xx(CdnProviderConfig config, String distributionId) + { + Instant end = Instant.now(); + try(CloudWatchClient client = clientFactory.cloudWatch(config)) + { + GetMetricDataResponse response = client.getMetricData(GetMetricDataRequest.builder() + .metricDataQueries(query("err5xx", "5xxErrorRate", "Average", distributionId, ERROR_RATE_PERIOD_SECONDS)) + .startTime(end.minusSeconds(ERROR_RATE_WINDOW_MINUTES * 60L)) + .endTime(end) + .build()); + OptionalDouble average = response.metricDataResults().stream() + .flatMap(result -> result.values().stream()) + .mapToDouble(Double::doubleValue) + .average(); + return average.isPresent() ? average.getAsDouble() : null; + } + catch(CdnException | SdkException e) + { + return null; + } + } + + + /******************************************************************************* ** The distribution config facts used for drift checks (GetDistributionConfig). *******************************************************************************/ @@ -368,6 +565,16 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo ** One daily CloudFront metric query. *******************************************************************************/ private static MetricDataQuery query(String id, String metricName, String stat, String distributionId) + { + return query(id, metricName, stat, distributionId, SECONDS_PER_DAY); + } + + + + /******************************************************************************* + ** One CloudFront metric query with the given period (seconds). + *******************************************************************************/ + private static MetricDataQuery query(String id, String metricName, String stat, String distributionId, int periodSeconds) { Metric metric = Metric.builder() .namespace("AWS/CloudFront") @@ -377,7 +584,7 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo .build(); return MetricDataQuery.builder() .id(id) - .metricStat(MetricStat.builder().metric(metric).period(SECONDS_PER_DAY).stat(stat).build()) + .metricStat(MetricStat.builder().metric(metric).period(periodSeconds).stat(stat).build()) .returnData(true) .build(); } diff --git a/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealth.java b/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealth.java new file mode 100644 index 0000000..4f3c825 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealth.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.spi; + + +import java.util.Map; + + +/******************************************************************************* + ** Result of an adapter's own health probe (see {@link CdnHealthProbe}). The + ** message and every detail value must be fixed text, numbers, booleans or + ** known provider states: never provider-supplied free text, hostnames, ids + ** or credentials, because they are copied to the central health tables. + ** + ** @param state UP, DEGRADED or DOWN + ** @param message a short fixed summary + ** @param details safe facts (may be empty, never null) + *******************************************************************************/ +public record CdnHealth(State state, String message, Map details) +{ + /******************************************************************************* + ** Health as the adapter sees it. + *******************************************************************************/ + public enum State + { + UP, + DEGRADED, + DOWN + } + + + + /******************************************************************************* + ** Null details become an empty map. + *******************************************************************************/ + public CdnHealth + { + details = details == null ? Map.of() : Map.copyOf(details); + } +} diff --git a/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealthProbe.java b/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealthProbe.java new file mode 100644 index 0000000..efcdfe3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealthProbe.java @@ -0,0 +1,32 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.spi; + + +/******************************************************************************* + ** Optional: an adapter that can say whether its CDN is healthy, beyond the + ** "Test connection" action ({@link CdnOperations}). The central health check + ** prefers this when an adapter implements it. + *******************************************************************************/ +public interface CdnHealthProbe +{ + /******************************************************************************* + ** Probe the CDN read-only. Never throws: failures come back as DOWN or + ** DEGRADED, with fixed messages (see {@link CdnHealth}). + *******************************************************************************/ + CdnHealth checkHealth(CdnProviderConfig config); +} diff --git a/src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java b/src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java index 003220f..35f6788 100644 --- a/src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java +++ b/src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java @@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit; import com.kingsrook.qbits.cdn.CdnRuntime; import com.kingsrook.qbits.cdn.CdnTestSupport; import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qbits.cdn.spi.CdnHealth; +import com.kingsrook.qbits.cdn.spi.CdnHealthProbe; import com.kingsrook.qbits.cdn.spi.CdnOperations; import com.kingsrook.qbits.cdn.spi.CdnProviderConfig; import com.kingsrook.qbits.cdn.spi.CdnProviderType; @@ -168,7 +170,14 @@ class CdnProviderHealthChecksTest { var check = check().getCheckInstance(); update(new QRecord().withValue("providerType", "NONE")); - assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + HealthCheckOutcome none = check.run(input()); + assertThat(none.getStatus()).isEqualTo(HealthStatus.UP); + assertThat(none.getMessage()).isEqualTo("No CDN configured; purges are recorded only"); + assertThat(none.getDetails().getValues()).containsEntry("testable", false).containsEntry("providerType", "NONE"); + update(new QRecord().withValue("providerType", "SOUIN")); + HealthCheckOutcome souin = check.run(input()); + assertThat(souin.getStatus()).isEqualTo(HealthStatus.UP); + assertThat(souin.getDetails().getValues()).containsEntry("testable", false).containsEntry("reason", "provider type has no connection test"); update(new QRecord().withValue("enabled", false)); assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); new DeleteAction().execute(new DeleteInput(CdnTables.PROVIDER).withPrimaryKeys(List.of(providerId))); @@ -178,6 +187,63 @@ class CdnProviderHealthChecksTest + /******************************************************************************* + ** An adapter's own health probe wins over the connection test; its state, + ** message and details pass through with the provider type added. + *******************************************************************************/ + @Test + void healthProbePreferred() throws Exception + { + ProbingProvider probing = new ProbingProvider(); + CdnRuntime.registerProvider(CdnProviderType.CLOUDFRONT, () -> probing); + var check = check().getCheckInstance(); + for(CdnHealth.State state : CdnHealth.State.values()) + { + probing.health = new CdnHealth(state, "probe " + state, Map.of("testable", false)); + HealthCheckOutcome outcome = check.run(input()); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.valueOf(state.name())); + assertThat(outcome.getMessage()).isEqualTo("probe " + state); + assertThat(outcome.getDetails().getValues()).containsExactlyInAnyOrderEntriesOf(Map.of("providerType", "CLOUDFRONT", "testable", false)); + } + assertThat(probing.connectionTests).isZero(); + } + + + + /******************************************************************************* + ** The fake path adapter plus a health probe. + *******************************************************************************/ + private static class ProbingProvider extends CdnTestSupport.FakePathProvider implements CdnHealthProbe + { + private CdnHealth health; + private int connectionTests; + + + + /******************************************************************************* + ** The configured answer. + *******************************************************************************/ + @Override + public CdnHealth checkHealth(CdnProviderConfig config) + { + return (health); + } + + + + /******************************************************************************* + ** Counted so the test can prove it is not used. + *******************************************************************************/ + @Override + public ConnectionTest connectionTest(CdnProviderConfig config) + { + connectionTests++; + return (super.connectionTest(config)); + } + } + + + /******************************************************************************* ** Poison credentials, every free-form probe field and thrown error messages. ** HealthRunner.sanitize is package-private, so assert the raw boundary here. diff --git a/src/test/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontHealthTest.java b/src/test/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontHealthTest.java new file mode 100644 index 0000000..fff6056 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontHealthTest.java @@ -0,0 +1,394 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.providers.cloudfront; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.cdn.spi.CdnHealth; +import com.kingsrook.qbits.cdn.spi.CdnProviderConfig; +import com.kingsrook.qbits.cdn.spi.CdnProviderType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.cloudfront.CloudFrontClient; +import software.amazon.awssdk.services.cloudfront.model.CloudFrontException; +import software.amazon.awssdk.services.cloudfront.model.CustomHeaders; +import software.amazon.awssdk.services.cloudfront.model.DefaultCacheBehavior; +import software.amazon.awssdk.services.cloudfront.model.Distribution; +import software.amazon.awssdk.services.cloudfront.model.DistributionConfig; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigRequest; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigResponse; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionRequest; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionResponse; +import software.amazon.awssdk.services.cloudfront.model.Origin; +import software.amazon.awssdk.services.cloudfront.model.OriginCustomHeader; +import software.amazon.awssdk.services.cloudfront.model.Origins; +import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; +import software.amazon.awssdk.services.cloudwatch.model.CloudWatchException; +import software.amazon.awssdk.services.cloudwatch.model.Dimension; +import software.amazon.awssdk.services.cloudwatch.model.GetMetricDataRequest; +import software.amazon.awssdk.services.cloudwatch.model.GetMetricDataResponse; +import software.amazon.awssdk.services.cloudwatch.model.MetricDataResult; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +/******************************************************************************* + ** The CloudFront health probe against mocked CloudFront and CloudWatch + ** clients (no network). + *******************************************************************************/ +class CloudFrontHealthTest +{ + private static final String DISTRIBUTION = "E2EXAMPLE123"; + private static final String SECRET = "FIXTURE-SECRET-NEVER-LEAK"; + private static final CdnProviderConfig CONFIG = new CdnProviderConfig("kof22-cloudfront", CdnProviderType.CLOUDFRONT, DISTRIBUTION, "CDN_AWS", null); + + private CloudFrontClient cloudFront; + private CloudWatchClient cloudWatch; + + + + /******************************************************************************* + ** Fresh mocks; by default CloudWatch has no datapoints. + *******************************************************************************/ + @BeforeEach + void mocks() + { + cloudFront = mock(CloudFrontClient.class); + cloudWatch = mock(CloudWatchClient.class); + errorRate(); + } + + + + /******************************************************************************* + ** An adapter whose clients are the mocks. + *******************************************************************************/ + private CloudFrontCdnProvider provider() + { + return new CloudFrontCdnProvider(new CloudFrontClientFactory(name -> null, null, null) + { + /*************************************************************************** + ** The CloudFront mock. + ***************************************************************************/ + @Override + public CloudFrontClient cloudFront(CdnProviderConfig config) + { + return cloudFront; + } + + + + /*************************************************************************** + ** The CloudWatch mock. + ***************************************************************************/ + @Override + public CloudWatchClient cloudWatch(CdnProviderConfig config) + { + return cloudWatch; + } + }); + } + + + + /******************************************************************************* + ** A distribution config, enabled or not, whose origin sends the verify header. + *******************************************************************************/ + private static DistributionConfig distributionConfig(boolean enabled) + { + Origin origin = Origin.builder().id("origin").domainName("origin.kof22.com") + .customHeaders(CustomHeaders.builder().quantity(1).items(OriginCustomHeader.builder().headerName("X-Origin-Verify").headerValue(SECRET).build()).build()) + .build(); + return DistributionConfig.builder().enabled(enabled) + .origins(Origins.builder().quantity(1).items(origin).build()) + .defaultCacheBehavior(DefaultCacheBehavior.builder().targetOriginId("origin").build()) + .build(); + } + + + + /******************************************************************************* + ** GetDistribution answers with this status and enabled flag. + *******************************************************************************/ + private void distribution(String status, boolean enabled) + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenReturn(GetDistributionResponse.builder() + .distribution(Distribution.builder().id(DISTRIBUTION).status(status).domainName("d1.cloudfront.net").distributionConfig(distributionConfig(enabled)).build()) + .build()); + } + + + + /******************************************************************************* + ** CloudWatch answers with these 5xxErrorRate datapoints. + *******************************************************************************/ + private void errorRate(Double... values) + { + when(cloudWatch.getMetricData(any(GetMetricDataRequest.class))).thenReturn(GetMetricDataResponse.builder() + .metricDataResults(MetricDataResult.builder().id("err5xx").values(values).build()).build()); + } + + + + /******************************************************************************* + ** A CloudFront service error. + *******************************************************************************/ + private static AwsServiceException error(String code, int status) + { + return CloudFrontException.builder().statusCode(status) + .awsErrorDetails(AwsErrorDetails.builder().errorCode(code).errorMessage(SECRET).build()).build(); + } + + + + /******************************************************************************* + ** Enabled and Deployed is UP; the 5xx rate is averaged into the details from + ** a five-minute Average query on the distribution's global metric. + *******************************************************************************/ + @Test + void deployedIsUp() + { + distribution("Deployed", true); + errorRate(1.0, 2.0); + + CdnHealth health = provider().checkHealth(CONFIG); + + assertThat(health.state()).isEqualTo(CdnHealth.State.UP); + assertThat(health.message()).isEqualTo("CloudFront distribution is enabled and deployed"); + assertThat(health.details()).containsExactlyInAnyOrderEntriesOf(Map.of("enabled", true, "status", "Deployed", "originHeaderConfigured", true, + "errorRateWindowMinutes", 30, "errorRate5xxPercent", 1.5)); + ArgumentCaptor request = ArgumentCaptor.forClass(GetMetricDataRequest.class); + verify(cloudWatch).getMetricData(request.capture()); + var stat = request.getValue().metricDataQueries().getFirst().metricStat(); + assertThat(stat.metric().metricName()).isEqualTo("5xxErrorRate"); + assertThat(stat.metric().namespace()).isEqualTo("AWS/CloudFront"); + assertThat(stat.metric().dimensions()).containsExactly(Dimension.builder().name("DistributionId").value(DISTRIBUTION).build(), + Dimension.builder().name("Region").value("Global").build()); + assertThat(stat.stat()).isEqualTo("Average"); + assertThat(stat.period()).isEqualTo(300); + assertThat(java.time.Duration.between(request.getValue().startTime(), request.getValue().endTime()).toMinutes()).isEqualTo(30); + verify(cloudFront, never()).getDistributionConfig(any(GetDistributionConfigRequest.class)); + assertClean(health); + } + + + + /******************************************************************************* + ** No traffic (no datapoints) or an unreadable metric leaves the rate out. + *******************************************************************************/ + @Test + void errorRateIsBestEffort() + { + distribution("Deployed", true); + assertThat(provider().checkHealth(CONFIG).details()).doesNotContainKey("errorRate5xxPercent").containsEntry("errorRateWindowMinutes", 30); + + when(cloudWatch.getMetricData(any(GetMetricDataRequest.class))).thenThrow(CloudWatchException.builder().statusCode(403) + .awsErrorDetails(AwsErrorDetails.builder().errorCode("AccessDenied").build()).build()); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.UP); + assertThat(health.details()).doesNotContainKey("errorRate5xxPercent"); + } + + + + /******************************************************************************* + ** A 5xx rate above 5% is DEGRADED; at 5% it is still UP. + *******************************************************************************/ + @Test + void highErrorRateIsDegraded() + { + distribution("Deployed", true); + errorRate(5.0); + assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(CdnHealth.State.UP); + + errorRate(4.0, 9.0); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED); + assertThat(health.message()).contains("5xx error rate"); + assertThat(health.details()).containsEntry("errorRate5xxPercent", 6.5); + } + + + + /******************************************************************************* + ** A deploying distribution is DEGRADED and says so. + *******************************************************************************/ + @Test + void inProgressIsDegraded() + { + distribution("InProgress", true); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED); + assertThat(health.message()).contains("deploying").contains("InProgress"); + assertThat(health.details()).containsEntry("status", "InProgress"); + } + + + + /******************************************************************************* + ** A disabled distribution is DOWN (and CloudWatch is not consulted); an + ** unexpected status value is reported as UNKNOWN, never echoed. + *******************************************************************************/ + @Test + void disabledIsDown() + { + distribution(SECRET, false); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.DOWN); + assertThat(health.message()).isEqualTo("CloudFront distribution is disabled"); + assertThat(health.details()).containsEntry("enabled", false).containsEntry("status", "UNKNOWN"); + verify(cloudWatch, never()).getMetricData(any(GetMetricDataRequest.class)); + assertClean(health); + } + + + + /******************************************************************************* + ** A different configured header name is reported missing: DEGRADED. + *******************************************************************************/ + @Test + void missingOriginHeaderIsDegraded() + { + distribution("Deployed", true); + CdnProviderConfig other = new CdnProviderConfig("x", CdnProviderType.CLOUDFRONT, DISTRIBUTION, "CDN_AWS", + Map.of(CloudFrontCdnProvider.SETTING_ORIGIN_VERIFY_HEADER, "X-Other-Verify")); + CdnHealth health = provider().checkHealth(other); + assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED); + assertThat(health.details()).containsEntry("originHeaderConfigured", false); + } + + + + /******************************************************************************* + ** Without GetDistribution (the section 7.5 policy) the probe falls back to + ** GetDistributionConfig: UP, not testable, Enabled still checked. + *******************************************************************************/ + @Test + void accessDeniedFallsBackToConfig() + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403)); + when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))) + .thenReturn(GetDistributionConfigResponse.builder().distributionConfig(distributionConfig(true)).build()); + + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.UP); + assertThat(health.message()).isEqualTo("CloudFront configured; distribution status not testable with current permissions"); + assertThat(health.details()).containsEntry("testable", false).containsEntry("reason", "credentials lack cloudfront:GetDistribution") + .containsEntry("enabled", true).containsEntry("status", "UNKNOWN"); + assertClean(health); + + when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))) + .thenReturn(GetDistributionConfigResponse.builder().distributionConfig(distributionConfig(false)).build()); + assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(CdnHealth.State.DOWN); + } + + + + /******************************************************************************* + ** When neither read is permitted the check is UP and not testable. + *******************************************************************************/ + @Test + void noReadPermissionIsUpNotTestable() + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403)); + when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))).thenThrow(error(null, 403)); + + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.UP); + assertThat(health.details()).containsExactlyInAnyOrderEntriesOf(Map.of("testable", false, + "reason", "credentials lack cloudfront:GetDistribution and cloudfront:GetDistributionConfig")); + verify(cloudWatch, never()).getMetricData(any(GetMetricDataRequest.class)); + } + + + + /******************************************************************************* + ** AWS errors: missing distribution and rejected credentials are DOWN, + ** throttling and AWS-side errors DEGRADED; also after the config fallback. + *******************************************************************************/ + @ParameterizedTest + @CsvSource({ + "NoSuchDistribution, 404, DOWN, CloudFront distribution not found", + "InvalidClientTokenId, 403, DOWN, CloudFront rejected the status check", + "Throttling, 400, DEGRADED, CloudFront API throttled or unavailable; distribution status not checked", + "ServiceUnavailable, 503, DEGRADED, CloudFront API throttled or unavailable; distribution status not checked", + }) + void apiErrors(String code, int status, CdnHealth.State expected, String message) + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error(code, status)); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(expected); + assertThat(health.message()).isEqualTo(message); + assertClean(health); + + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403)); + when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))).thenThrow(error(code, status)); + assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(expected); + } + + + + /******************************************************************************* + ** Network failures are DEGRADED (transient); missing credentials or + ** distribution id are DOWN; other SDK failures are DOWN. + *******************************************************************************/ + @Test + void clientAndConfigFailures() + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(SdkClientException.create(SECRET)); + CdnHealth unreachable = provider().checkHealth(CONFIG); + assertThat(unreachable.state()).isEqualTo(CdnHealth.State.DEGRADED); + assertThat(unreachable.details()).containsEntry("transient", true); + assertClean(unreachable); + + CdnHealth noCredentials = new CloudFrontCdnProvider(new CloudFrontClientFactory(name -> null, null, null)).checkHealth(CONFIG); + assertThat(noCredentials.state()).isEqualTo(CdnHealth.State.DOWN); + assertThat(noCredentials.message()).isEqualTo("CloudFront credentials are not configured"); + + CdnHealth noId = provider().checkHealth(new CdnProviderConfig("x", CdnProviderType.CLOUDFRONT, " ", "CDN_AWS", null)); + assertThat(noId.state()).isEqualTo(CdnHealth.State.DOWN); + assertThat(noId.message()).isEqualTo("CloudFront distribution id is not configured"); + + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(SdkException.builder().message(SECRET).build()); + CdnHealth other = provider().checkHealth(CONFIG); + assertThat(other.state()).isEqualTo(CdnHealth.State.DOWN); + assertClean(other); + } + + + + /******************************************************************************* + ** Nothing the AWS side controls as free text reaches the outcome. + *******************************************************************************/ + private static void assertClean(CdnHealth health) + { + assertThat(health.message()).doesNotContain(SECRET, DISTRIBUTION, "cloudfront.net", "origin.kof22.com"); + assertThat(List.copyOf(health.details().values()).toString()).doesNotContain(SECRET, DISTRIBUTION, "cloudfront.net", "origin.kof22.com"); + } +} -- 2.54.0