diff --git a/CHANGELOG.md b/CHANGELOG.md index 080b061..3f34cc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ All notable changes to this project are documented here. The format follows Keep ### Fixed +- CDN health checks no longer sit at UNKNOWN (which the central evaluator never alerts on). CloudFront checks the distribution itself: GetDistribution gives UP (enabled and Deployed), DEGRADED (InProgress, origin header missing, throttled or unreachable, or CloudWatch 5xxErrorRate above 5% over 30 minutes) or DOWN (missing or disabled distribution, rejected or missing credentials). Without `cloudfront:GetDistribution` it falls back to GetDistributionConfig and reports UP with `testable=false`. The NONE provider and adapters without a connection test report UP with `testable=false`. New optional SPI `CdnHealthProbe` / `CdnHealth`. - Derive the CDN qbit metadata version from the Maven artifact version, keeping release metadata and the help bundle in sync. - Tag rules accept ignored fields in `onlyWhenFieldsChange`, such as `!viewCount`, so counter-only updates do not purge rendered pages. Plain allow-lists still work, and ignored fields take precedence in mixed lists. No schema change is needed. - Purge actions warn when providers are NONE or hostnames are OFF, and report requests as recorded but not sent. The home explains recorded-only purges and that this version does not run origin header checks. diff --git a/src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java b/src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java index afea743..0538535 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java +++ b/src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java @@ -23,6 +23,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; @@ -30,9 +31,12 @@ import java.util.concurrent.TimeoutException; import com.kingsrook.qbits.cdn.CdnRuntime; import com.kingsrook.qbits.cdn.model.CdnTables; import com.kingsrook.qbits.cdn.purge.CdnPurgeDispatcher; +import com.kingsrook.qbits.cdn.spi.CdnHealth; +import com.kingsrook.qbits.cdn.spi.CdnHealthProbe; import com.kingsrook.qbits.cdn.spi.CdnOperations; import com.kingsrook.qbits.cdn.spi.CdnProviderConfig; import com.kingsrook.qbits.cdn.spi.CdnProviderInterface; +import com.kingsrook.qbits.cdn.spi.CdnProviderType; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -53,7 +57,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance; /******************************************************************************* - ** Central health checks reuse the read-only provider connection test. + ** Central health checks: an adapter's own health probe when it has one + ** (CloudFront), else the read-only connection test, else UP/not testable. *******************************************************************************/ public class CdnProviderHealthChecks implements HealthCheckSupplier { @@ -95,38 +100,86 @@ public class CdnProviderHealthChecks implements HealthCheckSupplier } CdnProviderConfig config = CdnPurgeDispatcher.configOf(provider); CdnProviderInterface adapter = CdnRuntime.provider(config.type()); - if(!(adapter instanceof CdnOperations operations)) + if(adapter instanceof CdnHealthProbe healthProbe) { - return (HealthCheckOutcome.unknown("CDN provider does not support connection tests")); + return (bounded(() -> fromProbe(healthProbe.checkHealth(config), config), input)); } - return (probe(operations, config, input)); + if(adapter instanceof CdnOperations operations) + { + return (bounded(() -> fromConnectionTest(operations.connectionTest(config), config), input)); + } + return (notTestable(config)); } /******************************************************************************* - ** The existing SPI has no timeout parameter. Bound its wait by the remaining + ** An adapter with nothing to probe (the NONE placeholder, or one without a + ** connection test) is UP and says it is not testable. UNKNOWN would sit on + ** the dashboard forever: the central evaluator never alerts or changes + ** state on UNKNOWN. + *******************************************************************************/ + private static HealthCheckOutcome notTestable(CdnProviderConfig config) + { + boolean none = config.type() == CdnProviderType.NONE; + return (HealthCheckOutcome.up(none ? "No CDN configured; purges are recorded only" : "CDN provider configured; not testable (no connection test)") + .withDetails(new HealthDetails().withValue("providerType", config.type().name()).withValue("testable", false) + .withValue("reason", none ? "no CDN provider" : "provider type has no connection test"))); + } + + + + /******************************************************************************* + ** Map an adapter health probe. Its contract (CdnHealth) is fixed messages + ** and safe details only. + *******************************************************************************/ + private static HealthCheckOutcome fromProbe(CdnHealth health, CdnProviderConfig config) + { + HealthCheckOutcome outcome = switch(health.state()) + { + case UP -> HealthCheckOutcome.up(health.message()); + case DEGRADED -> HealthCheckOutcome.degraded(health.message()); + case DOWN -> HealthCheckOutcome.down(health.message()); + }; + HealthDetails details = new HealthDetails().withValue("providerType", config.type().name()); + health.details().forEach(details::withValue); + return (outcome.withDetails(details)); + } + + + + /******************************************************************************* + ** Map a provider-neutral connection test. + *******************************************************************************/ + private static HealthCheckOutcome fromConnectionTest(CdnOperations.ConnectionTest test, CdnProviderConfig config) + { + HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed") + : !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured") + : HealthCheckOutcome.up("CDN connection test passed"); + // Only known provider states are safe; arbitrary status text may contain secrets. + String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN"; + return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name()) + .withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured()))); + } + + + + /******************************************************************************* + ** The SPI has no timeout parameter. Bound its wait by the remaining ** deadline and interrupt the probe on timeout or runner cancellation. *******************************************************************************/ - private static HealthCheckOutcome probe(CdnOperations operations, CdnProviderConfig config, HealthCheckInput input) + private static HealthCheckOutcome bounded(Callable probe, HealthCheckInput input) { long remaining = Duration.between(Instant.now(), input.getDeadline()).toNanos(); if(remaining <= 0) { return (HealthCheckOutcome.down("CDN connection test timed out")); } - FutureTask task = new FutureTask<>(() -> operations.connectionTest(config)); + FutureTask task = new FutureTask<>(probe); Thread.startVirtualThread(task); try { - CdnOperations.ConnectionTest test = task.get(remaining, TimeUnit.NANOSECONDS); - HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed") - : !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured") - : HealthCheckOutcome.up("CDN connection test passed"); - // Only known provider states are safe; arbitrary status text may contain secrets. - String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN"; - return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name()) - .withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured()))); + return (task.get(remaining, TimeUnit.NANOSECONDS)); } catch(TimeoutException e) { diff --git a/src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java b/src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java index b92a3df..295254d 100644 --- a/src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java +++ b/src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java @@ -22,14 +22,18 @@ import java.time.LocalDate; import java.time.ZoneOffset; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.OptionalDouble; import java.util.Set; import java.util.TreeMap; import com.kingsrook.qbits.cdn.spi.CdnCapability; import com.kingsrook.qbits.cdn.spi.CdnEdgeStat; import com.kingsrook.qbits.cdn.spi.CdnException; import com.kingsrook.qbits.cdn.spi.CdnHeaderDialect; +import com.kingsrook.qbits.cdn.spi.CdnHealth; +import com.kingsrook.qbits.cdn.spi.CdnHealthProbe; import com.kingsrook.qbits.cdn.spi.CdnProviderConfig; import com.kingsrook.qbits.cdn.spi.CdnProviderInterface; import com.kingsrook.qbits.cdn.spi.CdnProviderType; @@ -41,6 +45,8 @@ import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.cloudfront.CloudFrontClient; import software.amazon.awssdk.services.cloudfront.model.CreateInvalidationResponse; import software.amazon.awssdk.services.cloudfront.model.Distribution; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigRequest; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionRequest; import software.amazon.awssdk.services.cloudfront.model.GetInvalidationResponse; import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; import software.amazon.awssdk.services.cloudwatch.model.Dimension; @@ -70,7 +76,7 @@ import software.amazon.awssdk.services.cloudwatch.model.ScanBy; ** {@link #testConnection}, {@link #importStats} and {@link #readConfig} ** (compare the result with {@link CloudFrontDriftCheck#compare}).

*******************************************************************************/ -public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsrook.qbits.cdn.spi.CdnOperations +public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsrook.qbits.cdn.spi.CdnOperations, CdnHealthProbe { /** Default origin custom header that proves a request came through CloudFront. */ public static final String DEFAULT_ORIGIN_VERIFY_HEADER = "X-Origin-Verify"; @@ -81,6 +87,22 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo static final String STATUS_COMPLETED = "Completed"; static final int SECONDS_PER_DAY = 86_400; + /** Distribution states CloudFront reports (anything else is reported as UNKNOWN). */ + static final String STATUS_DEPLOYED = "Deployed"; + static final String STATUS_IN_PROGRESS = "InProgress"; + + /** 5xx error rate (percent, CloudWatch 5xxErrorRate) above which the health check is DEGRADED. */ + static final double ERROR_RATE_5XX_DEGRADED_PERCENT = 5.0; + + /** How far back the health check averages the 5xx error rate. */ + static final int ERROR_RATE_WINDOW_MINUTES = 30; + + /** CloudWatch period for the health check's 5xx error rate. */ + static final int ERROR_RATE_PERIOD_SECONDS = 300; + + /** Health message when the credentials cannot read the distribution status. */ + static final String NOT_TESTABLE_MESSAGE = "CloudFront configured; distribution status not testable with current permissions"; + private static final Set RETRYABLE_CODES = Set.of( "TooManyInvalidationsInProgress", "Throttling", "ThrottlingException", "RequestLimitExceeded", "ServiceUnavailable", "InternalError"); @@ -278,6 +300,181 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo + /******************************************************************************* + ** Central health probe. GetDistribution gives existence, Enabled and the + ** deployment Status: UP when enabled and Deployed, DEGRADED while + ** InProgress, DOWN when missing or disabled. The section 7.5 IAM policy + ** grants GetDistributionConfig but not GetDistribution, so on AccessDenied + ** this falls back to GetDistributionConfig (existence and Enabled, no + ** Status) and reports {@code testable=false}. If neither read is permitted + ** it reports UP with {@code testable=false}: UNKNOWN never alerts anyone, + ** and failed invalidations still show up as FAILED/DEAD purge requests. + ** The CloudWatch 5xxErrorRate over the last + ** {@value #ERROR_RATE_WINDOW_MINUTES} minutes is added when readable and + ** makes the check DEGRADED above {@value #ERROR_RATE_5XX_DEGRADED_PERCENT} + ** percent. Messages are fixed text; no AWS message, id or hostname is + ** copied. Never throws. + *******************************************************************************/ + @Override + public CdnHealth checkHealth(CdnProviderConfig config) + { + String headerName = config.settings().getOrDefault(SETTING_ORIGIN_VERIFY_HEADER, DEFAULT_ORIGIN_VERIFY_HEADER); + String distributionId; + try + { + distributionId = distributionId(config); + } + catch(CdnException e) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution id is not configured", null); + } + + Map details = new LinkedHashMap<>(); + CloudFrontConfigSnapshot snapshot; + String status = null; + try(CloudFrontClient client = clientFactory.cloudFront(config)) + { + try + { + Distribution distribution = client.getDistribution(GetDistributionRequest.builder().id(distributionId).build()).distribution(); + status = distribution.status(); + snapshot = CloudFrontConfigSnapshot.from(distribution.distributionConfig()); + } + catch(AwsServiceException e) + { + if(!isAccessDenied(e)) + { + return apiFailure(e); + } + details.put("testable", false); + try + { + snapshot = CloudFrontConfigSnapshot.from(client.getDistributionConfig(GetDistributionConfigRequest.builder().id(distributionId).build()).distributionConfig()); + details.put("reason", "credentials lack cloudfront:GetDistribution"); + } + catch(AwsServiceException e2) + { + if(!isAccessDenied(e2)) + { + return apiFailure(e2); + } + details.put("reason", "credentials lack cloudfront:GetDistribution and cloudfront:GetDistributionConfig"); + return new CdnHealth(CdnHealth.State.UP, NOT_TESTABLE_MESSAGE, details); + } + } + } + catch(CdnException e) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront credentials are not configured", null); + } + catch(SdkClientException e) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront API unreachable; distribution status not checked", Map.of("transient", true)); + } + catch(SdkException e) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront status check failed", null); + } + + boolean statusTestable = !details.containsKey("testable"); + boolean headerConfigured = snapshot.hasOriginHeader(headerName); + details.put("enabled", snapshot.enabled()); + details.put("status", STATUS_DEPLOYED.equals(status) || STATUS_IN_PROGRESS.equals(status) ? status : "UNKNOWN"); + details.put("originHeaderConfigured", headerConfigured); + if(!snapshot.enabled()) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution is disabled", details); + } + + Double errorRate = errorRate5xx(config, distributionId); + details.put("errorRateWindowMinutes", ERROR_RATE_WINDOW_MINUTES); + if(errorRate != null) + { + details.put("errorRate5xxPercent", Math.round(errorRate * 100.0) / 100.0); + } + + if(STATUS_IN_PROGRESS.equals(status)) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront distribution is deploying a change (InProgress)", details); + } + if(!headerConfigured) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CDN origin header is not configured", details); + } + if(errorRate != null && errorRate > ERROR_RATE_5XX_DEGRADED_PERCENT) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront 5xx error rate is above 5% over the last 30 minutes", details); + } + if(!statusTestable) + { + return new CdnHealth(CdnHealth.State.UP, NOT_TESTABLE_MESSAGE, details); + } + return new CdnHealth(CdnHealth.State.UP, "CloudFront distribution is enabled and deployed", details); + } + + + + /******************************************************************************* + ** Whether an AWS error is a permissions refusal. + *******************************************************************************/ + private static boolean isAccessDenied(AwsServiceException e) + { + String code = e.awsErrorDetails() == null ? null : e.awsErrorDetails().errorCode(); + return "AccessDenied".equals(code) || "AccessDeniedException".equals(code) || (code == null && e.statusCode() == 403); + } + + + + /******************************************************************************* + ** Health for an AWS error other than AccessDenied: a missing distribution is + ** DOWN, throttling and AWS-side errors are DEGRADED (transient), anything + ** else (rejected credentials, bad id) is DOWN. + *******************************************************************************/ + private static CdnHealth apiFailure(AwsServiceException e) + { + String code = e.awsErrorDetails() == null ? null : e.awsErrorDetails().errorCode(); + if("NoSuchDistribution".equals(code)) + { + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution not found", null); + } + if(toCdnException("GetDistribution", e).isRetryable()) + { + return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront API throttled or unavailable; distribution status not checked", Map.of("transient", true)); + } + return new CdnHealth(CdnHealth.State.DOWN, "CloudFront rejected the status check", null); + } + + + + /******************************************************************************* + ** Average CloudWatch 5xxErrorRate (percent) over the health window, or null + ** when there is no traffic or the metric cannot be read. Best effort: it + ** never fails the health check. + *******************************************************************************/ + private Double errorRate5xx(CdnProviderConfig config, String distributionId) + { + Instant end = Instant.now(); + try(CloudWatchClient client = clientFactory.cloudWatch(config)) + { + GetMetricDataResponse response = client.getMetricData(GetMetricDataRequest.builder() + .metricDataQueries(query("err5xx", "5xxErrorRate", "Average", distributionId, ERROR_RATE_PERIOD_SECONDS)) + .startTime(end.minusSeconds(ERROR_RATE_WINDOW_MINUTES * 60L)) + .endTime(end) + .build()); + OptionalDouble average = response.metricDataResults().stream() + .flatMap(result -> result.values().stream()) + .mapToDouble(Double::doubleValue) + .average(); + return average.isPresent() ? average.getAsDouble() : null; + } + catch(CdnException | SdkException e) + { + return null; + } + } + + + /******************************************************************************* ** The distribution config facts used for drift checks (GetDistributionConfig). *******************************************************************************/ @@ -368,6 +565,16 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo ** One daily CloudFront metric query. *******************************************************************************/ private static MetricDataQuery query(String id, String metricName, String stat, String distributionId) + { + return query(id, metricName, stat, distributionId, SECONDS_PER_DAY); + } + + + + /******************************************************************************* + ** One CloudFront metric query with the given period (seconds). + *******************************************************************************/ + private static MetricDataQuery query(String id, String metricName, String stat, String distributionId, int periodSeconds) { Metric metric = Metric.builder() .namespace("AWS/CloudFront") @@ -377,7 +584,7 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo .build(); return MetricDataQuery.builder() .id(id) - .metricStat(MetricStat.builder().metric(metric).period(SECONDS_PER_DAY).stat(stat).build()) + .metricStat(MetricStat.builder().metric(metric).period(periodSeconds).stat(stat).build()) .returnData(true) .build(); } diff --git a/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealth.java b/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealth.java new file mode 100644 index 0000000..4f3c825 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealth.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.spi; + + +import java.util.Map; + + +/******************************************************************************* + ** Result of an adapter's own health probe (see {@link CdnHealthProbe}). The + ** message and every detail value must be fixed text, numbers, booleans or + ** known provider states: never provider-supplied free text, hostnames, ids + ** or credentials, because they are copied to the central health tables. + ** + ** @param state UP, DEGRADED or DOWN + ** @param message a short fixed summary + ** @param details safe facts (may be empty, never null) + *******************************************************************************/ +public record CdnHealth(State state, String message, Map details) +{ + /******************************************************************************* + ** Health as the adapter sees it. + *******************************************************************************/ + public enum State + { + UP, + DEGRADED, + DOWN + } + + + + /******************************************************************************* + ** Null details become an empty map. + *******************************************************************************/ + public CdnHealth + { + details = details == null ? Map.of() : Map.copyOf(details); + } +} diff --git a/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealthProbe.java b/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealthProbe.java new file mode 100644 index 0000000..efcdfe3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealthProbe.java @@ -0,0 +1,32 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.spi; + + +/******************************************************************************* + ** Optional: an adapter that can say whether its CDN is healthy, beyond the + ** "Test connection" action ({@link CdnOperations}). The central health check + ** prefers this when an adapter implements it. + *******************************************************************************/ +public interface CdnHealthProbe +{ + /******************************************************************************* + ** Probe the CDN read-only. Never throws: failures come back as DOWN or + ** DEGRADED, with fixed messages (see {@link CdnHealth}). + *******************************************************************************/ + CdnHealth checkHealth(CdnProviderConfig config); +} diff --git a/src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java b/src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java index 003220f..35f6788 100644 --- a/src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java +++ b/src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java @@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit; import com.kingsrook.qbits.cdn.CdnRuntime; import com.kingsrook.qbits.cdn.CdnTestSupport; import com.kingsrook.qbits.cdn.model.CdnTables; +import com.kingsrook.qbits.cdn.spi.CdnHealth; +import com.kingsrook.qbits.cdn.spi.CdnHealthProbe; import com.kingsrook.qbits.cdn.spi.CdnOperations; import com.kingsrook.qbits.cdn.spi.CdnProviderConfig; import com.kingsrook.qbits.cdn.spi.CdnProviderType; @@ -168,7 +170,14 @@ class CdnProviderHealthChecksTest { var check = check().getCheckInstance(); update(new QRecord().withValue("providerType", "NONE")); - assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + HealthCheckOutcome none = check.run(input()); + assertThat(none.getStatus()).isEqualTo(HealthStatus.UP); + assertThat(none.getMessage()).isEqualTo("No CDN configured; purges are recorded only"); + assertThat(none.getDetails().getValues()).containsEntry("testable", false).containsEntry("providerType", "NONE"); + update(new QRecord().withValue("providerType", "SOUIN")); + HealthCheckOutcome souin = check.run(input()); + assertThat(souin.getStatus()).isEqualTo(HealthStatus.UP); + assertThat(souin.getDetails().getValues()).containsEntry("testable", false).containsEntry("reason", "provider type has no connection test"); update(new QRecord().withValue("enabled", false)); assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); new DeleteAction().execute(new DeleteInput(CdnTables.PROVIDER).withPrimaryKeys(List.of(providerId))); @@ -178,6 +187,63 @@ class CdnProviderHealthChecksTest + /******************************************************************************* + ** An adapter's own health probe wins over the connection test; its state, + ** message and details pass through with the provider type added. + *******************************************************************************/ + @Test + void healthProbePreferred() throws Exception + { + ProbingProvider probing = new ProbingProvider(); + CdnRuntime.registerProvider(CdnProviderType.CLOUDFRONT, () -> probing); + var check = check().getCheckInstance(); + for(CdnHealth.State state : CdnHealth.State.values()) + { + probing.health = new CdnHealth(state, "probe " + state, Map.of("testable", false)); + HealthCheckOutcome outcome = check.run(input()); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.valueOf(state.name())); + assertThat(outcome.getMessage()).isEqualTo("probe " + state); + assertThat(outcome.getDetails().getValues()).containsExactlyInAnyOrderEntriesOf(Map.of("providerType", "CLOUDFRONT", "testable", false)); + } + assertThat(probing.connectionTests).isZero(); + } + + + + /******************************************************************************* + ** The fake path adapter plus a health probe. + *******************************************************************************/ + private static class ProbingProvider extends CdnTestSupport.FakePathProvider implements CdnHealthProbe + { + private CdnHealth health; + private int connectionTests; + + + + /******************************************************************************* + ** The configured answer. + *******************************************************************************/ + @Override + public CdnHealth checkHealth(CdnProviderConfig config) + { + return (health); + } + + + + /******************************************************************************* + ** Counted so the test can prove it is not used. + *******************************************************************************/ + @Override + public ConnectionTest connectionTest(CdnProviderConfig config) + { + connectionTests++; + return (super.connectionTest(config)); + } + } + + + /******************************************************************************* ** Poison credentials, every free-form probe field and thrown error messages. ** HealthRunner.sanitize is package-private, so assert the raw boundary here. diff --git a/src/test/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontHealthTest.java b/src/test/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontHealthTest.java new file mode 100644 index 0000000..fff6056 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontHealthTest.java @@ -0,0 +1,394 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.cdn.providers.cloudfront; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.cdn.spi.CdnHealth; +import com.kingsrook.qbits.cdn.spi.CdnProviderConfig; +import com.kingsrook.qbits.cdn.spi.CdnProviderType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.mockito.ArgumentCaptor; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.core.exception.SdkClientException; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.cloudfront.CloudFrontClient; +import software.amazon.awssdk.services.cloudfront.model.CloudFrontException; +import software.amazon.awssdk.services.cloudfront.model.CustomHeaders; +import software.amazon.awssdk.services.cloudfront.model.DefaultCacheBehavior; +import software.amazon.awssdk.services.cloudfront.model.Distribution; +import software.amazon.awssdk.services.cloudfront.model.DistributionConfig; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigRequest; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigResponse; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionRequest; +import software.amazon.awssdk.services.cloudfront.model.GetDistributionResponse; +import software.amazon.awssdk.services.cloudfront.model.Origin; +import software.amazon.awssdk.services.cloudfront.model.OriginCustomHeader; +import software.amazon.awssdk.services.cloudfront.model.Origins; +import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; +import software.amazon.awssdk.services.cloudwatch.model.CloudWatchException; +import software.amazon.awssdk.services.cloudwatch.model.Dimension; +import software.amazon.awssdk.services.cloudwatch.model.GetMetricDataRequest; +import software.amazon.awssdk.services.cloudwatch.model.GetMetricDataResponse; +import software.amazon.awssdk.services.cloudwatch.model.MetricDataResult; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +/******************************************************************************* + ** The CloudFront health probe against mocked CloudFront and CloudWatch + ** clients (no network). + *******************************************************************************/ +class CloudFrontHealthTest +{ + private static final String DISTRIBUTION = "E2EXAMPLE123"; + private static final String SECRET = "FIXTURE-SECRET-NEVER-LEAK"; + private static final CdnProviderConfig CONFIG = new CdnProviderConfig("kof22-cloudfront", CdnProviderType.CLOUDFRONT, DISTRIBUTION, "CDN_AWS", null); + + private CloudFrontClient cloudFront; + private CloudWatchClient cloudWatch; + + + + /******************************************************************************* + ** Fresh mocks; by default CloudWatch has no datapoints. + *******************************************************************************/ + @BeforeEach + void mocks() + { + cloudFront = mock(CloudFrontClient.class); + cloudWatch = mock(CloudWatchClient.class); + errorRate(); + } + + + + /******************************************************************************* + ** An adapter whose clients are the mocks. + *******************************************************************************/ + private CloudFrontCdnProvider provider() + { + return new CloudFrontCdnProvider(new CloudFrontClientFactory(name -> null, null, null) + { + /*************************************************************************** + ** The CloudFront mock. + ***************************************************************************/ + @Override + public CloudFrontClient cloudFront(CdnProviderConfig config) + { + return cloudFront; + } + + + + /*************************************************************************** + ** The CloudWatch mock. + ***************************************************************************/ + @Override + public CloudWatchClient cloudWatch(CdnProviderConfig config) + { + return cloudWatch; + } + }); + } + + + + /******************************************************************************* + ** A distribution config, enabled or not, whose origin sends the verify header. + *******************************************************************************/ + private static DistributionConfig distributionConfig(boolean enabled) + { + Origin origin = Origin.builder().id("origin").domainName("origin.kof22.com") + .customHeaders(CustomHeaders.builder().quantity(1).items(OriginCustomHeader.builder().headerName("X-Origin-Verify").headerValue(SECRET).build()).build()) + .build(); + return DistributionConfig.builder().enabled(enabled) + .origins(Origins.builder().quantity(1).items(origin).build()) + .defaultCacheBehavior(DefaultCacheBehavior.builder().targetOriginId("origin").build()) + .build(); + } + + + + /******************************************************************************* + ** GetDistribution answers with this status and enabled flag. + *******************************************************************************/ + private void distribution(String status, boolean enabled) + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenReturn(GetDistributionResponse.builder() + .distribution(Distribution.builder().id(DISTRIBUTION).status(status).domainName("d1.cloudfront.net").distributionConfig(distributionConfig(enabled)).build()) + .build()); + } + + + + /******************************************************************************* + ** CloudWatch answers with these 5xxErrorRate datapoints. + *******************************************************************************/ + private void errorRate(Double... values) + { + when(cloudWatch.getMetricData(any(GetMetricDataRequest.class))).thenReturn(GetMetricDataResponse.builder() + .metricDataResults(MetricDataResult.builder().id("err5xx").values(values).build()).build()); + } + + + + /******************************************************************************* + ** A CloudFront service error. + *******************************************************************************/ + private static AwsServiceException error(String code, int status) + { + return CloudFrontException.builder().statusCode(status) + .awsErrorDetails(AwsErrorDetails.builder().errorCode(code).errorMessage(SECRET).build()).build(); + } + + + + /******************************************************************************* + ** Enabled and Deployed is UP; the 5xx rate is averaged into the details from + ** a five-minute Average query on the distribution's global metric. + *******************************************************************************/ + @Test + void deployedIsUp() + { + distribution("Deployed", true); + errorRate(1.0, 2.0); + + CdnHealth health = provider().checkHealth(CONFIG); + + assertThat(health.state()).isEqualTo(CdnHealth.State.UP); + assertThat(health.message()).isEqualTo("CloudFront distribution is enabled and deployed"); + assertThat(health.details()).containsExactlyInAnyOrderEntriesOf(Map.of("enabled", true, "status", "Deployed", "originHeaderConfigured", true, + "errorRateWindowMinutes", 30, "errorRate5xxPercent", 1.5)); + ArgumentCaptor request = ArgumentCaptor.forClass(GetMetricDataRequest.class); + verify(cloudWatch).getMetricData(request.capture()); + var stat = request.getValue().metricDataQueries().getFirst().metricStat(); + assertThat(stat.metric().metricName()).isEqualTo("5xxErrorRate"); + assertThat(stat.metric().namespace()).isEqualTo("AWS/CloudFront"); + assertThat(stat.metric().dimensions()).containsExactly(Dimension.builder().name("DistributionId").value(DISTRIBUTION).build(), + Dimension.builder().name("Region").value("Global").build()); + assertThat(stat.stat()).isEqualTo("Average"); + assertThat(stat.period()).isEqualTo(300); + assertThat(java.time.Duration.between(request.getValue().startTime(), request.getValue().endTime()).toMinutes()).isEqualTo(30); + verify(cloudFront, never()).getDistributionConfig(any(GetDistributionConfigRequest.class)); + assertClean(health); + } + + + + /******************************************************************************* + ** No traffic (no datapoints) or an unreadable metric leaves the rate out. + *******************************************************************************/ + @Test + void errorRateIsBestEffort() + { + distribution("Deployed", true); + assertThat(provider().checkHealth(CONFIG).details()).doesNotContainKey("errorRate5xxPercent").containsEntry("errorRateWindowMinutes", 30); + + when(cloudWatch.getMetricData(any(GetMetricDataRequest.class))).thenThrow(CloudWatchException.builder().statusCode(403) + .awsErrorDetails(AwsErrorDetails.builder().errorCode("AccessDenied").build()).build()); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.UP); + assertThat(health.details()).doesNotContainKey("errorRate5xxPercent"); + } + + + + /******************************************************************************* + ** A 5xx rate above 5% is DEGRADED; at 5% it is still UP. + *******************************************************************************/ + @Test + void highErrorRateIsDegraded() + { + distribution("Deployed", true); + errorRate(5.0); + assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(CdnHealth.State.UP); + + errorRate(4.0, 9.0); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED); + assertThat(health.message()).contains("5xx error rate"); + assertThat(health.details()).containsEntry("errorRate5xxPercent", 6.5); + } + + + + /******************************************************************************* + ** A deploying distribution is DEGRADED and says so. + *******************************************************************************/ + @Test + void inProgressIsDegraded() + { + distribution("InProgress", true); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED); + assertThat(health.message()).contains("deploying").contains("InProgress"); + assertThat(health.details()).containsEntry("status", "InProgress"); + } + + + + /******************************************************************************* + ** A disabled distribution is DOWN (and CloudWatch is not consulted); an + ** unexpected status value is reported as UNKNOWN, never echoed. + *******************************************************************************/ + @Test + void disabledIsDown() + { + distribution(SECRET, false); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.DOWN); + assertThat(health.message()).isEqualTo("CloudFront distribution is disabled"); + assertThat(health.details()).containsEntry("enabled", false).containsEntry("status", "UNKNOWN"); + verify(cloudWatch, never()).getMetricData(any(GetMetricDataRequest.class)); + assertClean(health); + } + + + + /******************************************************************************* + ** A different configured header name is reported missing: DEGRADED. + *******************************************************************************/ + @Test + void missingOriginHeaderIsDegraded() + { + distribution("Deployed", true); + CdnProviderConfig other = new CdnProviderConfig("x", CdnProviderType.CLOUDFRONT, DISTRIBUTION, "CDN_AWS", + Map.of(CloudFrontCdnProvider.SETTING_ORIGIN_VERIFY_HEADER, "X-Other-Verify")); + CdnHealth health = provider().checkHealth(other); + assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED); + assertThat(health.details()).containsEntry("originHeaderConfigured", false); + } + + + + /******************************************************************************* + ** Without GetDistribution (the section 7.5 policy) the probe falls back to + ** GetDistributionConfig: UP, not testable, Enabled still checked. + *******************************************************************************/ + @Test + void accessDeniedFallsBackToConfig() + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403)); + when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))) + .thenReturn(GetDistributionConfigResponse.builder().distributionConfig(distributionConfig(true)).build()); + + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.UP); + assertThat(health.message()).isEqualTo("CloudFront configured; distribution status not testable with current permissions"); + assertThat(health.details()).containsEntry("testable", false).containsEntry("reason", "credentials lack cloudfront:GetDistribution") + .containsEntry("enabled", true).containsEntry("status", "UNKNOWN"); + assertClean(health); + + when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))) + .thenReturn(GetDistributionConfigResponse.builder().distributionConfig(distributionConfig(false)).build()); + assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(CdnHealth.State.DOWN); + } + + + + /******************************************************************************* + ** When neither read is permitted the check is UP and not testable. + *******************************************************************************/ + @Test + void noReadPermissionIsUpNotTestable() + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403)); + when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))).thenThrow(error(null, 403)); + + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(CdnHealth.State.UP); + assertThat(health.details()).containsExactlyInAnyOrderEntriesOf(Map.of("testable", false, + "reason", "credentials lack cloudfront:GetDistribution and cloudfront:GetDistributionConfig")); + verify(cloudWatch, never()).getMetricData(any(GetMetricDataRequest.class)); + } + + + + /******************************************************************************* + ** AWS errors: missing distribution and rejected credentials are DOWN, + ** throttling and AWS-side errors DEGRADED; also after the config fallback. + *******************************************************************************/ + @ParameterizedTest + @CsvSource({ + "NoSuchDistribution, 404, DOWN, CloudFront distribution not found", + "InvalidClientTokenId, 403, DOWN, CloudFront rejected the status check", + "Throttling, 400, DEGRADED, CloudFront API throttled or unavailable; distribution status not checked", + "ServiceUnavailable, 503, DEGRADED, CloudFront API throttled or unavailable; distribution status not checked", + }) + void apiErrors(String code, int status, CdnHealth.State expected, String message) + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error(code, status)); + CdnHealth health = provider().checkHealth(CONFIG); + assertThat(health.state()).isEqualTo(expected); + assertThat(health.message()).isEqualTo(message); + assertClean(health); + + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403)); + when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))).thenThrow(error(code, status)); + assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(expected); + } + + + + /******************************************************************************* + ** Network failures are DEGRADED (transient); missing credentials or + ** distribution id are DOWN; other SDK failures are DOWN. + *******************************************************************************/ + @Test + void clientAndConfigFailures() + { + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(SdkClientException.create(SECRET)); + CdnHealth unreachable = provider().checkHealth(CONFIG); + assertThat(unreachable.state()).isEqualTo(CdnHealth.State.DEGRADED); + assertThat(unreachable.details()).containsEntry("transient", true); + assertClean(unreachable); + + CdnHealth noCredentials = new CloudFrontCdnProvider(new CloudFrontClientFactory(name -> null, null, null)).checkHealth(CONFIG); + assertThat(noCredentials.state()).isEqualTo(CdnHealth.State.DOWN); + assertThat(noCredentials.message()).isEqualTo("CloudFront credentials are not configured"); + + CdnHealth noId = provider().checkHealth(new CdnProviderConfig("x", CdnProviderType.CLOUDFRONT, " ", "CDN_AWS", null)); + assertThat(noId.state()).isEqualTo(CdnHealth.State.DOWN); + assertThat(noId.message()).isEqualTo("CloudFront distribution id is not configured"); + + when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(SdkException.builder().message(SECRET).build()); + CdnHealth other = provider().checkHealth(CONFIG); + assertThat(other.state()).isEqualTo(CdnHealth.State.DOWN); + assertClean(other); + } + + + + /******************************************************************************* + ** Nothing the AWS side controls as free text reaches the outcome. + *******************************************************************************/ + private static void assertClean(CdnHealth health) + { + assertThat(health.message()).doesNotContain(SECRET, DISTRIBUTION, "cloudfront.net", "origin.kof22.com"); + assertThat(List.copyOf(health.details().values()).toString()).doesNotContain(SECRET, DISTRIBUTION, "cloudfront.net", "origin.kof22.com"); + } +}