CHANGELOG.md

@@ -23,6 +23,7 @@ All notable changes to this project are documented here. The format follows Keep
 
 
 ### Fixed
+- CDN health checks no longer sit at UNKNOWN (which the central evaluator never alerts on). CloudFront checks the distribution itself: GetDistribution gives UP (enabled and Deployed), DEGRADED (InProgress, origin header missing, throttled or unreachable, or CloudWatch 5xxErrorRate above 5% over 30 minutes) or DOWN (missing or disabled distribution, rejected or missing credentials). Without `cloudfront:GetDistribution` it falls back to GetDistributionConfig and reports UP with `testable=false`. The NONE provider and adapters without a connection test report UP with `testable=false`. New optional SPI `CdnHealthProbe` / `CdnHealth`.
 - Derive the CDN qbit metadata version from the Maven artifact version, keeping release metadata and the help bundle in sync.
 - Tag rules accept ignored fields in `onlyWhenFieldsChange`, such as `!viewCount`, so counter-only updates do not purge rendered pages. Plain allow-lists still work, and ignored fields take precedence in mixed lists. No schema change is needed.
 - Purge actions warn when providers are NONE or hostnames are OFF, and report requests as recorded but not sent. The home explains recorded-only purges and that this version does not run origin header checks.

src/main/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecks.java

@@ -23,6 +23,7 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.FutureTask;
 import java.util.concurrent.TimeUnit;
@@ -30,9 +31,12 @@ import java.util.concurrent.TimeoutException;
 import com.kingsrook.qbits.cdn.CdnRuntime;
 import com.kingsrook.qbits.cdn.model.CdnTables;
 import com.kingsrook.qbits.cdn.purge.CdnPurgeDispatcher;
+import com.kingsrook.qbits.cdn.spi.CdnHealth;
+import com.kingsrook.qbits.cdn.spi.CdnHealthProbe;
 import com.kingsrook.qbits.cdn.spi.CdnOperations;
 import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
 import com.kingsrook.qbits.cdn.spi.CdnProviderInterface;
+import com.kingsrook.qbits.cdn.spi.CdnProviderType;
 import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -53,7 +57,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 
 
 /*******************************************************************************
- ** Central health checks reuse the read-only provider connection test.
+ ** Central health checks: an adapter's own health probe when it has one
+ ** (CloudFront), else the read-only connection test, else UP/not testable.
  *******************************************************************************/
 public class CdnProviderHealthChecks implements HealthCheckSupplier
 {
@@ -95,38 +100,86 @@ public class CdnProviderHealthChecks implements HealthCheckSupplier
       }
       CdnProviderConfig config = CdnPurgeDispatcher.configOf(provider);
       CdnProviderInterface adapter = CdnRuntime.provider(config.type());
-      if(!(adapter instanceof CdnOperations operations))
+      if(adapter instanceof CdnHealthProbe healthProbe)
       {
-         return (HealthCheckOutcome.unknown("CDN provider does not support connection tests"));
+         return (bounded(() -> fromProbe(healthProbe.checkHealth(config), config), input));
       }
-      return (probe(operations, config, input));
+      if(adapter instanceof CdnOperations operations)
+      {
+         return (bounded(() -> fromConnectionTest(operations.connectionTest(config), config), input));
+      }
+      return (notTestable(config));
    }
 
 
 
    /*******************************************************************************
-    ** The existing SPI has no timeout parameter. Bound its wait by the remaining
+    ** An adapter with nothing to probe (the NONE placeholder, or one without a
+    ** connection test) is UP and says it is not testable. UNKNOWN would sit on
+    ** the dashboard forever: the central evaluator never alerts or changes
+    ** state on UNKNOWN.
+    *******************************************************************************/
+   private static HealthCheckOutcome notTestable(CdnProviderConfig config)
+   {
+      boolean none = config.type() == CdnProviderType.NONE;
+      return (HealthCheckOutcome.up(none ? "No CDN configured; purges are recorded only" : "CDN provider configured; not testable (no connection test)")
+         .withDetails(new HealthDetails().withValue("providerType", config.type().name()).withValue("testable", false)
+            .withValue("reason", none ? "no CDN provider" : "provider type has no connection test")));
+   }
+
+
+
+   /*******************************************************************************
+    ** Map an adapter health probe. Its contract (CdnHealth) is fixed messages
+    ** and safe details only.
+    *******************************************************************************/
+   private static HealthCheckOutcome fromProbe(CdnHealth health, CdnProviderConfig config)
+   {
+      HealthCheckOutcome outcome = switch(health.state())
+      {
+         case UP -> HealthCheckOutcome.up(health.message());
+         case DEGRADED -> HealthCheckOutcome.degraded(health.message());
+         case DOWN -> HealthCheckOutcome.down(health.message());
+      };
+      HealthDetails details = new HealthDetails().withValue("providerType", config.type().name());
+      health.details().forEach(details::withValue);
+      return (outcome.withDetails(details));
+   }
+
+
+
+   /*******************************************************************************
+    ** Map a provider-neutral connection test.
+    *******************************************************************************/
+   private static HealthCheckOutcome fromConnectionTest(CdnOperations.ConnectionTest test, CdnProviderConfig config)
+   {
+      HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed")
+         : !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured")
+         : HealthCheckOutcome.up("CDN connection test passed");
+      // Only known provider states are safe; arbitrary status text may contain secrets.
+      String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN";
+      return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name())
+         .withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured())));
+   }
+
+
+
+   /*******************************************************************************
+    ** The SPI has no timeout parameter. Bound its wait by the remaining
     ** deadline and interrupt the probe on timeout or runner cancellation.
     *******************************************************************************/
-   private static HealthCheckOutcome probe(CdnOperations operations, CdnProviderConfig config, HealthCheckInput input)
+   private static HealthCheckOutcome bounded(Callable<HealthCheckOutcome> probe, HealthCheckInput input)
    {
       long remaining = Duration.between(Instant.now(), input.getDeadline()).toNanos();
       if(remaining <= 0)
       {
          return (HealthCheckOutcome.down("CDN connection test timed out"));
       }
-      FutureTask<CdnOperations.ConnectionTest> task = new FutureTask<>(() -> operations.connectionTest(config));
+      FutureTask<HealthCheckOutcome> task = new FutureTask<>(probe);
       Thread.startVirtualThread(task);
       try
       {
-         CdnOperations.ConnectionTest test = task.get(remaining, TimeUnit.NANOSECONDS);
-         HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed")
-            : !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured")
-            : HealthCheckOutcome.up("CDN connection test passed");
-         // Only known provider states are safe; arbitrary status text may contain secrets.
-         String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN";
-         return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name())
-            .withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured())));
+         return (task.get(remaining, TimeUnit.NANOSECONDS));
       }
       catch(TimeoutException e)
       {

src/main/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontCdnProvider.java

@@ -22,14 +22,18 @@ import java.time.LocalDate;
 import java.time.ZoneOffset;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.OptionalDouble;
 import java.util.Set;
 import java.util.TreeMap;
 import com.kingsrook.qbits.cdn.spi.CdnCapability;
 import com.kingsrook.qbits.cdn.spi.CdnEdgeStat;
 import com.kingsrook.qbits.cdn.spi.CdnException;
 import com.kingsrook.qbits.cdn.spi.CdnHeaderDialect;
+import com.kingsrook.qbits.cdn.spi.CdnHealth;
+import com.kingsrook.qbits.cdn.spi.CdnHealthProbe;
 import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
 import com.kingsrook.qbits.cdn.spi.CdnProviderInterface;
 import com.kingsrook.qbits.cdn.spi.CdnProviderType;
@@ -41,6 +45,8 @@ import software.amazon.awssdk.core.exception.SdkException;
 import software.amazon.awssdk.services.cloudfront.CloudFrontClient;
 import software.amazon.awssdk.services.cloudfront.model.CreateInvalidationResponse;
 import software.amazon.awssdk.services.cloudfront.model.Distribution;
+import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigRequest;
+import software.amazon.awssdk.services.cloudfront.model.GetDistributionRequest;
 import software.amazon.awssdk.services.cloudfront.model.GetInvalidationResponse;
 import software.amazon.awssdk.services.cloudwatch.CloudWatchClient;
 import software.amazon.awssdk.services.cloudwatch.model.Dimension;
@@ -70,7 +76,7 @@ import software.amazon.awssdk.services.cloudwatch.model.ScanBy;
  ** {@link #testConnection}, {@link #importStats} and {@link #readConfig}
  ** (compare the result with {@link CloudFrontDriftCheck#compare}).</p>
  *******************************************************************************/
-public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsrook.qbits.cdn.spi.CdnOperations
+public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsrook.qbits.cdn.spi.CdnOperations, CdnHealthProbe
 {
    /** Default origin custom header that proves a request came through CloudFront. */
    public static final String DEFAULT_ORIGIN_VERIFY_HEADER = "X-Origin-Verify";
@@ -81,6 +87,22 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo
    static final String STATUS_COMPLETED = "Completed";
    static final int    SECONDS_PER_DAY  = 86_400;
 
+   /** Distribution states CloudFront reports (anything else is reported as UNKNOWN). */
+   static final String STATUS_DEPLOYED    = "Deployed";
+   static final String STATUS_IN_PROGRESS = "InProgress";
+
+   /** 5xx error rate (percent, CloudWatch 5xxErrorRate) above which the health check is DEGRADED. */
+   static final double ERROR_RATE_5XX_DEGRADED_PERCENT = 5.0;
+
+   /** How far back the health check averages the 5xx error rate. */
+   static final int ERROR_RATE_WINDOW_MINUTES = 30;
+
+   /** CloudWatch period for the health check's 5xx error rate. */
+   static final int ERROR_RATE_PERIOD_SECONDS = 300;
+
+   /** Health message when the credentials cannot read the distribution status. */
+   static final String NOT_TESTABLE_MESSAGE = "CloudFront configured; distribution status not testable with current permissions";
+
    private static final Set<String> RETRYABLE_CODES = Set.of(
       "TooManyInvalidationsInProgress", "Throttling", "ThrottlingException", "RequestLimitExceeded", "ServiceUnavailable", "InternalError");
 
@@ -278,6 +300,181 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo
 
 
 
+   /*******************************************************************************
+    ** Central health probe. GetDistribution gives existence, Enabled and the
+    ** deployment Status: UP when enabled and Deployed, DEGRADED while
+    ** InProgress, DOWN when missing or disabled. The section 7.5 IAM policy
+    ** grants GetDistributionConfig but not GetDistribution, so on AccessDenied
+    ** this falls back to GetDistributionConfig (existence and Enabled, no
+    ** Status) and reports {@code testable=false}. If neither read is permitted
+    ** it reports UP with {@code testable=false}: UNKNOWN never alerts anyone,
+    ** and failed invalidations still show up as FAILED/DEAD purge requests.
+    ** The CloudWatch 5xxErrorRate over the last
+    ** {@value #ERROR_RATE_WINDOW_MINUTES} minutes is added when readable and
+    ** makes the check DEGRADED above {@value #ERROR_RATE_5XX_DEGRADED_PERCENT}
+    ** percent. Messages are fixed text; no AWS message, id or hostname is
+    ** copied. Never throws.
+    *******************************************************************************/
+   @Override
+   public CdnHealth checkHealth(CdnProviderConfig config)
+   {
+      String headerName = config.settings().getOrDefault(SETTING_ORIGIN_VERIFY_HEADER, DEFAULT_ORIGIN_VERIFY_HEADER);
+      String distributionId;
+      try
+      {
+         distributionId = distributionId(config);
+      }
+      catch(CdnException e)
+      {
+         return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution id is not configured", null);
+      }
+
+      Map<String, Object> details = new LinkedHashMap<>();
+      CloudFrontConfigSnapshot snapshot;
+      String status = null;
+      try(CloudFrontClient client = clientFactory.cloudFront(config))
+      {
+         try
+         {
+            Distribution distribution = client.getDistribution(GetDistributionRequest.builder().id(distributionId).build()).distribution();
+            status = distribution.status();
+            snapshot = CloudFrontConfigSnapshot.from(distribution.distributionConfig());
+         }
+         catch(AwsServiceException e)
+         {
+            if(!isAccessDenied(e))
+            {
+               return apiFailure(e);
+            }
+            details.put("testable", false);
+            try
+            {
+               snapshot = CloudFrontConfigSnapshot.from(client.getDistributionConfig(GetDistributionConfigRequest.builder().id(distributionId).build()).distributionConfig());
+               details.put("reason", "credentials lack cloudfront:GetDistribution");
+            }
+            catch(AwsServiceException e2)
+            {
+               if(!isAccessDenied(e2))
+               {
+                  return apiFailure(e2);
+               }
+               details.put("reason", "credentials lack cloudfront:GetDistribution and cloudfront:GetDistributionConfig");
+               return new CdnHealth(CdnHealth.State.UP, NOT_TESTABLE_MESSAGE, details);
+            }
+         }
+      }
+      catch(CdnException e)
+      {
+         return new CdnHealth(CdnHealth.State.DOWN, "CloudFront credentials are not configured", null);
+      }
+      catch(SdkClientException e)
+      {
+         return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront API unreachable; distribution status not checked", Map.of("transient", true));
+      }
+      catch(SdkException e)
+      {
+         return new CdnHealth(CdnHealth.State.DOWN, "CloudFront status check failed", null);
+      }
+
+      boolean statusTestable = !details.containsKey("testable");
+      boolean headerConfigured = snapshot.hasOriginHeader(headerName);
+      details.put("enabled", snapshot.enabled());
+      details.put("status", STATUS_DEPLOYED.equals(status) || STATUS_IN_PROGRESS.equals(status) ? status : "UNKNOWN");
+      details.put("originHeaderConfigured", headerConfigured);
+      if(!snapshot.enabled())
+      {
+         return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution is disabled", details);
+      }
+
+      Double errorRate = errorRate5xx(config, distributionId);
+      details.put("errorRateWindowMinutes", ERROR_RATE_WINDOW_MINUTES);
+      if(errorRate != null)
+      {
+         details.put("errorRate5xxPercent", Math.round(errorRate * 100.0) / 100.0);
+      }
+
+      if(STATUS_IN_PROGRESS.equals(status))
+      {
+         return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront distribution is deploying a change (InProgress)", details);
+      }
+      if(!headerConfigured)
+      {
+         return new CdnHealth(CdnHealth.State.DEGRADED, "CDN origin header is not configured", details);
+      }
+      if(errorRate != null && errorRate > ERROR_RATE_5XX_DEGRADED_PERCENT)
+      {
+         return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront 5xx error rate is above 5% over the last 30 minutes", details);
+      }
+      if(!statusTestable)
+      {
+         return new CdnHealth(CdnHealth.State.UP, NOT_TESTABLE_MESSAGE, details);
+      }
+      return new CdnHealth(CdnHealth.State.UP, "CloudFront distribution is enabled and deployed", details);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether an AWS error is a permissions refusal.
+    *******************************************************************************/
+   private static boolean isAccessDenied(AwsServiceException e)
+   {
+      String code = e.awsErrorDetails() == null ? null : e.awsErrorDetails().errorCode();
+      return "AccessDenied".equals(code) || "AccessDeniedException".equals(code) || (code == null && e.statusCode() == 403);
+   }
+
+
+
+   /*******************************************************************************
+    ** Health for an AWS error other than AccessDenied: a missing distribution is
+    ** DOWN, throttling and AWS-side errors are DEGRADED (transient), anything
+    ** else (rejected credentials, bad id) is DOWN.
+    *******************************************************************************/
+   private static CdnHealth apiFailure(AwsServiceException e)
+   {
+      String code = e.awsErrorDetails() == null ? null : e.awsErrorDetails().errorCode();
+      if("NoSuchDistribution".equals(code))
+      {
+         return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution not found", null);
+      }
+      if(toCdnException("GetDistribution", e).isRetryable())
+      {
+         return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront API throttled or unavailable; distribution status not checked", Map.of("transient", true));
+      }
+      return new CdnHealth(CdnHealth.State.DOWN, "CloudFront rejected the status check", null);
+   }
+
+
+
+   /*******************************************************************************
+    ** Average CloudWatch 5xxErrorRate (percent) over the health window, or null
+    ** when there is no traffic or the metric cannot be read. Best effort: it
+    ** never fails the health check.
+    *******************************************************************************/
+   private Double errorRate5xx(CdnProviderConfig config, String distributionId)
+   {
+      Instant end = Instant.now();
+      try(CloudWatchClient client = clientFactory.cloudWatch(config))
+      {
+         GetMetricDataResponse response = client.getMetricData(GetMetricDataRequest.builder()
+            .metricDataQueries(query("err5xx", "5xxErrorRate", "Average", distributionId, ERROR_RATE_PERIOD_SECONDS))
+            .startTime(end.minusSeconds(ERROR_RATE_WINDOW_MINUTES * 60L))
+            .endTime(end)
+            .build());
+         OptionalDouble average = response.metricDataResults().stream()
+            .flatMap(result -> result.values().stream())
+            .mapToDouble(Double::doubleValue)
+            .average();
+         return average.isPresent() ? average.getAsDouble() : null;
+      }
+      catch(CdnException | SdkException e)
+      {
+         return null;
+      }
+   }
+
+
+
    /*******************************************************************************
     ** The distribution config facts used for drift checks (GetDistributionConfig).
     *******************************************************************************/
@@ -368,6 +565,16 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo
     ** One daily CloudFront metric query.
     *******************************************************************************/
    private static MetricDataQuery query(String id, String metricName, String stat, String distributionId)
+   {
+      return query(id, metricName, stat, distributionId, SECONDS_PER_DAY);
+   }
+
+
+
+   /*******************************************************************************
+    ** One CloudFront metric query with the given period (seconds).
+    *******************************************************************************/
+   private static MetricDataQuery query(String id, String metricName, String stat, String distributionId, int periodSeconds)
    {
       Metric metric = Metric.builder()
          .namespace("AWS/CloudFront")
@@ -377,7 +584,7 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo
          .build();
       return MetricDataQuery.builder()
          .id(id)
-         .metricStat(MetricStat.builder().metric(metric).period(SECONDS_PER_DAY).stat(stat).build())
+         .metricStat(MetricStat.builder().metric(metric).period(periodSeconds).stat(stat).build())
          .returnData(true)
          .build();
    }

src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealth.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.cdn.spi;
+
+
+import java.util.Map;
+
+
+/*******************************************************************************
+ ** Result of an adapter's own health probe (see {@link CdnHealthProbe}). The
+ ** message and every detail value must be fixed text, numbers, booleans or
+ ** known provider states: never provider-supplied free text, hostnames, ids
+ ** or credentials, because they are copied to the central health tables.
+ **
+ ** @param state   UP, DEGRADED or DOWN
+ ** @param message a short fixed summary
+ ** @param details safe facts (may be empty, never null)
+ *******************************************************************************/
+public record CdnHealth(State state, String message, Map<String, Object> details)
+{
+   /*******************************************************************************
+    ** Health as the adapter sees it.
+    *******************************************************************************/
+   public enum State
+   {
+      UP,
+      DEGRADED,
+      DOWN
+   }
+
+
+
+   /*******************************************************************************
+    ** Null details become an empty map.
+    *******************************************************************************/
+   public CdnHealth
+   {
+      details = details == null ? Map.of() : Map.copyOf(details);
+   }
+}

src/main/java/com/kingsrook/qbits/cdn/spi/CdnHealthProbe.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.cdn.spi;
+
+
+/*******************************************************************************
+ ** Optional: an adapter that can say whether its CDN is healthy, beyond the
+ ** "Test connection" action ({@link CdnOperations}). The central health check
+ ** prefers this when an adapter implements it.
+ *******************************************************************************/
+public interface CdnHealthProbe
+{
+   /*******************************************************************************
+    ** Probe the CDN read-only. Never throws: failures come back as DOWN or
+    ** DEGRADED, with fixed messages (see {@link CdnHealth}).
+    *******************************************************************************/
+   CdnHealth checkHealth(CdnProviderConfig config);
+}

src/test/java/com/kingsrook/qbits/cdn/health/CdnProviderHealthChecksTest.java

@@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit;
 import com.kingsrook.qbits.cdn.CdnRuntime;
 import com.kingsrook.qbits.cdn.CdnTestSupport;
 import com.kingsrook.qbits.cdn.model.CdnTables;
+import com.kingsrook.qbits.cdn.spi.CdnHealth;
+import com.kingsrook.qbits.cdn.spi.CdnHealthProbe;
 import com.kingsrook.qbits.cdn.spi.CdnOperations;
 import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
 import com.kingsrook.qbits.cdn.spi.CdnProviderType;
@@ -168,7 +170,14 @@ class CdnProviderHealthChecksTest
    {
       var check = check().getCheckInstance();
       update(new QRecord().withValue("providerType", "NONE"));
-      assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
+      HealthCheckOutcome none = check.run(input());
+      assertThat(none.getStatus()).isEqualTo(HealthStatus.UP);
+      assertThat(none.getMessage()).isEqualTo("No CDN configured; purges are recorded only");
+      assertThat(none.getDetails().getValues()).containsEntry("testable", false).containsEntry("providerType", "NONE");
+      update(new QRecord().withValue("providerType", "SOUIN"));
+      HealthCheckOutcome souin = check.run(input());
+      assertThat(souin.getStatus()).isEqualTo(HealthStatus.UP);
+      assertThat(souin.getDetails().getValues()).containsEntry("testable", false).containsEntry("reason", "provider type has no connection test");
       update(new QRecord().withValue("enabled", false));
       assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
       new DeleteAction().execute(new DeleteInput(CdnTables.PROVIDER).withPrimaryKeys(List.of(providerId)));
@@ -178,6 +187,63 @@ class CdnProviderHealthChecksTest
 
 
 
+   /*******************************************************************************
+    ** An adapter's own health probe wins over the connection test; its state,
+    ** message and details pass through with the provider type added.
+    *******************************************************************************/
+   @Test
+   void healthProbePreferred() throws Exception
+   {
+      ProbingProvider probing = new ProbingProvider();
+      CdnRuntime.registerProvider(CdnProviderType.CLOUDFRONT, () -> probing);
+      var check = check().getCheckInstance();
+      for(CdnHealth.State state : CdnHealth.State.values())
+      {
+         probing.health = new CdnHealth(state, "probe " + state, Map.of("testable", false));
+         HealthCheckOutcome outcome = check.run(input());
+         assertThat(outcome.getStatus()).isEqualTo(HealthStatus.valueOf(state.name()));
+         assertThat(outcome.getMessage()).isEqualTo("probe " + state);
+         assertThat(outcome.getDetails().getValues()).containsExactlyInAnyOrderEntriesOf(Map.of("providerType", "CLOUDFRONT", "testable", false));
+      }
+      assertThat(probing.connectionTests).isZero();
+   }
+
+
+
+   /*******************************************************************************
+    ** The fake path adapter plus a health probe.
+    *******************************************************************************/
+   private static class ProbingProvider extends CdnTestSupport.FakePathProvider implements CdnHealthProbe
+   {
+      private CdnHealth health;
+      private int       connectionTests;
+
+
+
+      /*******************************************************************************
+       ** The configured answer.
+       *******************************************************************************/
+      @Override
+      public CdnHealth checkHealth(CdnProviderConfig config)
+      {
+         return (health);
+      }
+
+
+
+      /*******************************************************************************
+       ** Counted so the test can prove it is not used.
+       *******************************************************************************/
+      @Override
+      public ConnectionTest connectionTest(CdnProviderConfig config)
+      {
+         connectionTests++;
+         return (super.connectionTest(config));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Poison credentials, every free-form probe field and thrown error messages.
     ** HealthRunner.sanitize is package-private, so assert the raw boundary here.

src/test/java/com/kingsrook/qbits/cdn/providers/cloudfront/CloudFrontHealthTest.java

@@ -0,0 +1,394 @@
+/*
+ * Copyright 2026 Kingsrook, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.cdn.providers.cloudfront;
+
+
+import java.util.List;
+import java.util.Map;
+import com.kingsrook.qbits.cdn.spi.CdnHealth;
+import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
+import com.kingsrook.qbits.cdn.spi.CdnProviderType;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.mockito.ArgumentCaptor;
+import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
+import software.amazon.awssdk.awscore.exception.AwsServiceException;
+import software.amazon.awssdk.core.exception.SdkClientException;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.services.cloudfront.CloudFrontClient;
+import software.amazon.awssdk.services.cloudfront.model.CloudFrontException;
+import software.amazon.awssdk.services.cloudfront.model.CustomHeaders;
+import software.amazon.awssdk.services.cloudfront.model.DefaultCacheBehavior;
+import software.amazon.awssdk.services.cloudfront.model.Distribution;
+import software.amazon.awssdk.services.cloudfront.model.DistributionConfig;
+import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigRequest;
+import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigResponse;
+import software.amazon.awssdk.services.cloudfront.model.GetDistributionRequest;
+import software.amazon.awssdk.services.cloudfront.model.GetDistributionResponse;
+import software.amazon.awssdk.services.cloudfront.model.Origin;
+import software.amazon.awssdk.services.cloudfront.model.OriginCustomHeader;
+import software.amazon.awssdk.services.cloudfront.model.Origins;
+import software.amazon.awssdk.services.cloudwatch.CloudWatchClient;
+import software.amazon.awssdk.services.cloudwatch.model.CloudWatchException;
+import software.amazon.awssdk.services.cloudwatch.model.Dimension;
+import software.amazon.awssdk.services.cloudwatch.model.GetMetricDataRequest;
+import software.amazon.awssdk.services.cloudwatch.model.GetMetricDataResponse;
+import software.amazon.awssdk.services.cloudwatch.model.MetricDataResult;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+
+/*******************************************************************************
+ ** The CloudFront health probe against mocked CloudFront and CloudWatch
+ ** clients (no network).
+ *******************************************************************************/
+class CloudFrontHealthTest
+{
+   private static final String            DISTRIBUTION = "E2EXAMPLE123";
+   private static final String            SECRET       = "FIXTURE-SECRET-NEVER-LEAK";
+   private static final CdnProviderConfig CONFIG       = new CdnProviderConfig("kof22-cloudfront", CdnProviderType.CLOUDFRONT, DISTRIBUTION, "CDN_AWS", null);
+
+   private CloudFrontClient cloudFront;
+   private CloudWatchClient cloudWatch;
+
+
+
+   /*******************************************************************************
+    ** Fresh mocks; by default CloudWatch has no datapoints.
+    *******************************************************************************/
+   @BeforeEach
+   void mocks()
+   {
+      cloudFront = mock(CloudFrontClient.class);
+      cloudWatch = mock(CloudWatchClient.class);
+      errorRate();
+   }
+
+
+
+   /*******************************************************************************
+    ** An adapter whose clients are the mocks.
+    *******************************************************************************/
+   private CloudFrontCdnProvider provider()
+   {
+      return new CloudFrontCdnProvider(new CloudFrontClientFactory(name -> null, null, null)
+      {
+         /***************************************************************************
+          ** The CloudFront mock.
+          ***************************************************************************/
+         @Override
+         public CloudFrontClient cloudFront(CdnProviderConfig config)
+         {
+            return cloudFront;
+         }
+
+
+
+         /***************************************************************************
+          ** The CloudWatch mock.
+          ***************************************************************************/
+         @Override
+         public CloudWatchClient cloudWatch(CdnProviderConfig config)
+         {
+            return cloudWatch;
+         }
+      });
+   }
+
+
+
+   /*******************************************************************************
+    ** A distribution config, enabled or not, whose origin sends the verify header.
+    *******************************************************************************/
+   private static DistributionConfig distributionConfig(boolean enabled)
+   {
+      Origin origin = Origin.builder().id("origin").domainName("origin.kof22.com")
+         .customHeaders(CustomHeaders.builder().quantity(1).items(OriginCustomHeader.builder().headerName("X-Origin-Verify").headerValue(SECRET).build()).build())
+         .build();
+      return DistributionConfig.builder().enabled(enabled)
+         .origins(Origins.builder().quantity(1).items(origin).build())
+         .defaultCacheBehavior(DefaultCacheBehavior.builder().targetOriginId("origin").build())
+         .build();
+   }
+
+
+
+   /*******************************************************************************
+    ** GetDistribution answers with this status and enabled flag.
+    *******************************************************************************/
+   private void distribution(String status, boolean enabled)
+   {
+      when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenReturn(GetDistributionResponse.builder()
+         .distribution(Distribution.builder().id(DISTRIBUTION).status(status).domainName("d1.cloudfront.net").distributionConfig(distributionConfig(enabled)).build())
+         .build());
+   }
+
+
+
+   /*******************************************************************************
+    ** CloudWatch answers with these 5xxErrorRate datapoints.
+    *******************************************************************************/
+   private void errorRate(Double... values)
+   {
+      when(cloudWatch.getMetricData(any(GetMetricDataRequest.class))).thenReturn(GetMetricDataResponse.builder()
+         .metricDataResults(MetricDataResult.builder().id("err5xx").values(values).build()).build());
+   }
+
+
+
+   /*******************************************************************************
+    ** A CloudFront service error.
+    *******************************************************************************/
+   private static AwsServiceException error(String code, int status)
+   {
+      return CloudFrontException.builder().statusCode(status)
+         .awsErrorDetails(AwsErrorDetails.builder().errorCode(code).errorMessage(SECRET).build()).build();
+   }
+
+
+
+   /*******************************************************************************
+    ** Enabled and Deployed is UP; the 5xx rate is averaged into the details from
+    ** a five-minute Average query on the distribution's global metric.
+    *******************************************************************************/
+   @Test
+   void deployedIsUp()
+   {
+      distribution("Deployed", true);
+      errorRate(1.0, 2.0);
+
+      CdnHealth health = provider().checkHealth(CONFIG);
+
+      assertThat(health.state()).isEqualTo(CdnHealth.State.UP);
+      assertThat(health.message()).isEqualTo("CloudFront distribution is enabled and deployed");
+      assertThat(health.details()).containsExactlyInAnyOrderEntriesOf(Map.of("enabled", true, "status", "Deployed", "originHeaderConfigured", true,
+         "errorRateWindowMinutes", 30, "errorRate5xxPercent", 1.5));
+      ArgumentCaptor<GetMetricDataRequest> request = ArgumentCaptor.forClass(GetMetricDataRequest.class);
+      verify(cloudWatch).getMetricData(request.capture());
+      var stat = request.getValue().metricDataQueries().getFirst().metricStat();
+      assertThat(stat.metric().metricName()).isEqualTo("5xxErrorRate");
+      assertThat(stat.metric().namespace()).isEqualTo("AWS/CloudFront");
+      assertThat(stat.metric().dimensions()).containsExactly(Dimension.builder().name("DistributionId").value(DISTRIBUTION).build(),
+         Dimension.builder().name("Region").value("Global").build());
+      assertThat(stat.stat()).isEqualTo("Average");
+      assertThat(stat.period()).isEqualTo(300);
+      assertThat(java.time.Duration.between(request.getValue().startTime(), request.getValue().endTime()).toMinutes()).isEqualTo(30);
+      verify(cloudFront, never()).getDistributionConfig(any(GetDistributionConfigRequest.class));
+      assertClean(health);
+   }
+
+
+
+   /*******************************************************************************
+    ** No traffic (no datapoints) or an unreadable metric leaves the rate out.
+    *******************************************************************************/
+   @Test
+   void errorRateIsBestEffort()
+   {
+      distribution("Deployed", true);
+      assertThat(provider().checkHealth(CONFIG).details()).doesNotContainKey("errorRate5xxPercent").containsEntry("errorRateWindowMinutes", 30);
+
+      when(cloudWatch.getMetricData(any(GetMetricDataRequest.class))).thenThrow(CloudWatchException.builder().statusCode(403)
+         .awsErrorDetails(AwsErrorDetails.builder().errorCode("AccessDenied").build()).build());
+      CdnHealth health = provider().checkHealth(CONFIG);
+      assertThat(health.state()).isEqualTo(CdnHealth.State.UP);
+      assertThat(health.details()).doesNotContainKey("errorRate5xxPercent");
+   }
+
+
+
+   /*******************************************************************************
+    ** A 5xx rate above 5% is DEGRADED; at 5% it is still UP.
+    *******************************************************************************/
+   @Test
+   void highErrorRateIsDegraded()
+   {
+      distribution("Deployed", true);
+      errorRate(5.0);
+      assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(CdnHealth.State.UP);
+
+      errorRate(4.0, 9.0);
+      CdnHealth health = provider().checkHealth(CONFIG);
+      assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED);
+      assertThat(health.message()).contains("5xx error rate");
+      assertThat(health.details()).containsEntry("errorRate5xxPercent", 6.5);
+   }
+
+
+
+   /*******************************************************************************
+    ** A deploying distribution is DEGRADED and says so.
+    *******************************************************************************/
+   @Test
+   void inProgressIsDegraded()
+   {
+      distribution("InProgress", true);
+      CdnHealth health = provider().checkHealth(CONFIG);
+      assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED);
+      assertThat(health.message()).contains("deploying").contains("InProgress");
+      assertThat(health.details()).containsEntry("status", "InProgress");
+   }
+
+
+
+   /*******************************************************************************
+    ** A disabled distribution is DOWN (and CloudWatch is not consulted); an
+    ** unexpected status value is reported as UNKNOWN, never echoed.
+    *******************************************************************************/
+   @Test
+   void disabledIsDown()
+   {
+      distribution(SECRET, false);
+      CdnHealth health = provider().checkHealth(CONFIG);
+      assertThat(health.state()).isEqualTo(CdnHealth.State.DOWN);
+      assertThat(health.message()).isEqualTo("CloudFront distribution is disabled");
+      assertThat(health.details()).containsEntry("enabled", false).containsEntry("status", "UNKNOWN");
+      verify(cloudWatch, never()).getMetricData(any(GetMetricDataRequest.class));
+      assertClean(health);
+   }
+
+
+
+   /*******************************************************************************
+    ** A different configured header name is reported missing: DEGRADED.
+    *******************************************************************************/
+   @Test
+   void missingOriginHeaderIsDegraded()
+   {
+      distribution("Deployed", true);
+      CdnProviderConfig other = new CdnProviderConfig("x", CdnProviderType.CLOUDFRONT, DISTRIBUTION, "CDN_AWS",
+         Map.of(CloudFrontCdnProvider.SETTING_ORIGIN_VERIFY_HEADER, "X-Other-Verify"));
+      CdnHealth health = provider().checkHealth(other);
+      assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED);
+      assertThat(health.details()).containsEntry("originHeaderConfigured", false);
+   }
+
+
+
+   /*******************************************************************************
+    ** Without GetDistribution (the section 7.5 policy) the probe falls back to
+    ** GetDistributionConfig: UP, not testable, Enabled still checked.
+    *******************************************************************************/
+   @Test
+   void accessDeniedFallsBackToConfig()
+   {
+      when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403));
+      when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class)))
+         .thenReturn(GetDistributionConfigResponse.builder().distributionConfig(distributionConfig(true)).build());
+
+      CdnHealth health = provider().checkHealth(CONFIG);
+      assertThat(health.state()).isEqualTo(CdnHealth.State.UP);
+      assertThat(health.message()).isEqualTo("CloudFront configured; distribution status not testable with current permissions");
+      assertThat(health.details()).containsEntry("testable", false).containsEntry("reason", "credentials lack cloudfront:GetDistribution")
+         .containsEntry("enabled", true).containsEntry("status", "UNKNOWN");
+      assertClean(health);
+
+      when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class)))
+         .thenReturn(GetDistributionConfigResponse.builder().distributionConfig(distributionConfig(false)).build());
+      assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(CdnHealth.State.DOWN);
+   }
+
+
+
+   /*******************************************************************************
+    ** When neither read is permitted the check is UP and not testable.
+    *******************************************************************************/
+   @Test
+   void noReadPermissionIsUpNotTestable()
+   {
+      when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403));
+      when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))).thenThrow(error(null, 403));
+
+      CdnHealth health = provider().checkHealth(CONFIG);
+      assertThat(health.state()).isEqualTo(CdnHealth.State.UP);
+      assertThat(health.details()).containsExactlyInAnyOrderEntriesOf(Map.of("testable", false,
+         "reason", "credentials lack cloudfront:GetDistribution and cloudfront:GetDistributionConfig"));
+      verify(cloudWatch, never()).getMetricData(any(GetMetricDataRequest.class));
+   }
+
+
+
+   /*******************************************************************************
+    ** AWS errors: missing distribution and rejected credentials are DOWN,
+    ** throttling and AWS-side errors DEGRADED; also after the config fallback.
+    *******************************************************************************/
+   @ParameterizedTest
+   @CsvSource({
+      "NoSuchDistribution, 404, DOWN, CloudFront distribution not found",
+      "InvalidClientTokenId, 403, DOWN, CloudFront rejected the status check",
+      "Throttling, 400, DEGRADED, CloudFront API throttled or unavailable; distribution status not checked",
+      "ServiceUnavailable, 503, DEGRADED, CloudFront API throttled or unavailable; distribution status not checked",
+   })
+   void apiErrors(String code, int status, CdnHealth.State expected, String message)
+   {
+      when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error(code, status));
+      CdnHealth health = provider().checkHealth(CONFIG);
+      assertThat(health.state()).isEqualTo(expected);
+      assertThat(health.message()).isEqualTo(message);
+      assertClean(health);
+
+      when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403));
+      when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))).thenThrow(error(code, status));
+      assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(expected);
+   }
+
+
+
+   /*******************************************************************************
+    ** Network failures are DEGRADED (transient); missing credentials or
+    ** distribution id are DOWN; other SDK failures are DOWN.
+    *******************************************************************************/
+   @Test
+   void clientAndConfigFailures()
+   {
+      when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(SdkClientException.create(SECRET));
+      CdnHealth unreachable = provider().checkHealth(CONFIG);
+      assertThat(unreachable.state()).isEqualTo(CdnHealth.State.DEGRADED);
+      assertThat(unreachable.details()).containsEntry("transient", true);
+      assertClean(unreachable);
+
+      CdnHealth noCredentials = new CloudFrontCdnProvider(new CloudFrontClientFactory(name -> null, null, null)).checkHealth(CONFIG);
+      assertThat(noCredentials.state()).isEqualTo(CdnHealth.State.DOWN);
+      assertThat(noCredentials.message()).isEqualTo("CloudFront credentials are not configured");
+
+      CdnHealth noId = provider().checkHealth(new CdnProviderConfig("x", CdnProviderType.CLOUDFRONT, " ", "CDN_AWS", null));
+      assertThat(noId.state()).isEqualTo(CdnHealth.State.DOWN);
+      assertThat(noId.message()).isEqualTo("CloudFront distribution id is not configured");
+
+      when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(SdkException.builder().message(SECRET).build());
+      CdnHealth other = provider().checkHealth(CONFIG);
+      assertThat(other.state()).isEqualTo(CdnHealth.State.DOWN);
+      assertClean(other);
+   }
+
+
+
+   /*******************************************************************************
+    ** Nothing the AWS side controls as free text reaches the outcome.
+    *******************************************************************************/
+   private static void assertClean(CdnHealth health)
+   {
+      assertThat(health.message()).doesNotContain(SECRET, DISTRIBUTION, "cloudfront.net", "origin.kof22.com");
+      assertThat(List.copyOf(health.details().values()).toString()).doesNotContain(SECRET, DISTRIBUTION, "cloudfront.net", "origin.kof22.com");
+   }
+}