Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2c32095eb5 | ||
|
|
88a1f1c235 | ||
|
|
2ab5d4d186 | ||
|
|
5f4e37e023 | ||
|
|
d496d8defb | ||
|
|
47b9c52b32 | ||
|
|
d683c20449 | ||
|
|
638e988461 | ||
|
|
8de023efa7 | ||
|
|
90cf7fd158
|
||
|
|
7b492f857c | ||
|
|
9ff55f4c23
|
||
|
|
407b7674c6 | ||
|
|
3ff13a663e
|
||
|
|
85f2ba60ef | ||
|
|
79ae6264f8
|
||
|
|
98114a6482 | ||
|
|
3d72a5d05d
|
||
|
|
4fb8d74458 |
No files matched your search
+11
-57
@@ -1,60 +1,14 @@
|
||||
version: 2.1
|
||||
|
||||
setup: true
|
||||
orbs:
|
||||
qqq-orb: kingsrook/qqq-orb@0.6.9
|
||||
|
||||
commands:
|
||||
configure-package-install:
|
||||
steps:
|
||||
- run:
|
||||
name: Keep package installation unattended
|
||||
command: |
|
||||
# These build machines are discarded after the job.
|
||||
sudo install -d /etc/needrestart/conf.d
|
||||
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
|
||||
|
||||
qqq-orb: kingsrook/qqq-orb@0.8
|
||||
workflows:
|
||||
test_only:
|
||||
jobs:
|
||||
- qqq-orb/mvn_test_only:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
filters:
|
||||
branches:
|
||||
ignore: /(develop|release\/.*)/
|
||||
|
||||
publish_snapshot:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: snapshot
|
||||
filters:
|
||||
branches:
|
||||
only: [develop]
|
||||
|
||||
publish_release_candidate:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release_candidate
|
||||
filters:
|
||||
branches:
|
||||
only: [/release\/.*/]
|
||||
|
||||
publish_release:
|
||||
jobs:
|
||||
- qqq-orb/mvn_publish:
|
||||
pre-steps:
|
||||
- configure-package-install
|
||||
context: [qqq-maven-registry-credentials]
|
||||
branch_type: release
|
||||
filters:
|
||||
branches:
|
||||
ignore: /.*/
|
||||
tags:
|
||||
only: [/^v\d+\.\d+\.\d+$/]
|
||||
setup:
|
||||
jobs:
|
||||
- qqq-orb/generate_pipeline:
|
||||
orb_version: '0.8'
|
||||
filters:
|
||||
branches:
|
||||
ignore: automation/qbit-rc-pending
|
||||
tags:
|
||||
only: /^v\d+\.\d+\.\d+$/
|
||||
@@ -0,0 +1,23 @@
|
||||
name: CodeQL
|
||||
'on':
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- develop
|
||||
- release/**
|
||||
- hotfix/**
|
||||
pull_request: {}
|
||||
schedule:
|
||||
- cron: 41 16 * * 0
|
||||
workflow_dispatch: {}
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
security-events: write
|
||||
packages: read
|
||||
concurrency:
|
||||
group: codeql-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
jobs:
|
||||
analyze:
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@1c389210e6dfda1630d053c07ae4dae523dd5576
|
||||
@@ -1,25 +1,74 @@
|
||||
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
|
||||
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
|
||||
# the release line and the release/* push runs CircleCI publish_release_candidate.
|
||||
name: Release candidate
|
||||
|
||||
on:
|
||||
'on':
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dry_run:
|
||||
description: Print the plan without merging or creating branches
|
||||
description: Plan a cut without changing release state
|
||||
type: boolean
|
||||
default: false
|
||||
finalize:
|
||||
description: Retry finalization of an existing cut; do not cut another RC
|
||||
type: boolean
|
||||
default: false
|
||||
recover:
|
||||
description: Recover the same published RC after a deployment timeout; never
|
||||
reupload
|
||||
type: boolean
|
||||
default: false
|
||||
circle_build:
|
||||
description: Original Circle publisher build number (recovery only)
|
||||
type: string
|
||||
default: ''
|
||||
merge_sha:
|
||||
description: Existing cut merge_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
bump_sha:
|
||||
description: Existing cut bump_sha (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
branch:
|
||||
description: Existing cut branch (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
version:
|
||||
description: Existing cut version (finalize retry only)
|
||||
type: string
|
||||
default: ''
|
||||
pull_request:
|
||||
types: [closed]
|
||||
branches: [develop]
|
||||
|
||||
types:
|
||||
- closed
|
||||
branches:
|
||||
- develop
|
||||
repository_dispatch:
|
||||
types:
|
||||
- qbit-rc-published
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
actions: read
|
||||
checks: read
|
||||
jobs:
|
||||
release-rc:
|
||||
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
|
||||
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
|
||||
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|
||||
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
|
||||
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
|
||||
'release:rc'))
|
||||
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@1c389210e6dfda1630d053c07ae4dae523dd5576
|
||||
with:
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
|
||||
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
|
||||
}}
|
||||
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
|
||||
== 'workflow_dispatch' && inputs.finalize) }}
|
||||
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
|
||||
}}
|
||||
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|
||||
|| '' }}
|
||||
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|
||||
|| inputs.merge_sha || '' }}
|
||||
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|
||||
|| inputs.bump_sha || '' }}
|
||||
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|
||||
|| inputs.branch || '' }}
|
||||
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|
||||
|| inputs.version || '' }}
|
||||
secrets: inherit
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
profile: qbit
|
||||
orb_version: '0.8'
|
||||
default_branch: develop
|
||||
java_version: '21'
|
||||
browsers: true
|
||||
api_checks: false
|
||||
docs_only: false
|
||||
release_on_main: false
|
||||
release_tag_policy: stable
|
||||
test_branch_policy: non-release
|
||||
publish:
|
||||
snapshot: true
|
||||
release_candidate: true
|
||||
release: true
|
||||
asciidoc: false
|
||||
release_completion: true
|
||||
@@ -23,6 +23,8 @@ All notable changes to this project are documented here. The format follows Keep
|
||||
|
||||
|
||||
### Fixed
|
||||
- CDN health checks no longer sit at UNKNOWN (which the central evaluator never alerts on). CloudFront checks the distribution itself: GetDistribution gives UP (enabled and Deployed), DEGRADED (InProgress, origin header missing, throttled or unreachable, or CloudWatch 5xxErrorRate above 5% over 30 minutes) or DOWN (missing or disabled distribution, rejected or missing credentials). Without `cloudfront:GetDistribution` it falls back to GetDistributionConfig and reports UP with `testable=false`. The NONE provider and adapters without a connection test report UP with `testable=false`. New optional SPI `CdnHealthProbe` / `CdnHealth`.
|
||||
- Derive the CDN qbit metadata version from the Maven artifact version, keeping release metadata and the help bundle in sync.
|
||||
- Tag rules accept ignored fields in `onlyWhenFieldsChange`, such as `!viewCount`, so counter-only updates do not purge rendered pages. Plain allow-lists still work, and ignored fields take precedence in mixed lists. No schema change is needed.
|
||||
- Purge actions warn when providers are NONE or hostnames are OFF, and report requests as recorded but not sent. The home explains recorded-only purges and that this version does not run origin header checks.
|
||||
- Tag rule `{prefix.field}` placeholders now resolve the joined table from the `<prefix>Id` field's possible-value source, so prefixed tables (`cms_blog`) work; a missing joined table is logged instead of silently skipped.
|
||||
|
||||
@@ -113,6 +113,7 @@
|
||||
<filtering>true</filtering>
|
||||
<includes>
|
||||
<include>help/*.yaml</include>
|
||||
<include>qbit-cdn.properties</include>
|
||||
</includes>
|
||||
</resource>
|
||||
<resource>
|
||||
@@ -120,10 +121,21 @@
|
||||
<filtering>false</filtering>
|
||||
<excludes>
|
||||
<exclude>help/*.yaml</exclude>
|
||||
<exclude>qbit-cdn.properties</exclude>
|
||||
</excludes>
|
||||
</resource>
|
||||
</resources>
|
||||
<plugins>
|
||||
<!-- the artifact version, for the help bundle version test -->
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<systemPropertyVariables combine.children="append">
|
||||
<qbit.artifactVersion>${project.version}</qbit.artifactVersion>
|
||||
</systemPropertyVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||
|
||||
@@ -17,8 +17,11 @@
|
||||
package com.kingsrook.qbits.cdn;
|
||||
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import com.kingsrook.qbits.cdn.app.CdnWidgets;
|
||||
import com.kingsrook.qbits.cdn.health.CdnProviderHealthChecks;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
@@ -48,13 +51,41 @@ public class CdnQBitProducer implements QBitMetaDataProducer<CdnQBitConfig>
|
||||
{
|
||||
public static final String GROUP_ID = "com.kingsrook.qbits";
|
||||
public static final String ARTIFACT_ID = "qbit-cdn";
|
||||
public static final String VERSION = "0.1.0";
|
||||
public static final String VERSION = loadVersion();
|
||||
|
||||
private CdnQBitConfig qBitConfig;
|
||||
private String namespace;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Read the artifact version from the Maven-filtered resource.
|
||||
*******************************************************************************/
|
||||
private static String loadVersion()
|
||||
{
|
||||
try(InputStream inputStream = CdnQBitProducer.class.getClassLoader().getResourceAsStream("qbit-cdn.properties"))
|
||||
{
|
||||
if(inputStream == null)
|
||||
{
|
||||
throw (new IllegalStateException("Missing qbit-cdn.properties; cannot determine the CDN qbit artifact version."));
|
||||
}
|
||||
Properties properties = new Properties();
|
||||
properties.load(inputStream);
|
||||
String version = properties.getProperty("version");
|
||||
if(version == null || version.isBlank() || version.contains("${"))
|
||||
{
|
||||
throw (new IllegalStateException("Missing or unfiltered version in qbit-cdn.properties; build with Maven resource filtering."));
|
||||
}
|
||||
return (version);
|
||||
}
|
||||
catch(IOException e)
|
||||
{
|
||||
throw (new IllegalStateException("Could not read the CDN qbit artifact version from qbit-cdn.properties.", e));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fluent setter for the config.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -23,6 +23,7 @@ import java.time.Instant;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Locale;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.FutureTask;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
@@ -30,9 +31,12 @@ import java.util.concurrent.TimeoutException;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.purge.CdnPurgeDispatcher;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnHealth;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnHealthProbe;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnOperations;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderInterface;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
@@ -53,7 +57,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Central health checks reuse the read-only provider connection test.
|
||||
** Central health checks: an adapter's own health probe when it has one
|
||||
** (CloudFront), else the read-only connection test, else UP/not testable.
|
||||
*******************************************************************************/
|
||||
public class CdnProviderHealthChecks implements HealthCheckSupplier
|
||||
{
|
||||
@@ -95,38 +100,86 @@ public class CdnProviderHealthChecks implements HealthCheckSupplier
|
||||
}
|
||||
CdnProviderConfig config = CdnPurgeDispatcher.configOf(provider);
|
||||
CdnProviderInterface adapter = CdnRuntime.provider(config.type());
|
||||
if(!(adapter instanceof CdnOperations operations))
|
||||
if(adapter instanceof CdnHealthProbe healthProbe)
|
||||
{
|
||||
return (HealthCheckOutcome.unknown("CDN provider does not support connection tests"));
|
||||
return (bounded(() -> fromProbe(healthProbe.checkHealth(config), config), input));
|
||||
}
|
||||
return (probe(operations, config, input));
|
||||
if(adapter instanceof CdnOperations operations)
|
||||
{
|
||||
return (bounded(() -> fromConnectionTest(operations.connectionTest(config), config), input));
|
||||
}
|
||||
return (notTestable(config));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The existing SPI has no timeout parameter. Bound its wait by the remaining
|
||||
** An adapter with nothing to probe (the NONE placeholder, or one without a
|
||||
** connection test) is UP and says it is not testable. UNKNOWN would sit on
|
||||
** the dashboard forever: the central evaluator never alerts or changes
|
||||
** state on UNKNOWN.
|
||||
*******************************************************************************/
|
||||
private static HealthCheckOutcome notTestable(CdnProviderConfig config)
|
||||
{
|
||||
boolean none = config.type() == CdnProviderType.NONE;
|
||||
return (HealthCheckOutcome.up(none ? "No CDN configured; purges are recorded only" : "CDN provider configured; not testable (no connection test)")
|
||||
.withDetails(new HealthDetails().withValue("providerType", config.type().name()).withValue("testable", false)
|
||||
.withValue("reason", none ? "no CDN provider" : "provider type has no connection test")));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Map an adapter health probe. Its contract (CdnHealth) is fixed messages
|
||||
** and safe details only.
|
||||
*******************************************************************************/
|
||||
private static HealthCheckOutcome fromProbe(CdnHealth health, CdnProviderConfig config)
|
||||
{
|
||||
HealthCheckOutcome outcome = switch(health.state())
|
||||
{
|
||||
case UP -> HealthCheckOutcome.up(health.message());
|
||||
case DEGRADED -> HealthCheckOutcome.degraded(health.message());
|
||||
case DOWN -> HealthCheckOutcome.down(health.message());
|
||||
};
|
||||
HealthDetails details = new HealthDetails().withValue("providerType", config.type().name());
|
||||
health.details().forEach(details::withValue);
|
||||
return (outcome.withDetails(details));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Map a provider-neutral connection test.
|
||||
*******************************************************************************/
|
||||
private static HealthCheckOutcome fromConnectionTest(CdnOperations.ConnectionTest test, CdnProviderConfig config)
|
||||
{
|
||||
HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed")
|
||||
: !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured")
|
||||
: HealthCheckOutcome.up("CDN connection test passed");
|
||||
// Only known provider states are safe; arbitrary status text may contain secrets.
|
||||
String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN";
|
||||
return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name())
|
||||
.withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured())));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The SPI has no timeout parameter. Bound its wait by the remaining
|
||||
** deadline and interrupt the probe on timeout or runner cancellation.
|
||||
*******************************************************************************/
|
||||
private static HealthCheckOutcome probe(CdnOperations operations, CdnProviderConfig config, HealthCheckInput input)
|
||||
private static HealthCheckOutcome bounded(Callable<HealthCheckOutcome> probe, HealthCheckInput input)
|
||||
{
|
||||
long remaining = Duration.between(Instant.now(), input.getDeadline()).toNanos();
|
||||
if(remaining <= 0)
|
||||
{
|
||||
return (HealthCheckOutcome.down("CDN connection test timed out"));
|
||||
}
|
||||
FutureTask<CdnOperations.ConnectionTest> task = new FutureTask<>(() -> operations.connectionTest(config));
|
||||
FutureTask<HealthCheckOutcome> task = new FutureTask<>(probe);
|
||||
Thread.startVirtualThread(task);
|
||||
try
|
||||
{
|
||||
CdnOperations.ConnectionTest test = task.get(remaining, TimeUnit.NANOSECONDS);
|
||||
HealthCheckOutcome outcome = !test.ok() ? HealthCheckOutcome.down("CDN connection test failed")
|
||||
: !test.originHeaderConfigured() ? HealthCheckOutcome.degraded("CDN origin header is not configured")
|
||||
: HealthCheckOutcome.up("CDN connection test passed");
|
||||
// Only known provider states are safe; arbitrary status text may contain secrets.
|
||||
String status = "Deployed".equals(test.status()) ? "Deployed" : "InProgress".equals(test.status()) ? "InProgress" : "UNKNOWN";
|
||||
return (outcome.withDetails(new HealthDetails().withValue("providerType", config.type().name())
|
||||
.withValue("status", status).withValue("originHeaderConfigured", test.originHeaderConfigured())));
|
||||
return (task.get(remaining, TimeUnit.NANOSECONDS));
|
||||
}
|
||||
catch(TimeoutException e)
|
||||
{
|
||||
|
||||
+209
-2
@@ -22,14 +22,18 @@ import java.time.LocalDate;
|
||||
import java.time.ZoneOffset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.OptionalDouble;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnCapability;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnEdgeStat;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnException;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnHeaderDialect;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnHealth;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnHealthProbe;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderInterface;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
@@ -41,6 +45,8 @@ import software.amazon.awssdk.core.exception.SdkException;
|
||||
import software.amazon.awssdk.services.cloudfront.CloudFrontClient;
|
||||
import software.amazon.awssdk.services.cloudfront.model.CreateInvalidationResponse;
|
||||
import software.amazon.awssdk.services.cloudfront.model.Distribution;
|
||||
import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigRequest;
|
||||
import software.amazon.awssdk.services.cloudfront.model.GetDistributionRequest;
|
||||
import software.amazon.awssdk.services.cloudfront.model.GetInvalidationResponse;
|
||||
import software.amazon.awssdk.services.cloudwatch.CloudWatchClient;
|
||||
import software.amazon.awssdk.services.cloudwatch.model.Dimension;
|
||||
@@ -70,7 +76,7 @@ import software.amazon.awssdk.services.cloudwatch.model.ScanBy;
|
||||
** {@link #testConnection}, {@link #importStats} and {@link #readConfig}
|
||||
** (compare the result with {@link CloudFrontDriftCheck#compare}).</p>
|
||||
*******************************************************************************/
|
||||
public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsrook.qbits.cdn.spi.CdnOperations
|
||||
public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsrook.qbits.cdn.spi.CdnOperations, CdnHealthProbe
|
||||
{
|
||||
/** Default origin custom header that proves a request came through CloudFront. */
|
||||
public static final String DEFAULT_ORIGIN_VERIFY_HEADER = "X-Origin-Verify";
|
||||
@@ -81,6 +87,22 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo
|
||||
static final String STATUS_COMPLETED = "Completed";
|
||||
static final int SECONDS_PER_DAY = 86_400;
|
||||
|
||||
/** Distribution states CloudFront reports (anything else is reported as UNKNOWN). */
|
||||
static final String STATUS_DEPLOYED = "Deployed";
|
||||
static final String STATUS_IN_PROGRESS = "InProgress";
|
||||
|
||||
/** 5xx error rate (percent, CloudWatch 5xxErrorRate) above which the health check is DEGRADED. */
|
||||
static final double ERROR_RATE_5XX_DEGRADED_PERCENT = 5.0;
|
||||
|
||||
/** How far back the health check averages the 5xx error rate. */
|
||||
static final int ERROR_RATE_WINDOW_MINUTES = 30;
|
||||
|
||||
/** CloudWatch period for the health check's 5xx error rate. */
|
||||
static final int ERROR_RATE_PERIOD_SECONDS = 300;
|
||||
|
||||
/** Health message when the credentials cannot read the distribution status. */
|
||||
static final String NOT_TESTABLE_MESSAGE = "CloudFront configured; distribution status not testable with current permissions";
|
||||
|
||||
private static final Set<String> RETRYABLE_CODES = Set.of(
|
||||
"TooManyInvalidationsInProgress", "Throttling", "ThrottlingException", "RequestLimitExceeded", "ServiceUnavailable", "InternalError");
|
||||
|
||||
@@ -278,6 +300,181 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Central health probe. GetDistribution gives existence, Enabled and the
|
||||
** deployment Status: UP when enabled and Deployed, DEGRADED while
|
||||
** InProgress, DOWN when missing or disabled. The section 7.5 IAM policy
|
||||
** grants GetDistributionConfig but not GetDistribution, so on AccessDenied
|
||||
** this falls back to GetDistributionConfig (existence and Enabled, no
|
||||
** Status) and reports {@code testable=false}. If neither read is permitted
|
||||
** it reports UP with {@code testable=false}: UNKNOWN never alerts anyone,
|
||||
** and failed invalidations still show up as FAILED/DEAD purge requests.
|
||||
** The CloudWatch 5xxErrorRate over the last
|
||||
** {@value #ERROR_RATE_WINDOW_MINUTES} minutes is added when readable and
|
||||
** makes the check DEGRADED above {@value #ERROR_RATE_5XX_DEGRADED_PERCENT}
|
||||
** percent. Messages are fixed text; no AWS message, id or hostname is
|
||||
** copied. Never throws.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public CdnHealth checkHealth(CdnProviderConfig config)
|
||||
{
|
||||
String headerName = config.settings().getOrDefault(SETTING_ORIGIN_VERIFY_HEADER, DEFAULT_ORIGIN_VERIFY_HEADER);
|
||||
String distributionId;
|
||||
try
|
||||
{
|
||||
distributionId = distributionId(config);
|
||||
}
|
||||
catch(CdnException e)
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution id is not configured", null);
|
||||
}
|
||||
|
||||
Map<String, Object> details = new LinkedHashMap<>();
|
||||
CloudFrontConfigSnapshot snapshot;
|
||||
String status = null;
|
||||
try(CloudFrontClient client = clientFactory.cloudFront(config))
|
||||
{
|
||||
try
|
||||
{
|
||||
Distribution distribution = client.getDistribution(GetDistributionRequest.builder().id(distributionId).build()).distribution();
|
||||
status = distribution.status();
|
||||
snapshot = CloudFrontConfigSnapshot.from(distribution.distributionConfig());
|
||||
}
|
||||
catch(AwsServiceException e)
|
||||
{
|
||||
if(!isAccessDenied(e))
|
||||
{
|
||||
return apiFailure(e);
|
||||
}
|
||||
details.put("testable", false);
|
||||
try
|
||||
{
|
||||
snapshot = CloudFrontConfigSnapshot.from(client.getDistributionConfig(GetDistributionConfigRequest.builder().id(distributionId).build()).distributionConfig());
|
||||
details.put("reason", "credentials lack cloudfront:GetDistribution");
|
||||
}
|
||||
catch(AwsServiceException e2)
|
||||
{
|
||||
if(!isAccessDenied(e2))
|
||||
{
|
||||
return apiFailure(e2);
|
||||
}
|
||||
details.put("reason", "credentials lack cloudfront:GetDistribution and cloudfront:GetDistributionConfig");
|
||||
return new CdnHealth(CdnHealth.State.UP, NOT_TESTABLE_MESSAGE, details);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(CdnException e)
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DOWN, "CloudFront credentials are not configured", null);
|
||||
}
|
||||
catch(SdkClientException e)
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront API unreachable; distribution status not checked", Map.of("transient", true));
|
||||
}
|
||||
catch(SdkException e)
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DOWN, "CloudFront status check failed", null);
|
||||
}
|
||||
|
||||
boolean statusTestable = !details.containsKey("testable");
|
||||
boolean headerConfigured = snapshot.hasOriginHeader(headerName);
|
||||
details.put("enabled", snapshot.enabled());
|
||||
details.put("status", STATUS_DEPLOYED.equals(status) || STATUS_IN_PROGRESS.equals(status) ? status : "UNKNOWN");
|
||||
details.put("originHeaderConfigured", headerConfigured);
|
||||
if(!snapshot.enabled())
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution is disabled", details);
|
||||
}
|
||||
|
||||
Double errorRate = errorRate5xx(config, distributionId);
|
||||
details.put("errorRateWindowMinutes", ERROR_RATE_WINDOW_MINUTES);
|
||||
if(errorRate != null)
|
||||
{
|
||||
details.put("errorRate5xxPercent", Math.round(errorRate * 100.0) / 100.0);
|
||||
}
|
||||
|
||||
if(STATUS_IN_PROGRESS.equals(status))
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront distribution is deploying a change (InProgress)", details);
|
||||
}
|
||||
if(!headerConfigured)
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DEGRADED, "CDN origin header is not configured", details);
|
||||
}
|
||||
if(errorRate != null && errorRate > ERROR_RATE_5XX_DEGRADED_PERCENT)
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront 5xx error rate is above 5% over the last 30 minutes", details);
|
||||
}
|
||||
if(!statusTestable)
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.UP, NOT_TESTABLE_MESSAGE, details);
|
||||
}
|
||||
return new CdnHealth(CdnHealth.State.UP, "CloudFront distribution is enabled and deployed", details);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Whether an AWS error is a permissions refusal.
|
||||
*******************************************************************************/
|
||||
private static boolean isAccessDenied(AwsServiceException e)
|
||||
{
|
||||
String code = e.awsErrorDetails() == null ? null : e.awsErrorDetails().errorCode();
|
||||
return "AccessDenied".equals(code) || "AccessDeniedException".equals(code) || (code == null && e.statusCode() == 403);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Health for an AWS error other than AccessDenied: a missing distribution is
|
||||
** DOWN, throttling and AWS-side errors are DEGRADED (transient), anything
|
||||
** else (rejected credentials, bad id) is DOWN.
|
||||
*******************************************************************************/
|
||||
private static CdnHealth apiFailure(AwsServiceException e)
|
||||
{
|
||||
String code = e.awsErrorDetails() == null ? null : e.awsErrorDetails().errorCode();
|
||||
if("NoSuchDistribution".equals(code))
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DOWN, "CloudFront distribution not found", null);
|
||||
}
|
||||
if(toCdnException("GetDistribution", e).isRetryable())
|
||||
{
|
||||
return new CdnHealth(CdnHealth.State.DEGRADED, "CloudFront API throttled or unavailable; distribution status not checked", Map.of("transient", true));
|
||||
}
|
||||
return new CdnHealth(CdnHealth.State.DOWN, "CloudFront rejected the status check", null);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Average CloudWatch 5xxErrorRate (percent) over the health window, or null
|
||||
** when there is no traffic or the metric cannot be read. Best effort: it
|
||||
** never fails the health check.
|
||||
*******************************************************************************/
|
||||
private Double errorRate5xx(CdnProviderConfig config, String distributionId)
|
||||
{
|
||||
Instant end = Instant.now();
|
||||
try(CloudWatchClient client = clientFactory.cloudWatch(config))
|
||||
{
|
||||
GetMetricDataResponse response = client.getMetricData(GetMetricDataRequest.builder()
|
||||
.metricDataQueries(query("err5xx", "5xxErrorRate", "Average", distributionId, ERROR_RATE_PERIOD_SECONDS))
|
||||
.startTime(end.minusSeconds(ERROR_RATE_WINDOW_MINUTES * 60L))
|
||||
.endTime(end)
|
||||
.build());
|
||||
OptionalDouble average = response.metricDataResults().stream()
|
||||
.flatMap(result -> result.values().stream())
|
||||
.mapToDouble(Double::doubleValue)
|
||||
.average();
|
||||
return average.isPresent() ? average.getAsDouble() : null;
|
||||
}
|
||||
catch(CdnException | SdkException e)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The distribution config facts used for drift checks (GetDistributionConfig).
|
||||
*******************************************************************************/
|
||||
@@ -368,6 +565,16 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo
|
||||
** One daily CloudFront metric query.
|
||||
*******************************************************************************/
|
||||
private static MetricDataQuery query(String id, String metricName, String stat, String distributionId)
|
||||
{
|
||||
return query(id, metricName, stat, distributionId, SECONDS_PER_DAY);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** One CloudFront metric query with the given period (seconds).
|
||||
*******************************************************************************/
|
||||
private static MetricDataQuery query(String id, String metricName, String stat, String distributionId, int periodSeconds)
|
||||
{
|
||||
Metric metric = Metric.builder()
|
||||
.namespace("AWS/CloudFront")
|
||||
@@ -377,7 +584,7 @@ public class CloudFrontCdnProvider implements CdnProviderInterface, com.kingsroo
|
||||
.build();
|
||||
return MetricDataQuery.builder()
|
||||
.id(id)
|
||||
.metricStat(MetricStat.builder().metric(metric).period(SECONDS_PER_DAY).stat(stat).build())
|
||||
.metricStat(MetricStat.builder().metric(metric).period(periodSeconds).stat(stat).build())
|
||||
.returnData(true)
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.spi;
|
||||
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Result of an adapter's own health probe (see {@link CdnHealthProbe}). The
|
||||
** message and every detail value must be fixed text, numbers, booleans or
|
||||
** known provider states: never provider-supplied free text, hostnames, ids
|
||||
** or credentials, because they are copied to the central health tables.
|
||||
**
|
||||
** @param state UP, DEGRADED or DOWN
|
||||
** @param message a short fixed summary
|
||||
** @param details safe facts (may be empty, never null)
|
||||
*******************************************************************************/
|
||||
public record CdnHealth(State state, String message, Map<String, Object> details)
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Health as the adapter sees it.
|
||||
*******************************************************************************/
|
||||
public enum State
|
||||
{
|
||||
UP,
|
||||
DEGRADED,
|
||||
DOWN
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Null details become an empty map.
|
||||
*******************************************************************************/
|
||||
public CdnHealth
|
||||
{
|
||||
details = details == null ? Map.of() : Map.copyOf(details);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.spi;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Optional: an adapter that can say whether its CDN is healthy, beyond the
|
||||
** "Test connection" action ({@link CdnOperations}). The central health check
|
||||
** prefers this when an adapter implements it.
|
||||
*******************************************************************************/
|
||||
public interface CdnHealthProbe
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Probe the CDN read-only. Never throws: failures come back as DOWN or
|
||||
** DEGRADED, with fixed messages (see {@link CdnHealth}).
|
||||
*******************************************************************************/
|
||||
CdnHealth checkHealth(CdnProviderConfig config);
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
version=${project.version}
|
||||
@@ -65,7 +65,7 @@ class CdnHelpBundleTest
|
||||
assertThat(instance.getTable("helpContent")).isNull();
|
||||
var bundle = HelpBundles.of(instance).getBundles().get(0);
|
||||
assertThat(bundle.getId()).isEqualTo("qbit-cdn");
|
||||
assertThat(bundle.getVersion()).isEqualTo("0.1.0-SNAPSHOT");
|
||||
assertThat(bundle.getVersion()).as("the help bundle carries the artifact version").isEqualTo(System.getProperty("qbit.artifactVersion")).isEqualTo(CdnQBitProducer.VERSION);
|
||||
assertThat(bundle.getLayer()).isEqualTo(HelpBundleLayer.QBIT);
|
||||
assertThat(bundle.getEntries()).extracting(entry -> entry.getKey()).doesNotHaveDuplicates();
|
||||
// Core's gate does not require widgets or possible-value sources yet.
|
||||
|
||||
@@ -34,6 +34,18 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
|
||||
*******************************************************************************/
|
||||
class CdnQBitProducerTest
|
||||
{
|
||||
/*******************************************************************************
|
||||
** Metadata must report the version of the artifact being built.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void metadataVersionMatchesArtifact()
|
||||
{
|
||||
assertThat(System.getProperty("qbit.artifactVersion")).isNotBlank().isEqualTo(CdnQBitProducer.VERSION);
|
||||
assertThat(new CdnQBitProducer().getQBitMetaData().getVersion()).isEqualTo(System.getProperty("qbit.artifactVersion"));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Everything is produced and the instance is valid.
|
||||
*******************************************************************************/
|
||||
|
||||
@@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit;
|
||||
import com.kingsrook.qbits.cdn.CdnRuntime;
|
||||
import com.kingsrook.qbits.cdn.CdnTestSupport;
|
||||
import com.kingsrook.qbits.cdn.model.CdnTables;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnHealth;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnHealthProbe;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnOperations;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
@@ -168,7 +170,14 @@ class CdnProviderHealthChecksTest
|
||||
{
|
||||
var check = check().getCheckInstance();
|
||||
update(new QRecord().withValue("providerType", "NONE"));
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
HealthCheckOutcome none = check.run(input());
|
||||
assertThat(none.getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(none.getMessage()).isEqualTo("No CDN configured; purges are recorded only");
|
||||
assertThat(none.getDetails().getValues()).containsEntry("testable", false).containsEntry("providerType", "NONE");
|
||||
update(new QRecord().withValue("providerType", "SOUIN"));
|
||||
HealthCheckOutcome souin = check.run(input());
|
||||
assertThat(souin.getStatus()).isEqualTo(HealthStatus.UP);
|
||||
assertThat(souin.getDetails().getValues()).containsEntry("testable", false).containsEntry("reason", "provider type has no connection test");
|
||||
update(new QRecord().withValue("enabled", false));
|
||||
assertThat(check.run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN);
|
||||
new DeleteAction().execute(new DeleteInput(CdnTables.PROVIDER).withPrimaryKeys(List.of(providerId)));
|
||||
@@ -178,6 +187,63 @@ class CdnProviderHealthChecksTest
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An adapter's own health probe wins over the connection test; its state,
|
||||
** message and details pass through with the provider type added.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void healthProbePreferred() throws Exception
|
||||
{
|
||||
ProbingProvider probing = new ProbingProvider();
|
||||
CdnRuntime.registerProvider(CdnProviderType.CLOUDFRONT, () -> probing);
|
||||
var check = check().getCheckInstance();
|
||||
for(CdnHealth.State state : CdnHealth.State.values())
|
||||
{
|
||||
probing.health = new CdnHealth(state, "probe " + state, Map.of("testable", false));
|
||||
HealthCheckOutcome outcome = check.run(input());
|
||||
assertThat(outcome.getStatus()).isEqualTo(HealthStatus.valueOf(state.name()));
|
||||
assertThat(outcome.getMessage()).isEqualTo("probe " + state);
|
||||
assertThat(outcome.getDetails().getValues()).containsExactlyInAnyOrderEntriesOf(Map.of("providerType", "CLOUDFRONT", "testable", false));
|
||||
}
|
||||
assertThat(probing.connectionTests).isZero();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The fake path adapter plus a health probe.
|
||||
*******************************************************************************/
|
||||
private static class ProbingProvider extends CdnTestSupport.FakePathProvider implements CdnHealthProbe
|
||||
{
|
||||
private CdnHealth health;
|
||||
private int connectionTests;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The configured answer.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public CdnHealth checkHealth(CdnProviderConfig config)
|
||||
{
|
||||
return (health);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Counted so the test can prove it is not used.
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ConnectionTest connectionTest(CdnProviderConfig config)
|
||||
{
|
||||
connectionTests++;
|
||||
return (super.connectionTest(config));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Poison credentials, every free-form probe field and thrown error messages.
|
||||
** HealthRunner.sanitize is package-private, so assert the raw boundary here.
|
||||
|
||||
@@ -0,0 +1,394 @@
|
||||
/*
|
||||
* Copyright 2026 Kingsrook, LLC
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.cdn.providers.cloudfront;
|
||||
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnHealth;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderConfig;
|
||||
import com.kingsrook.qbits.cdn.spi.CdnProviderType;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import software.amazon.awssdk.awscore.exception.AwsErrorDetails;
|
||||
import software.amazon.awssdk.awscore.exception.AwsServiceException;
|
||||
import software.amazon.awssdk.core.exception.SdkClientException;
|
||||
import software.amazon.awssdk.core.exception.SdkException;
|
||||
import software.amazon.awssdk.services.cloudfront.CloudFrontClient;
|
||||
import software.amazon.awssdk.services.cloudfront.model.CloudFrontException;
|
||||
import software.amazon.awssdk.services.cloudfront.model.CustomHeaders;
|
||||
import software.amazon.awssdk.services.cloudfront.model.DefaultCacheBehavior;
|
||||
import software.amazon.awssdk.services.cloudfront.model.Distribution;
|
||||
import software.amazon.awssdk.services.cloudfront.model.DistributionConfig;
|
||||
import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigRequest;
|
||||
import software.amazon.awssdk.services.cloudfront.model.GetDistributionConfigResponse;
|
||||
import software.amazon.awssdk.services.cloudfront.model.GetDistributionRequest;
|
||||
import software.amazon.awssdk.services.cloudfront.model.GetDistributionResponse;
|
||||
import software.amazon.awssdk.services.cloudfront.model.Origin;
|
||||
import software.amazon.awssdk.services.cloudfront.model.OriginCustomHeader;
|
||||
import software.amazon.awssdk.services.cloudfront.model.Origins;
|
||||
import software.amazon.awssdk.services.cloudwatch.CloudWatchClient;
|
||||
import software.amazon.awssdk.services.cloudwatch.model.CloudWatchException;
|
||||
import software.amazon.awssdk.services.cloudwatch.model.Dimension;
|
||||
import software.amazon.awssdk.services.cloudwatch.model.GetMetricDataRequest;
|
||||
import software.amazon.awssdk.services.cloudwatch.model.GetMetricDataResponse;
|
||||
import software.amazon.awssdk.services.cloudwatch.model.MetricDataResult;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The CloudFront health probe against mocked CloudFront and CloudWatch
|
||||
** clients (no network).
|
||||
*******************************************************************************/
|
||||
class CloudFrontHealthTest
|
||||
{
|
||||
private static final String DISTRIBUTION = "E2EXAMPLE123";
|
||||
private static final String SECRET = "FIXTURE-SECRET-NEVER-LEAK";
|
||||
private static final CdnProviderConfig CONFIG = new CdnProviderConfig("kof22-cloudfront", CdnProviderType.CLOUDFRONT, DISTRIBUTION, "CDN_AWS", null);
|
||||
|
||||
private CloudFrontClient cloudFront;
|
||||
private CloudWatchClient cloudWatch;
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Fresh mocks; by default CloudWatch has no datapoints.
|
||||
*******************************************************************************/
|
||||
@BeforeEach
|
||||
void mocks()
|
||||
{
|
||||
cloudFront = mock(CloudFrontClient.class);
|
||||
cloudWatch = mock(CloudWatchClient.class);
|
||||
errorRate();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** An adapter whose clients are the mocks.
|
||||
*******************************************************************************/
|
||||
private CloudFrontCdnProvider provider()
|
||||
{
|
||||
return new CloudFrontCdnProvider(new CloudFrontClientFactory(name -> null, null, null)
|
||||
{
|
||||
/***************************************************************************
|
||||
** The CloudFront mock.
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public CloudFrontClient cloudFront(CdnProviderConfig config)
|
||||
{
|
||||
return cloudFront;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/***************************************************************************
|
||||
** The CloudWatch mock.
|
||||
***************************************************************************/
|
||||
@Override
|
||||
public CloudWatchClient cloudWatch(CdnProviderConfig config)
|
||||
{
|
||||
return cloudWatch;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A distribution config, enabled or not, whose origin sends the verify header.
|
||||
*******************************************************************************/
|
||||
private static DistributionConfig distributionConfig(boolean enabled)
|
||||
{
|
||||
Origin origin = Origin.builder().id("origin").domainName("origin.kof22.com")
|
||||
.customHeaders(CustomHeaders.builder().quantity(1).items(OriginCustomHeader.builder().headerName("X-Origin-Verify").headerValue(SECRET).build()).build())
|
||||
.build();
|
||||
return DistributionConfig.builder().enabled(enabled)
|
||||
.origins(Origins.builder().quantity(1).items(origin).build())
|
||||
.defaultCacheBehavior(DefaultCacheBehavior.builder().targetOriginId("origin").build())
|
||||
.build();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** GetDistribution answers with this status and enabled flag.
|
||||
*******************************************************************************/
|
||||
private void distribution(String status, boolean enabled)
|
||||
{
|
||||
when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenReturn(GetDistributionResponse.builder()
|
||||
.distribution(Distribution.builder().id(DISTRIBUTION).status(status).domainName("d1.cloudfront.net").distributionConfig(distributionConfig(enabled)).build())
|
||||
.build());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** CloudWatch answers with these 5xxErrorRate datapoints.
|
||||
*******************************************************************************/
|
||||
private void errorRate(Double... values)
|
||||
{
|
||||
when(cloudWatch.getMetricData(any(GetMetricDataRequest.class))).thenReturn(GetMetricDataResponse.builder()
|
||||
.metricDataResults(MetricDataResult.builder().id("err5xx").values(values).build()).build());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A CloudFront service error.
|
||||
*******************************************************************************/
|
||||
private static AwsServiceException error(String code, int status)
|
||||
{
|
||||
return CloudFrontException.builder().statusCode(status)
|
||||
.awsErrorDetails(AwsErrorDetails.builder().errorCode(code).errorMessage(SECRET).build()).build();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Enabled and Deployed is UP; the 5xx rate is averaged into the details from
|
||||
** a five-minute Average query on the distribution's global metric.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void deployedIsUp()
|
||||
{
|
||||
distribution("Deployed", true);
|
||||
errorRate(1.0, 2.0);
|
||||
|
||||
CdnHealth health = provider().checkHealth(CONFIG);
|
||||
|
||||
assertThat(health.state()).isEqualTo(CdnHealth.State.UP);
|
||||
assertThat(health.message()).isEqualTo("CloudFront distribution is enabled and deployed");
|
||||
assertThat(health.details()).containsExactlyInAnyOrderEntriesOf(Map.of("enabled", true, "status", "Deployed", "originHeaderConfigured", true,
|
||||
"errorRateWindowMinutes", 30, "errorRate5xxPercent", 1.5));
|
||||
ArgumentCaptor<GetMetricDataRequest> request = ArgumentCaptor.forClass(GetMetricDataRequest.class);
|
||||
verify(cloudWatch).getMetricData(request.capture());
|
||||
var stat = request.getValue().metricDataQueries().getFirst().metricStat();
|
||||
assertThat(stat.metric().metricName()).isEqualTo("5xxErrorRate");
|
||||
assertThat(stat.metric().namespace()).isEqualTo("AWS/CloudFront");
|
||||
assertThat(stat.metric().dimensions()).containsExactly(Dimension.builder().name("DistributionId").value(DISTRIBUTION).build(),
|
||||
Dimension.builder().name("Region").value("Global").build());
|
||||
assertThat(stat.stat()).isEqualTo("Average");
|
||||
assertThat(stat.period()).isEqualTo(300);
|
||||
assertThat(java.time.Duration.between(request.getValue().startTime(), request.getValue().endTime()).toMinutes()).isEqualTo(30);
|
||||
verify(cloudFront, never()).getDistributionConfig(any(GetDistributionConfigRequest.class));
|
||||
assertClean(health);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** No traffic (no datapoints) or an unreadable metric leaves the rate out.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void errorRateIsBestEffort()
|
||||
{
|
||||
distribution("Deployed", true);
|
||||
assertThat(provider().checkHealth(CONFIG).details()).doesNotContainKey("errorRate5xxPercent").containsEntry("errorRateWindowMinutes", 30);
|
||||
|
||||
when(cloudWatch.getMetricData(any(GetMetricDataRequest.class))).thenThrow(CloudWatchException.builder().statusCode(403)
|
||||
.awsErrorDetails(AwsErrorDetails.builder().errorCode("AccessDenied").build()).build());
|
||||
CdnHealth health = provider().checkHealth(CONFIG);
|
||||
assertThat(health.state()).isEqualTo(CdnHealth.State.UP);
|
||||
assertThat(health.details()).doesNotContainKey("errorRate5xxPercent");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A 5xx rate above 5% is DEGRADED; at 5% it is still UP.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void highErrorRateIsDegraded()
|
||||
{
|
||||
distribution("Deployed", true);
|
||||
errorRate(5.0);
|
||||
assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(CdnHealth.State.UP);
|
||||
|
||||
errorRate(4.0, 9.0);
|
||||
CdnHealth health = provider().checkHealth(CONFIG);
|
||||
assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED);
|
||||
assertThat(health.message()).contains("5xx error rate");
|
||||
assertThat(health.details()).containsEntry("errorRate5xxPercent", 6.5);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A deploying distribution is DEGRADED and says so.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void inProgressIsDegraded()
|
||||
{
|
||||
distribution("InProgress", true);
|
||||
CdnHealth health = provider().checkHealth(CONFIG);
|
||||
assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED);
|
||||
assertThat(health.message()).contains("deploying").contains("InProgress");
|
||||
assertThat(health.details()).containsEntry("status", "InProgress");
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A disabled distribution is DOWN (and CloudWatch is not consulted); an
|
||||
** unexpected status value is reported as UNKNOWN, never echoed.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void disabledIsDown()
|
||||
{
|
||||
distribution(SECRET, false);
|
||||
CdnHealth health = provider().checkHealth(CONFIG);
|
||||
assertThat(health.state()).isEqualTo(CdnHealth.State.DOWN);
|
||||
assertThat(health.message()).isEqualTo("CloudFront distribution is disabled");
|
||||
assertThat(health.details()).containsEntry("enabled", false).containsEntry("status", "UNKNOWN");
|
||||
verify(cloudWatch, never()).getMetricData(any(GetMetricDataRequest.class));
|
||||
assertClean(health);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A different configured header name is reported missing: DEGRADED.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void missingOriginHeaderIsDegraded()
|
||||
{
|
||||
distribution("Deployed", true);
|
||||
CdnProviderConfig other = new CdnProviderConfig("x", CdnProviderType.CLOUDFRONT, DISTRIBUTION, "CDN_AWS",
|
||||
Map.of(CloudFrontCdnProvider.SETTING_ORIGIN_VERIFY_HEADER, "X-Other-Verify"));
|
||||
CdnHealth health = provider().checkHealth(other);
|
||||
assertThat(health.state()).isEqualTo(CdnHealth.State.DEGRADED);
|
||||
assertThat(health.details()).containsEntry("originHeaderConfigured", false);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Without GetDistribution (the section 7.5 policy) the probe falls back to
|
||||
** GetDistributionConfig: UP, not testable, Enabled still checked.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void accessDeniedFallsBackToConfig()
|
||||
{
|
||||
when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403));
|
||||
when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class)))
|
||||
.thenReturn(GetDistributionConfigResponse.builder().distributionConfig(distributionConfig(true)).build());
|
||||
|
||||
CdnHealth health = provider().checkHealth(CONFIG);
|
||||
assertThat(health.state()).isEqualTo(CdnHealth.State.UP);
|
||||
assertThat(health.message()).isEqualTo("CloudFront configured; distribution status not testable with current permissions");
|
||||
assertThat(health.details()).containsEntry("testable", false).containsEntry("reason", "credentials lack cloudfront:GetDistribution")
|
||||
.containsEntry("enabled", true).containsEntry("status", "UNKNOWN");
|
||||
assertClean(health);
|
||||
|
||||
when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class)))
|
||||
.thenReturn(GetDistributionConfigResponse.builder().distributionConfig(distributionConfig(false)).build());
|
||||
assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(CdnHealth.State.DOWN);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** When neither read is permitted the check is UP and not testable.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void noReadPermissionIsUpNotTestable()
|
||||
{
|
||||
when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403));
|
||||
when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))).thenThrow(error(null, 403));
|
||||
|
||||
CdnHealth health = provider().checkHealth(CONFIG);
|
||||
assertThat(health.state()).isEqualTo(CdnHealth.State.UP);
|
||||
assertThat(health.details()).containsExactlyInAnyOrderEntriesOf(Map.of("testable", false,
|
||||
"reason", "credentials lack cloudfront:GetDistribution and cloudfront:GetDistributionConfig"));
|
||||
verify(cloudWatch, never()).getMetricData(any(GetMetricDataRequest.class));
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** AWS errors: missing distribution and rejected credentials are DOWN,
|
||||
** throttling and AWS-side errors DEGRADED; also after the config fallback.
|
||||
*******************************************************************************/
|
||||
@ParameterizedTest
|
||||
@CsvSource({
|
||||
"NoSuchDistribution, 404, DOWN, CloudFront distribution not found",
|
||||
"InvalidClientTokenId, 403, DOWN, CloudFront rejected the status check",
|
||||
"Throttling, 400, DEGRADED, CloudFront API throttled or unavailable; distribution status not checked",
|
||||
"ServiceUnavailable, 503, DEGRADED, CloudFront API throttled or unavailable; distribution status not checked",
|
||||
})
|
||||
void apiErrors(String code, int status, CdnHealth.State expected, String message)
|
||||
{
|
||||
when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error(code, status));
|
||||
CdnHealth health = provider().checkHealth(CONFIG);
|
||||
assertThat(health.state()).isEqualTo(expected);
|
||||
assertThat(health.message()).isEqualTo(message);
|
||||
assertClean(health);
|
||||
|
||||
when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(error("AccessDenied", 403));
|
||||
when(cloudFront.getDistributionConfig(any(GetDistributionConfigRequest.class))).thenThrow(error(code, status));
|
||||
assertThat(provider().checkHealth(CONFIG).state()).isEqualTo(expected);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Network failures are DEGRADED (transient); missing credentials or
|
||||
** distribution id are DOWN; other SDK failures are DOWN.
|
||||
*******************************************************************************/
|
||||
@Test
|
||||
void clientAndConfigFailures()
|
||||
{
|
||||
when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(SdkClientException.create(SECRET));
|
||||
CdnHealth unreachable = provider().checkHealth(CONFIG);
|
||||
assertThat(unreachable.state()).isEqualTo(CdnHealth.State.DEGRADED);
|
||||
assertThat(unreachable.details()).containsEntry("transient", true);
|
||||
assertClean(unreachable);
|
||||
|
||||
CdnHealth noCredentials = new CloudFrontCdnProvider(new CloudFrontClientFactory(name -> null, null, null)).checkHealth(CONFIG);
|
||||
assertThat(noCredentials.state()).isEqualTo(CdnHealth.State.DOWN);
|
||||
assertThat(noCredentials.message()).isEqualTo("CloudFront credentials are not configured");
|
||||
|
||||
CdnHealth noId = provider().checkHealth(new CdnProviderConfig("x", CdnProviderType.CLOUDFRONT, " ", "CDN_AWS", null));
|
||||
assertThat(noId.state()).isEqualTo(CdnHealth.State.DOWN);
|
||||
assertThat(noId.message()).isEqualTo("CloudFront distribution id is not configured");
|
||||
|
||||
when(cloudFront.getDistribution(any(GetDistributionRequest.class))).thenThrow(SdkException.builder().message(SECRET).build());
|
||||
CdnHealth other = provider().checkHealth(CONFIG);
|
||||
assertThat(other.state()).isEqualTo(CdnHealth.State.DOWN);
|
||||
assertClean(other);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Nothing the AWS side controls as free text reaches the outcome.
|
||||
*******************************************************************************/
|
||||
private static void assertClean(CdnHealth health)
|
||||
{
|
||||
assertThat(health.message()).doesNotContain(SECRET, DISTRIBUTION, "cloudfront.net", "origin.kof22.com");
|
||||
assertThat(List.copyOf(health.details().values()).toString()).doesNotContain(SECRET, DISTRIBUTION, "cloudfront.net", "origin.kof22.com");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user