From 3cf073e2d53684cda1f3356d2b1b5ec5eb8374d3 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 14:55:11 -0500 Subject: [PATCH 1/3] feat(health): register webhook endpoint and dispatcher checks with core central health Co-Authored-By: Claude Opus 5.5 --- .../qbits/webhooks/WebhooksQBitProducer.java | 3 + .../webhooks/health/WebhooksHealthChecks.java | 230 +++++++++++ .../webhooks/health/WebhooksHealthWidget.java | 70 ++++ .../webhooks/WebhooksQBitProducerTest.java | 2 +- .../health/WebhooksHealthChecksTest.java | 366 ++++++++++++++++++ .../health/WebhooksHealthWidgetTest.java | 82 ++++ 6 files changed, 752 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthChecks.java create mode 100644 src/main/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidget.java create mode 100644 src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthChecksTest.java create mode 100644 src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidgetTest.java diff --git a/src/main/java/com/kingsrook/qbits/webhooks/WebhooksQBitProducer.java b/src/main/java/com/kingsrook/qbits/webhooks/WebhooksQBitProducer.java index 93bcdad..3b7f4d6 100644 --- a/src/main/java/com/kingsrook/qbits/webhooks/WebhooksQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/webhooks/WebhooksQBitProducer.java @@ -28,6 +28,7 @@ import java.util.Map; import com.kingsrook.qbits.webhooks.app.WebhooksApp; import com.kingsrook.qbits.webhooks.events.WebhooksProcessListener; import com.kingsrook.qbits.webhooks.events.WebhooksRecordChangeListener; +import com.kingsrook.qbits.webhooks.health.WebhooksHealthChecks; import com.kingsrook.qbits.webhooks.metadata.WebhooksTableCustomizers; import com.kingsrook.qbits.webhooks.metadata.WebhooksTables; import com.kingsrook.qbits.webhooks.processes.WebhooksProcesses; @@ -199,6 +200,8 @@ public class WebhooksQBitProducer implements QBitMetaDataProducer supply(QInstance qInstance) throws QException + { + List checks = new ArrayList<>(); + for(QRecord endpoint : SystemWrites.call(() -> new QueryAction().execute(new QueryInput(WebhooksTables.ENDPOINT) + .withFilter(new QQueryFilter(new QFilterCriteria("activeStatus", QCriteriaOperator.IN, ActiveStatus.ACTIVE.name(), ActiveStatus.DISABLED.name()))) + .withFieldNamesToInclude(Set.of("id", "activeStatus", "autoDisabledDate")).withTimeoutSeconds(5)).getRecords())) + { + if(monitored(endpoint)) + { + Integer id = endpoint.getValueInteger("id"); + checks.add(definition("webhooks.endpoint." + id.toString().toLowerCase(Locale.ROOT), input -> endpoint(input, id))); + } + } + return (checks); + } + + + + /******************************************************************************* + ** Common metadata and a boundary that never exposes backend error text. + *******************************************************************************/ + private static HealthCheckMetaData definition(String name, HealthCheck check) + { + return (new HealthCheckMetaData().withName(name).withLabel(name).withComponent("webhooks") + .withCategory(HealthCategory.INTEGRATION).withKind(HealthCheckKind.DEPENDENCY).withIntervalSeconds(300).withTimeoutMs(5000) + .withCheckInstance(input -> + { + if(expired(input)) + { + return (HealthCheckOutcome.down("Health check timed out")); + } + try + { + HealthCheckOutcome outcome = check.run(input); + return (expired(input) ? HealthCheckOutcome.down("Health check timed out") : outcome); + } + catch(Exception e) + { + return (HealthCheckOutcome.down("Unable to read webhook health")); + } + })); + } + + + + /******************************************************************************* + ** Read the latest EndpointHealth state, without probing or mutating it. + *******************************************************************************/ + private static HealthCheckOutcome endpoint(HealthCheckInput input, Integer id) throws QException + { + List records = query(input, WebhooksTables.ENDPOINT, + new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, id)).withLimit(1), + Set.of("id", "activeStatus", "autoDisabledDate", "healthStatus", "consecutiveFailures", "unhealthySinceDate")); + if(records.isEmpty() || !monitored(records.getFirst())) + { + return (HealthCheckOutcome.unknown("Endpoint is absent or disabled")); + } + QRecord endpoint = records.getFirst(); + String health = endpoint.getValueString("healthStatus"); + HealthCheckOutcome outcome; + if(ActiveStatus.DISABLED.name().equals(endpoint.getValueString("activeStatus"))) + { + outcome = HealthCheckOutcome.down("Endpoint automatically disabled"); + } + else if(HealthStatus.HEALTHY.name().equals(health)) + { + outcome = HealthCheckOutcome.up("Endpoint healthy"); + } + else if(HealthStatus.UNHEALTHY.name().equals(health) || HealthStatus.PROBATION.name().equals(health)) + { + outcome = HealthCheckOutcome.degraded("Endpoint unhealthy or on probation"); + } + else + { + return (HealthCheckOutcome.unknown("Endpoint health is unavailable")); + } + // Only known state names may enter details, even if persisted data is corrupt. + String safeHealth = List.of("HEALTHY", "UNHEALTHY", "PROBATION").contains(health == null ? "" : health) ? health : "UNKNOWN"; + Integer failures = endpoint.getValueInteger("consecutiveFailures"); + HealthDetails details = new HealthDetails().withValue("healthStatus", safeHealth) + .withValue("consecutiveFailures", failures == null ? 0 : failures); + Instant unhealthySince = endpoint.getValueInstant("unhealthySinceDate"); + if(unhealthySince != null) + { + details.withValue("unhealthySince", unhealthySince); + } + return (outcome.withDetails(details)); + } + + + + /******************************************************************************* + ** Age of the oldest queued delivery, including retries and in-flight work. + *******************************************************************************/ + private static HealthCheckOutcome dispatcher(HealthCheckInput input) throws QException + { + List records = query(input, WebhooksTables.DELIVERY, + new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.IN, + DeliveryStatus.PENDING.name(), DeliveryStatus.RETRY_WAIT.name(), DeliveryStatus.SENDING.name())) + .withOrderBy(new QFilterOrderBy("createDate", true)).withLimit(1), Set.of("createDate")); + long seconds = 0; + if(!records.isEmpty()) + { + Instant queued = records.getFirst().getValueInstant("createDate"); + if(queued == null) + { + return (HealthCheckOutcome.unknown("Pending delivery age is unavailable")); + } + seconds = Math.max(0, Duration.between(queued, WebhooksRuntime.clock().instant()).getSeconds()); + } + HealthCheckOutcome outcome = seconds > 3600 ? HealthCheckOutcome.down("Delivery backlog exceeds one hour") + : seconds > 900 ? HealthCheckOutcome.degraded("Delivery backlog exceeds fifteen minutes") : HealthCheckOutcome.up("Delivery backlog within threshold"); + return (outcome.withMetrics(List.of(new HealthMetric().withName("oldestPendingSeconds").withValue(seconds).withUnit("seconds")))); + } + + + + /******************************************************************************* + ** Only active or automatically disabled endpoints participate. + *******************************************************************************/ + private static boolean monitored(QRecord endpoint) + { + String active = endpoint.getValueString("activeStatus"); + return (ActiveStatus.ACTIVE.name().equals(active) + || (ActiveStatus.DISABLED.name().equals(active) && endpoint.getValue("autoDisabledDate") != null)); + } + + + + /******************************************************************************* + ** Query only allow-listed fields with the remaining database time budget. + ** The runner enforces the precise deadline when the backend blocks. + *******************************************************************************/ + private static List query(HealthCheckInput input, String table, QQueryFilter filter, Set fields) throws QException + { + int seconds = (int) Math.max(1, Math.min(5, Duration.between(Instant.now(), input.getDeadline()).toSeconds())); + return (SystemWrites.call(() -> new QueryAction().execute(new QueryInput(table).withFilter(filter) + .withFieldNamesToInclude(fields).withTimeoutSeconds(seconds)).getRecords())); + } + + + + /******************************************************************************* + ** Use wall time for the runner deadline, independently of the domain clock. + *******************************************************************************/ + private static boolean expired(HealthCheckInput input) + { + return (!Instant.now().isBefore(input.getDeadline())); + } +} diff --git a/src/main/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidget.java b/src/main/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidget.java new file mode 100644 index 0000000..5979ed4 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidget.java @@ -0,0 +1,70 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.webhooks.health; + + +import java.util.List; +import java.util.Set; +import com.kingsrook.qqq.backend.core.actions.ai.output.AiLinkBuilder; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthMetaDataProvider; +import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData; +import com.kingsrook.qqq.backend.core.model.data.QRecord; + + +/******************************************************************************* + ** One native status line linking to central health, using the viewer's access. + *******************************************************************************/ +public class WebhooksHealthWidget extends AbstractWidgetRenderer +{ + /******************************************************************************* + ** Read the central component rollup; never run a probe from the home. + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + String status = "No checks have run yet"; + if(QContext.getQInstance().getTable(HealthMetaDataProvider.COMPONENT_TABLE) != null) + { + List records = new QueryAction().execute(new QueryInput(HealthMetaDataProvider.COMPONENT_TABLE) + .withFilter(new QQueryFilter(new QFilterCriteria("name", QCriteriaOperator.EQUALS, "webhooks")).withLimit(1)) + .withFieldNamesToInclude(Set.of("status"))).getRecords(); + if(!records.isEmpty()) + { + String stored = records.getFirst().getValueString("status"); + status = List.of("UP", "DEGRADED", "DOWN", "UNKNOWN").contains(stored == null ? "" : stored) ? stored : HealthStatus.UNKNOWN.name(); + } + } + return (new RenderWidgetOutput(new StatisticsData().withCount(status).withCountContext("Webhooks - System Health") + .withCountURL(AiLinkBuilder.filteredTableHref(HealthMetaDataProvider.CHECK_TABLE, + new QQueryFilter(new QFilterCriteria("component", QCriteriaOperator.EQUALS, "webhooks")))))); + } +} diff --git a/src/test/java/com/kingsrook/qbits/webhooks/WebhooksQBitProducerTest.java b/src/test/java/com/kingsrook/qbits/webhooks/WebhooksQBitProducerTest.java index c1b560e..2e89850 100644 --- a/src/test/java/com/kingsrook/qbits/webhooks/WebhooksQBitProducerTest.java +++ b/src/test/java/com/kingsrook/qbits/webhooks/WebhooksQBitProducerTest.java @@ -90,7 +90,7 @@ class WebhooksQBitProducerTest { assertThat(child).isNotInstanceOf(QProcessMetaData.class); } - assertThat(app.getWidgets()).containsExactly("webhooksHome"); + assertThat(app.getWidgets()).containsExactly("webhooksHealth", "webhooksHome"); Set endpointWidgets = instance.getTable(WebhooksTables.ENDPOINT).getSections().stream().map(s -> s.getWidgetName()).filter(w -> w != null).collect(Collectors.toSet()); assertThat(endpointWidgets).contains(WebhooksTables.WIDGET_ENDPOINT_SUBSCRIPTIONS, WebhooksTables.WIDGET_ENDPOINT_SECRETS, WebhooksTables.WIDGET_ENDPOINT_DELIVERIES); } diff --git a/src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthChecksTest.java b/src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthChecksTest.java new file mode 100644 index 0000000..33806ee --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthChecksTest.java @@ -0,0 +1,366 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.webhooks.health; + + +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.kingsrook.qbits.webhooks.WebhooksRuntime; +import com.kingsrook.qbits.webhooks.WebhooksTestSupport; +import com.kingsrook.qbits.webhooks.delivery.EndpointHealth; +import com.kingsrook.qbits.webhooks.metadata.WebhooksTables; +import com.kingsrook.qbits.webhooks.support.SystemWrites; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface; +import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCategory; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthCheckKind; +import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.model.actions.tables.QueryOrGetInputInterface; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Passive health checks use memory records and never contact a receiver. + *******************************************************************************/ +class WebhooksHealthChecksTest +{ + private static final String SENTINEL = "S3NT1NEL-secret"; + private QInstance instance; + + + + /******************************************************************************* + ** Set up the producer and a sentinel credential reference. + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + instance = WebhooksTestSupport.setUp(); + System.setProperty("WEBHOOKS_HEALTH_SECRET", SENTINEL + "-long-enough"); + } + + + + /******************************************************************************* + ** Remove the test credential. + *******************************************************************************/ + @AfterEach + void tearDown() + { + System.clearProperty("WEBHOOKS_HEALTH_SECRET"); + } + + + + /******************************************************************************* + ** Supplier refresh finds new endpoints and excludes manual disable/pause. + *******************************************************************************/ + @Test + void registrationAndDynamicDiscovery() throws Exception + { + HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(instance); + assertThat(registry.getChecks()).extracting(HealthCheckMetaData::getName).containsExactly("webhooks.dispatcher"); + assertThat(registry.getSuppliers()).hasSize(1); + assertThat(registry.getSuppliers().getFirst().getNamePrefix()).isEqualTo("webhooks.endpoint"); + assertThat(registry.getSuppliers().getFirst().getComponent()).isEqualTo("webhooks"); + assertThat(registry.resolveAll(instance)).hasSize(1); + Integer active = endpoint(); + Integer disabled = endpoint(); + update(disabled, "activeStatus", "DISABLED"); + Integer paused = endpoint(); + update(paused, "activeStatus", "PAUSED"); + Integer automatic = endpoint(); + update(automatic, "activeStatus", "DISABLED"); + update(automatic, "autoDisabledDate", WebhooksTestSupport.CLOCK.instant()); + QContext.init(instance, new QSession()); + assertThat(registry.resolveAll(instance)).extracting(HealthCheckMetaData::getName) + .containsExactlyInAnyOrder("webhooks.dispatcher", "webhooks.endpoint." + active, "webhooks.endpoint." + automatic); + assertThat(registry.resolveAll(instance)).allSatisfy(check -> + { + assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY); + assertThat(check.getCategory()).isEqualTo(HealthCategory.INTEGRATION); + assertThat(check.getComponent()).isEqualTo("webhooks"); + assertThat(check.getIntervalSeconds()).isEqualTo(300); + assertThat(check.getTimeoutMs()).isLessThan(300000); + assertThat(check.getIncludeInReadiness()).isFalse(); + }); + } + + + + /******************************************************************************* + ** Read fresh persisted states and keep the existing state machine intact. + *******************************************************************************/ + @Test + void endpointStatesAndRecovery() throws Exception + { + Integer id = endpoint(); + HealthCheckMetaData check = check("webhooks.endpoint." + id); + assertOutcome(check, HealthStatus.UP); + for(String state : List.of("UNHEALTHY", "PROBATION")) + { + update(id, "healthStatus", state); + update(id, "consecutiveFailures", 3); + update(id, "unhealthySinceDate", WebhooksTestSupport.CLOCK.instant()); + HealthCheckOutcome outcome = assertOutcome(check, HealthStatus.DEGRADED); + assertThat(outcome.getDetails().getValues()).containsExactlyInAnyOrderEntriesOf(Map.of( + "healthStatus", state, "consecutiveFailures", 3, "unhealthySince", WebhooksTestSupport.CLOCK.instant())); + } + update(id, "probationFailures", 1); + QRecord row = SystemWrites.query(WebhooksTables.ENDPOINT, new QQueryFilter()).getFirst(); + EndpointHealth.recordFailure(WebhooksRuntime.config(), row, WebhooksTestSupport.CLOCK.instant(), SENTINEL); + assertOutcome(check, HealthStatus.DOWN); + assertThat(check("webhooks.endpoint." + id)).isNotNull(); + update(id, "activeStatus", "ACTIVE"); + EndpointHealth.recordSuccess(row, WebhooksTestSupport.CLOCK.instant()); + assertOutcome(check, HealthStatus.UP); + update(id, "healthStatus", SENTINEL); + assertOutcome(check, HealthStatus.UNKNOWN); + update(id, "healthStatus", null); + assertOutcome(check, HealthStatus.UNKNOWN); + update(id, "activeStatus", "DISABLED"); + assertOutcome(check, HealthStatus.DOWN); + update(id, "autoDisabledDate", null); + assertOutcome(check, HealthStatus.UNKNOWN); + SystemWrites.delete(WebhooksTables.ENDPOINT, new QQueryFilter()); + assertOutcome(check, HealthStatus.UNKNOWN); + } + + + + /******************************************************************************* + ** Strict thresholds and terminal-status exclusion across every queue state. + *******************************************************************************/ + @Test + void dispatcherThresholds() throws Exception + { + HealthCheckMetaData check = check("webhooks.dispatcher"); + assertThat(assertOutcome(check, HealthStatus.UP).getMetrics().getFirst().getValue().longValue()).isZero(); + for(String terminal : List.of("DELIVERED", "CANCELLED", "DEAD_LETTER")) + { + delivery(terminal, 10000); + } + assertOutcome(check, HealthStatus.UP); + for(String status : List.of("PENDING", "RETRY_WAIT", "SENDING")) + { + for(long age : new long[] {-10, 0, 900, 901, 3600, 3601}) + { + QRecord record = delivery(status, age); + HealthCheckOutcome outcome = assertOutcome(check, age > 3600 ? HealthStatus.DOWN : age > 900 ? HealthStatus.DEGRADED : HealthStatus.UP); + assertThat(outcome.getMetrics().getFirst().getName()).isEqualTo("oldestPendingSeconds"); + assertThat(outcome.getMetrics().getFirst().getValue().longValue()).isEqualTo(Math.max(0, age)); + SystemWrites.updateOne(WebhooksTables.DELIVERY, new QRecord().withValue("id", record.getValue("id")).withValue("status", "DELIVERED")); + } + } + delivery("PENDING", 4000); + delivery("PENDING", 1); + assertThat(assertOutcome(check, HealthStatus.DOWN).getMetrics().getFirst().getValue().longValue()).isEqualTo(4000); + } + + + + /******************************************************************************* + ** A missing queue timestamp cannot be reported as a healthy backlog. + *******************************************************************************/ + @Test + void missingQueueTimestamp() throws Exception + { + QRecord record = delivery("PENDING", 0); + MemoryRecordStore.getInstance().update(new UpdateInput(WebhooksTables.DELIVERY).withRecords(List.of( + new QRecord().withValue("id", record.getValue("id")).withValue("createDate", null))), false); + assertOutcome(check("webhooks.dispatcher"), HealthStatus.UNKNOWN); + } + + + + /******************************************************************************* + ** No reads start after the deadline; late responses are not accepted as UP. + *******************************************************************************/ + @Test + void deadlines() throws Exception + { + endpoint(); + delivery("PENDING", 0); + for(HealthCheckMetaData check : HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance)) + { + assertThat(check.getCheckInstance().run(input().withDeadline(Instant.now().minusSeconds(1))).getStatus()).isEqualTo(HealthStatus.DOWN); + } + instance.getTable(WebhooksTables.DELIVERY).withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(SlowQuery.class)); + HealthCheckOutcome outcome = check("webhooks.dispatcher").getCheckInstance().run(input().withDeadline(Instant.now().plusMillis(50))); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(outcome.getMessage()).contains("timed out"); + } + + + + /******************************************************************************* + ** Exception messages, URLs, failure text and credentials never leave checks. + *******************************************************************************/ + @Test + void secretBearingFailures() throws Exception + { + Integer id = endpoint(); + HealthCheckMetaData endpoint = check("webhooks.endpoint." + id); + delivery("PENDING", 0); + assertOutcome(endpoint, HealthStatus.UP); + assertOutcome(check("webhooks.dispatcher"), HealthStatus.UP); + for(String table : List.of(WebhooksTables.ENDPOINT, WebhooksTables.DELIVERY)) + { + instance.getTable(table).withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(FailingQuery.class)); + } + assertOutcome(endpoint, HealthStatus.DOWN); + assertOutcome(HealthChecksMetaData.ofOrWithNew(instance).getChecks().getFirst(), HealthStatus.DOWN); + } + + + + /******************************************************************************* + ** Persist a sentinel credential without ever resolving it for a health check. + *******************************************************************************/ + private Integer endpoint() throws Exception + { + Integer id = WebhooksTestSupport.insert(WebhooksTables.ENDPOINT, new QRecord().withValue("name", SENTINEL + UUID.randomUUID()) + .withValue("url", "https://example.invalid/" + SENTINEL)).getValueInteger("id"); + WebhooksTestSupport.insert(WebhooksTables.SECRET, new QRecord().withValue("endpointId", id) + .withValue("source", "REFERENCE").withValue("secretRef", "WEBHOOKS_HEALTH_SECRET")); + update(id, "lastFailureMessage", SENTINEL); + return (id); + } + + + + /******************************************************************************* + ** Change an existing domain field as the qbit. + *******************************************************************************/ + private void update(Integer id, String field, java.io.Serializable value) throws Exception + { + SystemWrites.updateOne(WebhooksTables.ENDPOINT, new QRecord().withValue("id", id).withValue(field, value)); + } + + + + /******************************************************************************* + ** Insert a pending or terminal delivery with deterministic queue time. + *******************************************************************************/ + private QRecord delivery(String status, long age) throws Exception + { + QRecord record = SystemWrites.insertOne(WebhooksTables.DELIVERY, new QRecord().withValue("status", status) + .withValue("payload", SENTINEL).withValue("lastError", SENTINEL)); + MemoryRecordStore.getInstance().update(new UpdateInput(WebhooksTables.DELIVERY).withRecords(List.of( + new QRecord().withValue("id", record.getValue("id")).withValue("createDate", WebhooksTestSupport.CLOCK.instant().minusSeconds(age)))), false); + return (record); + } + + + + /******************************************************************************* + ** Find a producer-registered check, resolving the supplier afresh. + *******************************************************************************/ + private HealthCheckMetaData check(String name) throws Exception + { + return (HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance).stream().filter(c -> name.equals(c.getName())).findFirst().orElseThrow()); + } + + + + /******************************************************************************* + ** Bounded runner input. + *******************************************************************************/ + private HealthCheckInput input() + { + return (new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5))); + } + + + + /******************************************************************************* + ** Assert raw outcomes are clean; HealthRunner.sanitize is package-private. + *******************************************************************************/ + private HealthCheckOutcome assertOutcome(HealthCheckMetaData check, HealthStatus status) throws Exception + { + HealthCheckOutcome outcome = check.getCheckInstance().run(input()); + assertThat(outcome.getStatus()).isEqualTo(status); + assertThat(outcome.getMessage()).doesNotContain(SENTINEL).doesNotContain("example.invalid").doesNotContain("\n"); + assertThat(outcome.getDetails().toJson()).doesNotContain(SENTINEL).doesNotContain("example.invalid"); + return (outcome); + } + + + + /******************************************************************************* + ** A backend failure containing sensitive text. + *******************************************************************************/ + public static class FailingQuery implements TableCustomizerInterface + { + /*************************************************************************** + ** Throw after the read, before the check sees any records. + ***************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface input, List records) throws QException + { + throw (new QException(SENTINEL)); + } + } + + + + /******************************************************************************* + ** A database that responds after the check's deadline. + *******************************************************************************/ + public static class SlowQuery implements TableCustomizerInterface + { + /*************************************************************************** + ** Simulate a slow database, with no network. + ***************************************************************************/ + @Override + public List postQuery(QueryOrGetInputInterface input, List records) throws QException + { + try + { + Thread.sleep(100); + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + throw (new QException("Interrupted", e)); + } + return (records); + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidgetTest.java b/src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidgetTest.java new file mode 100644 index 0000000..60932dc --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidgetTest.java @@ -0,0 +1,82 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.webhooks.health; + + +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import com.kingsrook.qbits.webhooks.WebhooksTestSupport; +import com.kingsrook.qbits.webhooks.app.WebhooksApp; +import com.kingsrook.qbits.webhooks.support.SystemWrites; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.health.HealthMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** The native home line displays only the persisted central component state. + *******************************************************************************/ +class WebhooksHealthWidgetTest +{ + /******************************************************************************* + ** No data means unknown, with a component-filtered link to System Health. + *******************************************************************************/ + @Test + void emptyAndStoredComponent() throws Exception + { + var instance = WebhooksTestSupport.setUp(); + assertThat(instance.getWidget(WebhooksApp.HEALTH_WIDGET).getType()).isEqualTo(WidgetType.STATISTICS.getType()); + StatisticsData data = render(); + assertThat(data.getCount()).isEqualTo("No checks have run yet"); + assertThat(data.getCountURL()).startsWith("/app/health_check?filter="); + String filter = new String(Base64.getDecoder().decode(URLDecoder.decode(data.getCountURL().split("filter=")[1], StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + assertThat(filter).contains("component", "webhooks"); + new HealthMetaDataProvider().defineAll(instance, WebhooksTestSupport.MEMORY, null); + QContext.init(instance, new QSession().withPermissions(List.of("health.read"))); + assertThat(render().getCount()).isEqualTo("No checks have run yet"); + QRecord component = SystemWrites.insertOne(HealthMetaDataProvider.COMPONENT_TABLE, new QRecord().withValue("name", "webhooks").withValue("status", "DOWN")); + for(String status : List.of("UP", "DEGRADED", "DOWN", "UNKNOWN", "S3NT1NEL-secret")) + { + SystemWrites.updateOne(HealthMetaDataProvider.COMPONENT_TABLE, new QRecord().withValue("id", component.getValue("id")).withValue("status", status)); + assertThat(render().getCount()).isEqualTo(status.equals("S3NT1NEL-secret") ? "UNKNOWN" : status); + } + SystemWrites.updateOne(HealthMetaDataProvider.COMPONENT_TABLE, new QRecord().withValue("id", component.getValue("id")).withValue("status", null)); + assertThat(render().getCount()).isEqualTo("UNKNOWN"); + } + + + + /******************************************************************************* + ** Render the one-line native widget. + *******************************************************************************/ + private StatisticsData render() throws Exception + { + return ((StatisticsData) new WebhooksHealthWidget().render(new RenderWidgetInput()).getWidgetData()); + } +} -- 2.54.0 From d988e00be612d527dda59aad5c1036078d52f867 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 14:55:11 -0500 Subject: [PATCH 2/3] refactor(webhooks): home links to central health instead of its own endpoint health tile Co-Authored-By: Claude Opus 5.5 --- .../qbits/webhooks/app/WebhooksApp.java | 13 ++- .../qbits/webhooks/app/WebhooksHomeData.java | 89 ------------------- .../qbits/webhooks/app/WebhooksHomeView.java | 38 -------- .../qbits/webhooks/app/WebhooksHomeTest.java | 6 +- 4 files changed, 14 insertions(+), 132 deletions(-) diff --git a/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksApp.java b/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksApp.java index 520f9e4..2cb7e07 100644 --- a/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksApp.java +++ b/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksApp.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import com.kingsrook.qbits.webhooks.WebhooksQBitConfig; import com.kingsrook.qbits.webhooks.WebhooksRuntime; +import com.kingsrook.qbits.webhooks.health.WebhooksHealthWidget; import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.SecretSource; import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.SecretStatus; import com.kingsrook.qbits.webhooks.metadata.WebhooksTables; @@ -34,7 +35,10 @@ import com.kingsrook.qbits.webhooks.permissions.WebhooksPermissions; import com.kingsrook.qbits.webhooks.secrets.SecretCipher; import com.kingsrook.qbits.webhooks.support.SystemWrites; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; +import com.kingsrook.qqq.backend.core.actions.permissions.UseTablePermissionCustomPermissionChecker; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthMetaDataProvider; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; @@ -47,6 +51,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel; +import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; @@ -57,6 +63,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; *******************************************************************************/ public final class WebhooksApp { + public static final String HEALTH_WIDGET = "webhooksHealth"; public static final String HOME_WIDGET = "webhooksHome"; public static final String SECRET_VALUE_WIDGET = "webhooksSecretValue"; @@ -77,6 +84,10 @@ public final class WebhooksApp public static List widgets() { return (List.of( + new QWidgetMetaData().withName(HEALTH_WIDGET).withLabel("System Health").withType(WidgetType.STATISTICS.getType()) + .withCodeReference(new QCodeReference(WebhooksHealthWidget.class)).withGridColumns(12) + .withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION) + .withCustomPermissionChecker(UseTablePermissionCustomPermissionChecker.build(HealthMetaDataProvider.CHECK_TABLE, TablePermissionSubType.READ))), new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Overview").withType(WidgetType.HTML.getType()) .withCodeReference(new QCodeReference(HomeRenderer.class)).withGridColumns(12).withIsCard(false) .withPermissionRules(WebhooksTables.permissionRules()), @@ -97,7 +108,7 @@ public final class WebhooksApp .withLabel(config.getAppLabel()) .withIcon(new QIcon("webhook")) .withPermissionRules(WebhooksTables.permissionRules()) - .withWidgets(List.of(HOME_WIDGET)); + .withWidgets(List.of(HEALTH_WIDGET, HOME_WIDGET)); for(String name : WebhooksTables.MAIN) { if(tables.containsKey(name)) diff --git a/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeData.java b/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeData.java index c7ca7e8..e44e31d 100644 --- a/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeData.java +++ b/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeData.java @@ -26,15 +26,10 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; import java.util.List; -import java.util.Set; import com.kingsrook.qbits.webhooks.WebhooksRuntime; -import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.ActiveStatus; import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.AttemptOutcome; import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.DeliveryStatus; -import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.HealthStatus; -import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.SecretStatus; import com.kingsrook.qbits.webhooks.metadata.WebhooksTables; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; @@ -59,7 +54,6 @@ public final class WebhooksHomeData private static final int ATTEMPT_SAMPLE = 20000; private Instant now; - private int endpointCount; private int delivered24h; private int failedAttempts24h; private int rateLimited24h; @@ -69,8 +63,6 @@ public final class WebhooksHomeData private Long p95Millis; private final int[] hourlyOk = new int[24]; private final int[] hourlyFailed = new int[24]; - private final List attention = new ArrayList<>(); - private final List attentionWhy = new ArrayList<>(); private final List recentDeadLetters = new ArrayList<>(); @@ -84,24 +76,6 @@ public final class WebhooksHomeData data.now = WebhooksRuntime.clock().instant(); Instant since = data.now.minus(Duration.ofHours(24)); - List endpoints = query(WebhooksTables.ENDPOINT, new QQueryFilter().withOrderBy(new QFilterOrderBy("name")).withLimit(500)); - data.endpointCount = endpoints.size(); - - Set withSecret = new HashSet<>(); - for(QRecord secret : query(WebhooksTables.SECRET, new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.IN, SecretStatus.ACTIVE.name(), SecretStatus.PREVIOUS.name())))) - { - withSecret.add(secret.getValueInteger("endpointId")); - } - for(QRecord endpoint : endpoints) - { - String why = attentionReason(endpoint, withSecret.contains(endpoint.getValueInteger("id"))); - if(why != null) - { - data.attention.add(endpoint); - data.attentionWhy.add(why); - } - } - data.delivered24h = count(WebhooksTables.DELIVERY, new QQueryFilter(new QFilterCriteria("deliveredDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, since))); data.deadLetters = count(WebhooksTables.DELIVERY, new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, DeliveryStatus.DEAD_LETTER.name()))); data.backlog = count(WebhooksTables.DELIVERY, new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.IN, @@ -155,39 +129,6 @@ public final class WebhooksHomeData - /******************************************************************************* - ** Why an endpoint needs attention, or null. - *******************************************************************************/ - static String attentionReason(QRecord endpoint, boolean hasSecret) - { - String health = endpoint.getValueString("healthStatus"); - String active = endpoint.getValueString("activeStatus"); - if(ActiveStatus.DISABLED.name().equals(active) && endpoint.getValue("autoDisabledDate") != null) - { - return ("Disabled after failing on probation"); - } - if(HealthStatus.UNHEALTHY.name().equals(health)) - { - return ("Unhealthy: deliveries paused"); - } - if(HealthStatus.PROBATION.name().equals(health)) - { - return ("On probation: one try per run"); - } - if(ActiveStatus.ACTIVE.name().equals(active) && !hasSecret && !Boolean.TRUE.equals(endpoint.getValueBoolean("allowUnsigned"))) - { - return ("No signing secret: deliveries are waiting"); - } - Integer failures = endpoint.getValueInteger("consecutiveFailures"); - if(failures != null && failures > 0) - { - return (failures + (failures == 1 ? " failure" : " failures") + " in a row"); - } - return (null); - } - - - /******************************************************************************* ** Query as the viewer. *******************************************************************************/ @@ -219,16 +160,6 @@ public final class WebhooksHomeData - /******************************************************************************* - ** Getter for endpointCount. - *******************************************************************************/ - public int getEndpointCount() - { - return (endpointCount); - } - - - /******************************************************************************* ** Getter for delivered24h. *******************************************************************************/ @@ -319,26 +250,6 @@ public final class WebhooksHomeData - /******************************************************************************* - ** Endpoints that need attention. - *******************************************************************************/ - public List getAttention() - { - return (attention); - } - - - - /******************************************************************************* - ** The reason for each endpoint in {@link #getAttention()}. - *******************************************************************************/ - public List getAttentionWhy() - { - return (attentionWhy); - } - - - /******************************************************************************* ** The five newest dead letters. *******************************************************************************/ diff --git a/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeView.java b/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeView.java index e4dda57..e85d9f6 100644 --- a/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeView.java +++ b/src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeView.java @@ -25,7 +25,6 @@ import java.io.Serializable; import java.time.Instant; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.List; import java.util.Locale; import com.kingsrook.qbits.webhooks.metadata.WebhooksTables; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -87,7 +86,6 @@ public final class WebhooksHomeView StringBuilder html = new StringBuilder("
"); kpis(html, data, tablePrefix); volume(html, data); - attention(html, data, tablePrefix); deadLetters(html, data, tablePrefix); return (html.append("
").toString()); } @@ -104,7 +102,6 @@ public final class WebhooksHomeView kpi(html, String.valueOf(data.getFailedAttempts24h()), "Failed attempts (24 h)", data.getFailedAttempts24h() > 0, prefix + "/" + WebhooksTables.DELIVERY); kpi(html, String.valueOf(data.getDeadLetters()), "Dead letters", data.getDeadLetters() > 0, prefix + "/" + WebhooksTables.DELIVERY); kpi(html, String.valueOf(data.getBacklog()), "Waiting to send", false, prefix + "/" + WebhooksTables.DELIVERY); - kpi(html, String.valueOf(data.getAttention().size()), "Endpoints needing attention", !data.getAttention().isEmpty(), prefix + "/" + WebhooksTables.ENDPOINT); kpi(html, data.getP50Millis() == null ? "None yet" : data.getP50Millis() + " / " + data.getP95Millis() + " ms", "Latency, median / 95th (24 h)", false, null); html.append(""); } @@ -177,41 +174,6 @@ public final class WebhooksHomeView - /******************************************************************************* - ** Endpoints needing attention, one card row each. - *******************************************************************************/ - private static void attention(StringBuilder html, WebhooksHomeData data, String prefix) - { - html.append("

Endpoints needing attention

"); - if(data.getEndpointCount() == 0) - { - html.append("

No endpoints yet. Add an endpoint, then subscribe it to events.

"); - return; - } - if(data.getAttention().isEmpty()) - { - html.append("

All ").append(data.getEndpointCount()).append(" endpoints are healthy.

"); - return; - } - List rows = data.getAttention(); - for(int i = 0; i < rows.size(); i++) - { - QRecord endpoint = rows.get(i); - String message = endpoint.getValueString("lastFailureMessage"); - html.append("
") - .append(cell("Endpoint", "" - + esc(endpoint.getValueString("name")) + "")) - .append(cell("Why", esc(data.getAttentionWhy().get(i)) + (message == null ? "" : "
" + esc(clamp(message, 140)) + ""))) - .append(cell("Last failure", when(endpoint.getValue("lastFailureDate")))) - .append(cell("Last success", when(endpoint.getValue("lastSuccessDate")))) - .append("
"); - } - html.append(""); - } - - - /******************************************************************************* ** Newest dead letters. *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeTest.java b/src/test/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeTest.java index 45e62f1..5000af3 100644 --- a/src/test/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeTest.java @@ -90,7 +90,7 @@ class WebhooksHomeTest void emptyHome() throws Exception { String html = home(); - assertThat(html).contains("No endpoints yet").contains("/admin/webhooks/webhooks_endpoint").contains("No delivery attempts in the last 24 hours") + assertThat(html).doesNotContain("Endpoints needing attention").contains("No delivery attempts in the last 24 hours") .contains("No dead letters").contains("None yet"); } @@ -120,12 +120,10 @@ class WebhooksHomeTest assertThat(data.getDelivered24h()).isEqualTo(3); assertThat(data.getFailedAttempts24h()).isEqualTo(3); assertThat(data.getDeadLetters()).isEqualTo(1); - assertThat(data.getAttention()).hasSize(1); - assertThat(data.getAttentionWhy().get(0)).contains("Unhealthy"); assertThat(data.getP50Millis()).isNotNull(); String html = home(); - assertThat(html).contains("Delivered (24 h)").contains("attempts in the last 24 hours, by hour").contains("bad <b>").doesNotContain("bad ") + assertThat(html).contains("Delivered (24 h)").contains("attempts in the last 24 hours, by hour").doesNotContain("Endpoints needing attention").doesNotContain("bad ") .contains("/admin/webhooks/webhooks_delivery/" + deadOne.getValue("id")).contains("Replay"); //////////////////////////////////////////////////////////////// // readability rules: no fixed widths that force a scroll bar // -- 2.54.0 From d650deb46f907e56f4006768a0e558e611f5e053 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 14:55:12 -0500 Subject: [PATCH 3/3] docs: webhooks health checks Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 +++++++ README.md | 16 ++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7651621..b455f5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and ## [Unreleased] +### Added +- Central `webhooks.endpoint.` and `webhooks.dispatcher` dependency checks, every 300 seconds, using persisted endpoint health and outbox age. +- A native home status line linking to the webhooks checks in System Health. + +### Removed +- The home's separate endpoint-health attention tile, list and diagnostic queries; endpoint state, alerts and Send Test Event remain unchanged. + ### Changed - Require QQQ 4.1.0-RC.5 and use its webhook signer, verifier, secret and verification exception (QRun-IO/qqq#1160, F11). Removed the internal signing package; Java consumers must import `com.kingsrook.qqq.backend.core.utils.webhooks` instead. - Signing wire format, timestamp tolerance, rotation, endpoint management, encrypted secret storage and delivery behavior are unchanged. diff --git a/README.md b/README.md index 8e17069..f6aff7b 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,22 @@ Upgrading from 0.x: see [docs/migration-0.x-to-1.0.md](docs/migration-0.x-to-1.0 ./mvnw verify # tests, Checkstyle and the JaCoCo gate (always use the wrapper) ``` +## Health checks + +The producer registers passive checks with QQQ central health monitoring (requires the health SPI in QQQ 4.1.0-SNAPSHOT or a subsequent release): + +| Name | Kind | Category | Interval | +|---|---|---|---| +| `webhooks.endpoint.` | DEPENDENCY | INTEGRATION | 300 seconds | +| `webhooks.dispatcher` | DEPENDENCY | INTEGRATION | 300 seconds | + +Both belong to component `webhooks`. Endpoint checks read the existing state machine: HEALTHY means UP; UNHEALTHY or PROBATION means DEGRADED; automatically disabled means DOWN. Active and automatically disabled endpoints are discovered dynamically; paused and manually disabled endpoints are excluded. Details contain only healthStatus, consecutiveFailures and unhealthySince when present. + +The dispatcher reports `oldestPendingSeconds`, measured from the oldest queued delivery's createDate across PENDING, RETRY_WAIT and SENDING. More than 15 minutes is DEGRADED; more than 60 minutes is DOWN. An empty queue reports UP with zero age. Checks do not send requests or change endpoint state. + +The home shows one native central status line linking to the webhooks checks in System Health. Disable checks centrally with `health_check.enabled`. The endpoint state machine, alerts and Send Test Event action remain unchanged. + + ## License Apache-2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE). -- 2.54.0