From 99e7e0806a4720996f78dfa3c74177dcf8db3568 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 13:39:28 -0500 Subject: [PATCH 1/3] feat(health): deadline-bounded status reads for health checks Co-Authored-By: Claude Opus 5.5 --- .../quicksearch/diagnostics/StatusCache.java | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusCache.java b/src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusCache.java index 7c5258f..7fc6b77 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusCache.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/diagnostics/StatusCache.java @@ -21,12 +21,15 @@ package com.kingsrook.qbits.quicksearch.diagnostics; +import java.time.Duration; +import java.time.Instant; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.ReentrantLock; import com.kingsrook.qbits.quicksearch.QuickSearchRuntime; import com.kingsrook.qbits.quicksearch.api.QuickSearchApi; import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus; @@ -42,6 +45,7 @@ import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; *******************************************************************************/ public class StatusCache implements AutoCloseable { + private final ReentrantLock lock = new ReentrantLock(); private final ExecutorService executor = Executors.newSingleThreadExecutor(runnable -> { Thread thread = new Thread(runnable, "quick-search-status"); @@ -58,7 +62,47 @@ public class StatusCache implements AutoCloseable ** Authorization belongs to the public service, before reaching this cache. ** Only refreshes consume the existing per-user diagnostic rate limit. *******************************************************************************/ - public synchronized QuickSearchStatus get(QuickSearchRuntime runtime) throws QException + public QuickSearchStatus get(QuickSearchRuntime runtime) throws QException + { + return (get(runtime, Instant.now().plusSeconds(2))); + } + + + + /******************************************************************************* + ** Bound both lock contention and the shared refresh wait by the caller's deadline. + *******************************************************************************/ + public QuickSearchStatus get(QuickSearchRuntime runtime, Instant deadline) throws QException + { + try + { + long remaining = Duration.between(Instant.now(), deadline).toNanos(); + if(remaining <= 0 || !lock.tryLock(remaining, TimeUnit.NANOSECONDS)) + { + throw (new QException("Status deadline exceeded")); + } + try + { + return (read(runtime, deadline)); + } + finally + { + lock.unlock(); + } + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + throw (new QException("Status refresh interrupted", e)); + } + } + + + + /******************************************************************************* + ** Read or refresh while holding the deadline-aware cache lock. + *******************************************************************************/ + private QuickSearchStatus read(QuickSearchRuntime runtime, Instant deadline) throws QException { if(cached != null && System.nanoTime() < expiresAt) { @@ -90,7 +134,8 @@ public class StatusCache implements AutoCloseable } try { - cached = refresh.get(2, TimeUnit.SECONDS); + long remaining = Math.max(0, Duration.between(Instant.now(), deadline).toNanos()); + cached = refresh.get(Math.min(TimeUnit.SECONDS.toNanos(2), remaining), TimeUnit.NANOSECONDS); refresh = null; } catch(TimeoutException e) -- 2.54.0 From 6111d2fdf2b21763a1b9135ff6fe8a206108deb2 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 13:39:29 -0500 Subject: [PATCH 2/3] feat(health): register quick search checks with core central health Co-Authored-By: Claude Opus 5.5 --- .../quicksearch/QuickSearchQBitProducer.java | 2 + .../health/QuickSearchHealthChecks.java | 282 ++++++++++++ .../health/QuickSearchHealthChecksTest.java | 420 ++++++++++++++++++ 3 files changed, 704 insertions(+) create mode 100644 src/main/java/com/kingsrook/qbits/quicksearch/health/QuickSearchHealthChecks.java create mode 100644 src/test/java/com/kingsrook/qbits/quicksearch/health/QuickSearchHealthChecksTest.java diff --git a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java index 9152f7d..c99f057 100644 --- a/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/quicksearch/QuickSearchQBitProducer.java @@ -30,6 +30,7 @@ import com.kingsrook.qbits.quicksearch.annotations.QuickSearchField; import com.kingsrook.qbits.quicksearch.annotations.QuickSearchable; import com.kingsrook.qbits.quicksearch.api.QuickSearchApi; import com.kingsrook.qbits.quicksearch.config.QuickSearchFieldRules; +import com.kingsrook.qbits.quicksearch.health.QuickSearchHealthChecks; import com.kingsrook.qbits.quicksearch.listeners.QuickSearchRecordChangeListener; import com.kingsrook.qbits.quicksearch.metadata.QuickSearchAdminAppMetaDataProducer; import com.kingsrook.qbits.quicksearch.metadata.QuickSearchDiagnosticsProcessMetaDataHelper; @@ -257,6 +258,7 @@ public class QuickSearchQBitProducer implements QBitMetaDataProducer "quick_search.opensearch".equals(check.getName()))) + { + registry.withCheck(definition("quick_search.opensearch", input -> run(input, QuickSearchHealthChecks::opensearch))); + } + if(registry.getChecks().stream().noneMatch(check -> "quick_search.backlog".equals(check.getName()))) + { + registry.withCheck(definition("quick_search.backlog", input -> run(input, QuickSearchHealthChecks::backlog))); + } + if(registry.getSuppliers().stream().noneMatch(supplier -> "quick_search.index".equals(supplier.getNamePrefix()))) + { + registry.withSupplier(new HealthCheckSupplierMetaData().withNamePrefix("quick_search.index").withComponent("quick_search") + .withSupplier(new QCodeReference(QuickSearchHealthChecks.class))); + } + } + + + + /******************************************************************************* + ** Enumerate persisted index identities, including disabled indexes. Execution + ** reads a fresh cached snapshot, never a row captured at supplier resolution. + *******************************************************************************/ + @Override + public List supply(QInstance instance) throws QException + { + QuickSearchRuntime runtime = runtime(instance); + if(runtime == null) + { + return (List.of()); + } + List checks = new ArrayList<>(); + for(QRecord row : QuickSearchConfigPrimer.queryAll(runtime.getConfig().getQuickSearchIndexTableName())) + { + String tableName = row.getValueString("tableName"); + String name = tableName.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]+", "_").replaceAll("[._-]{2,}", "_").replaceAll("^[._-]|[._-]$", ""); + if(name.isEmpty()) + { + throw (new QException("Index name cannot form a health check identifier")); + } + checks.add(definition("quick_search.index." + name, input -> run(input, status -> index(status, tableName)))); + } + return (checks); + } + + + + /******************************************************************************* + ** All Quick Search checks are dependencies; none gate Kubernetes readiness. + *******************************************************************************/ + private static HealthCheckMetaData definition(String name, HealthCheck check) + { + return (new HealthCheckMetaData().withName(name).withLabel(name).withComponent("quick_search") + .withCategory(HealthCategory.SEARCH).withKind(HealthCheckKind.DEPENDENCY) + .withIntervalSeconds(60).withTimeoutMs(5000).withCheckInstance(check)); + } + + + + /******************************************************************************* + ** Resolve from the supplied instance, not a static or previously closed runtime. + *******************************************************************************/ + private static QuickSearchRuntime runtime(QInstance instance) + { + if(instance != null && instance.getQBits() != null) + { + for(QBitMetaData qbit : instance.getQBits().values()) + { + if(qbit.getConfig() instanceof QuickSearchQBitConfig config) + { + return (config.getRuntime()); + } + } + } + return (null); + } + + + + /******************************************************************************* + ** Share the diagnostic worker and respect the deadline even on a cache hit. + ** Expected diagnostic failures use fixed messages, never exception payloads. + *******************************************************************************/ + private static HealthCheckOutcome run(HealthCheckInput input, Function project) + { + if(!Instant.now().isBefore(input.getDeadline())) + { + return (HealthCheckOutcome.down("Quick Search health deadline exceeded")); + } + QuickSearchRuntime runtime = runtime(input.getQInstance()); + if(runtime == null) + { + return (HealthCheckOutcome.unknown("Quick Search is not configured")); + } + try + { + QuickSearchStatus status = runtime.getStatusCache().get(runtime, input.getDeadline()); + if(!Instant.now().isBefore(input.getDeadline())) + { + return (HealthCheckOutcome.down("Quick Search health deadline exceeded")); + } + return (project.apply(status)); + } + catch(QException e) + { + return (HealthCheckOutcome.down("Quick Search diagnostics unavailable")); + } + } + + + + /******************************************************************************* + ** Cluster health is independent of backlog and per-index degradation. + *******************************************************************************/ + private static HealthCheckOutcome opensearch(QuickSearchStatus status) + { + if(status.state() == QuickSearchStatus.State.DISABLED) + { + return (HealthCheckOutcome.unknown("Quick Search is disabled")); + } + HealthCheckOutcome outcome; + if(status.state() == QuickSearchStatus.State.DOWN || Boolean.FALSE.equals(status.clusterReachable()) || "red".equals(status.clusterHealth())) + { + outcome = HealthCheckOutcome.down("OpenSearch is unavailable or red"); + } + else if(Boolean.TRUE.equals(status.clusterReachable()) && "green".equals(status.clusterHealth())) + { + outcome = HealthCheckOutcome.up("OpenSearch is green"); + } + else + { + outcome = HealthCheckOutcome.degraded("OpenSearch health is yellow or incomplete"); + } + detail(outcome, "reachable", status.clusterReachable()); + detail(outcome, "nodes", status.nodes()); + detail(outcome, "activeShards", status.activeShards()); + detail(outcome, "unassignedShards", status.unassignedShards()); + return (outcome); + } + + + + /******************************************************************************* + ** Counts are lower bounds when truncated; do not declare an incomplete backlog healthy. + *******************************************************************************/ + private static HealthCheckOutcome backlog(QuickSearchStatus status) + { + HealthCheckOutcome outcome; + if(status.state() == QuickSearchStatus.State.DISABLED) + { + outcome = HealthCheckOutcome.unknown("Quick Search is disabled"); + } + else if(status.pending() == null || status.exhausted() == null || status.indexingPaused() == null) + { + outcome = HealthCheckOutcome.degraded("Indexing backlog measurements unavailable"); + } + else if(status.indexingPaused() || status.exhausted() > 0 || status.pending() > 100 || Boolean.TRUE.equals(status.backlogTruncated())) + { + outcome = HealthCheckOutcome.degraded("Indexing is paused or backlog needs attention"); + } + else + { + outcome = HealthCheckOutcome.up("Indexing backlog is within limits"); + } + detail(outcome, "pending", status.pending()); + detail(outcome, "exhausted", status.exhausted()); + detail(outcome, "indexingPaused", status.indexingPaused()); + detail(outcome, "backlogTruncated", status.backlogTruncated()); + return (outcome); + } + + + + /******************************************************************************* + ** Missing rows (including those outside the bounded snapshot) stay unknown. + ** Persisted error text is represented only by a boolean. + *******************************************************************************/ + private static HealthCheckOutcome index(QuickSearchStatus status, String tableName) + { + IndexRow row = status.indexes().stream().filter(candidate -> tableName.equals(candidate.tableName())).findFirst().orElse(null); + if(row == null) + { + return (HealthCheckOutcome.unknown("Index measurements unavailable")); + } + boolean hasError = row.lastError() != null && !row.lastError().isBlank(); + boolean mismatch = row.documentCount() != null && row.sourceCount() != null && !row.documentCount().equals(row.sourceCount()); + HealthCheckOutcome outcome; + if(Boolean.FALSE.equals(row.enabled())) + { + outcome = HealthCheckOutcome.unknown("Index is disabled"); + } + else if(!"ACTIVE".equals(row.status()) || hasError || mismatch || (row.realTimeErrorCount() != null && row.realTimeErrorCount() > 0)) + { + outcome = HealthCheckOutcome.degraded("Index needs attention"); + } + else + { + outcome = HealthCheckOutcome.up("Index is active"); + } + outcome.withDetails(new HealthDetails().withValue("hasError", hasError).withValue("countMismatch", mismatch)); + detail(outcome, "enabled", row.enabled()); + detail(outcome, "documentCount", row.documentCount()); + detail(outcome, "sourceCount", row.sourceCount()); + detail(outcome, "countsCheckedAt", row.countsCheckedAt()); + detail(outcome, "lastSuccessfulSync", row.lastSuccessfulSync()); + detail(outcome, "realTimeErrorCount", row.realTimeErrorCount()); + return (outcome); + } + + + + /******************************************************************************* + ** Unknown measurements are omitted rather than converted to zero. + *******************************************************************************/ + private static void detail(HealthCheckOutcome outcome, String name, Object value) + { + if(value != null) + { + outcome.getDetails().withValue(name, value); + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/quicksearch/health/QuickSearchHealthChecksTest.java b/src/test/java/com/kingsrook/qbits/quicksearch/health/QuickSearchHealthChecksTest.java new file mode 100644 index 0000000..998d31d --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/quicksearch/health/QuickSearchHealthChecksTest.java @@ -0,0 +1,420 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2026. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.quicksearch.health; + + +import java.time.Instant; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.quicksearch.BaseQuickSearchTest; +import com.kingsrook.qbits.quicksearch.QuickSearchAuthMode; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitConfig; +import com.kingsrook.qbits.quicksearch.QuickSearchQBitProducer; +import com.kingsrook.qbits.quicksearch.QuickSearchRuntime; +import com.kingsrook.qbits.quicksearch.QuickSearchStartupMode; +import com.kingsrook.qbits.quicksearch.SearchableFieldConfig; +import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus; +import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.IndexRow; +import com.kingsrook.qbits.quicksearch.api.QuickSearchStatus.State; +import com.kingsrook.qbits.quicksearch.config.QuickSearchConfigPrimer; +import com.kingsrook.qbits.quicksearch.diagnostics.StatusCache; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCategory; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthCheckKind; +import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + + +/******************************************************************************* + ** Registration and projections use memory records and mocked diagnostics only. + *******************************************************************************/ +class QuickSearchHealthChecksTest extends BaseQuickSearchTest +{ + private static final String SENTINEL = "S3NT1NEL-secret"; + private QInstance instance; + private QuickSearchQBitConfig config; + private QuickSearchRuntime runtime; + private QuickSearchRuntime originalRuntime; + private StatusCache cache; + + + /******************************************************************************* + ** Exercise the real producer, with credentials that must never enter outcomes. + *******************************************************************************/ + @BeforeEach + void setUpHealth() throws Exception + { + QInstance base = QContext.getQInstance(); + instance = new QInstance(); + instance.addBackend(base.getBackend(TEST_BACKEND_NAME)); + instance.addTable(base.getTable(TEST_ENTITY_TABLE)); + instance.withInstanceDefaultAuthentication(new QAuthenticationMetaData().withName("anonymous").withType(QAuthenticationType.FULLY_ANONYMOUS)); + QContext.init(instance, new QSystemUserSession()); + config = new QuickSearchQBitConfig().withBackendName(TEST_BACKEND_NAME).withStartupMode(QuickSearchStartupMode.DEGRADED) + .withEnableRealTimeIndexing(false).withOpensearchUrl("https://unused.invalid") + .withOpensearchIndexName("health-test").withSearchableTable(TEST_ENTITY_TABLE, List.of(new SearchableFieldConfig("name"))) + .withAuthMode(QuickSearchAuthMode.BASIC).withOpensearchUsername(SENTINEL).withOpensearchPassword(SENTINEL); + new QuickSearchQBitProducer().withConfig(config).produce(instance); + QuickSearchConfigPrimer.insertOne(config.getQuickSearchIndexTableName(), new QRecord().withValue("tableName", TEST_ENTITY_TABLE) + .withValue("enabled", true).withValue("status", "ACTIVE").withValue("documentCount", 12)); + originalRuntime = config.getRuntime(); + runtime = spy(originalRuntime); + cache = mock(StatusCache.class); + doReturn(cache).when(runtime).getStatusCache(); + config.setRuntime(runtime); + } + + + + /******************************************************************************* + ** Release the producer's real worker even when tests replace the cache. + *******************************************************************************/ + @AfterEach + void closeHealth() + { + if(originalRuntime != null) + { + originalRuntime.close(); + } + } + + + + /******************************************************************************* + ** Static checks and dynamically resolved indexes have the specified metadata. + *******************************************************************************/ + @Test + void registration() throws Exception + { + QuickSearchHealthChecks.register(instance); + HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(instance); + assertThat(registry.getChecks()).extracting(HealthCheckMetaData::getName) + .containsExactly("quick_search.opensearch", "quick_search.backlog"); + assertThat(registry.getSuppliers()).singleElement().satisfies(supplier -> + { + assertThat(supplier.getNamePrefix()).isEqualTo("quick_search.index"); + assertThat(supplier.getComponent()).isEqualTo("quick_search"); + }); + assertThat(registry.resolveAll(instance)).hasSize(3).allSatisfy(check -> + { + assertThat(check.getName()).matches("[a-z0-9]+([._-][a-z0-9]+)*"); + assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY); + assertThat(check.getCategory()).isEqualTo(HealthCategory.SEARCH); + assertThat(check.getComponent()).isEqualTo("quick_search"); + assertThat(check.getIntervalSeconds()).isEqualTo(60); + assertThat(check.getTimeoutMs()).isEqualTo(5000); + }); + assertThat(registry.resolveAll(instance)).extracting(HealthCheckMetaData::getName).contains("quick_search.index.testentity"); + verifyNoInteractions(cache); + } + + + + /******************************************************************************* + ** Overall degradation from indexing must not downgrade a healthy cluster. + *******************************************************************************/ + @ParameterizedTest + @CsvSource({"OK,true,green,UP", "DEGRADED,true,green,UP", "DEGRADED,true,yellow,DEGRADED", "DOWN,true,red,DOWN", + "DOWN,false,,DOWN", "DISABLED,,,UNKNOWN", "DEGRADED,,,DEGRADED", "OK,true,unexpected,DEGRADED"}) + void clusterStates(State state, Boolean reachable, String color, HealthStatus expected) throws Exception + { + snapshot(state, reachable, color, false, 0, 0, false, List.of(row(true, "ACTIVE", 12L, 12L, null, 0))); + assertThat(run("opensearch").getStatus()).isEqualTo(expected); + } + + + + /******************************************************************************* + ** Boundary values, exhausted events, pause, truncation and missing measurements. + *******************************************************************************/ + @ParameterizedTest + @CsvSource({"OK,false,0,0,false,UP", "DEGRADED,false,100,0,false,UP", "DEGRADED,false,101,0,false,DEGRADED", + "DEGRADED,false,0,1,false,DEGRADED", "DEGRADED,true,0,0,false,DEGRADED", "OK,false,0,0,true,DEGRADED", + "DEGRADED,false,,0,false,DEGRADED", "DEGRADED,false,0,,false,DEGRADED", "DEGRADED,,0,0,false,DEGRADED", + "DISABLED,false,0,0,false,UNKNOWN", "DOWN,false,0,0,false,UP"}) + void backlogStates(State state, Boolean paused, Integer pending, Integer exhausted, Boolean truncated, HealthStatus expected) throws Exception + { + snapshot(state, true, "green", paused, pending, exhausted, truncated, List.of()); + assertThat(run("backlog").getStatus()).isEqualTo(expected); + } + + + + /******************************************************************************* + ** Counts, index status, last errors and real-time errors each affect the check. + *******************************************************************************/ + @ParameterizedTest + @CsvSource({"true,ACTIVE,12,12,,0,UP", "false,ACTIVE,12,12,,0,UNKNOWN", "true,REBUILDING,12,12,,0,DEGRADED", + "true,ACTIVE,12,13,,0,DEGRADED", "true,ACTIVE,12,12,error,0,DEGRADED", "true,ACTIVE,12,12,,2,DEGRADED", + "true,ACTIVE,12,,,0,UP", "true,ACTIVE,,12,,,UP", "true,,12,12,,0,DEGRADED"}) + void indexStates(Boolean enabled, String state, Long documents, Long source, String error, Integer realTimeErrors, HealthStatus expected) throws Exception + { + snapshot(State.OK, true, "green", false, 0, 0, false, List.of(row(enabled, state, documents, source, error, realTimeErrors))); + HealthCheckOutcome outcome = run("index.testentity"); + assertThat(outcome.getStatus()).isEqualTo(expected); + if(realTimeErrors != null) + { + assertThat(outcome.getDetails().getValues()).containsEntry("realTimeErrorCount", realTimeErrors); + } + } + + + + /******************************************************************************* + ** Supplier identities are refreshed; checks never retain old measurements. + *******************************************************************************/ + @Test + void dynamicRowsAndMissingSnapshot() throws Exception + { + HealthCheckMetaData check = check("index.testentity"); + snapshot(State.OK, true, "green", false, 0, 0, false, List.of(row(true, "ACTIVE", 12L, 12L, null, 0))); + assertThat(check.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UP); + snapshot(State.OK, true, "green", false, 0, 0, false, List.of()); + assertThat(check.getCheckInstance().run(input()).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + QuickSearchConfigPrimer.insertOne(config.getQuickSearchIndexTableName(), new QRecord().withValue("tableName", "New.Index-Name") + .withValue("enabled", false).withValue("status", "ACTIVE")); + assertThat(HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance)).extracting(HealthCheckMetaData::getName) + .contains("quick_search.index.new.index-name"); + } + + + + /******************************************************************************* + ** Raw outcomes omit poisoned diagnostics and exceptions; sanitize is package-private in core. + *******************************************************************************/ + @Test + void noSecrets() throws Exception + { + List checks = HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance); + snapshot(State.DEGRADED, true, SENTINEL, true, 101, 1, false, List.of(row(true, SENTINEL, 12L, 13L, SENTINEL, 9))); + for(HealthCheckMetaData check : checks) + { + assertClean(check.getCheckInstance().run(input())); + } + when(cache.get(eq(runtime), any(Instant.class))).thenThrow(new QException("Failure " + SENTINEL)); + for(HealthCheckMetaData check : checks) + { + HealthCheckOutcome outcome = check.getCheckInstance().run(input()); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN); + assertClean(outcome); + } + } + + + + /******************************************************************************* + ** An expired deadline does no diagnostic work; missing runtime is UNKNOWN. + *******************************************************************************/ + @Test + void expiredAndUnconfigured() throws Exception + { + for(HealthCheckMetaData check : HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance)) + { + assertThat(check.getCheckInstance().run(input().withDeadline(Instant.now().minusSeconds(1))).getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(check.getCheckInstance().run(input().withQInstance(new QInstance())).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + } + assertThat(new QuickSearchHealthChecks().supply(new QInstance())).isEmpty(); + verifyNoInteractions(cache); + } + + + + /******************************************************************************* + ** The real cache also removes exception text from a failed diagnostic refresh. + *******************************************************************************/ + @Test + void realCacheFailureIsClean() throws Exception + { + doReturn(originalRuntime.getStatusCache()).when(runtime).getStatusCache(); + doThrow(new QException("Failed " + SENTINEL)).when(runtime).getClient(); + assertThat(run("opensearch").getStatus()).isEqualTo(HealthStatus.DOWN); + for(HealthCheckMetaData check : HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance)) + { + assertClean(check.getCheckInstance().run(input())); + } + } + + + + /******************************************************************************* + ** A stuck refresh cannot hold another caller beyond its shorter deadline. + *******************************************************************************/ + @Test + void cacheWaitAndLockRespectDeadline() throws Exception + { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + doReturn(originalRuntime.getStatusCache()).when(runtime).getStatusCache(); + org.mockito.Mockito.doAnswer(invocation -> + { + entered.countDown(); + release.await(5, TimeUnit.SECONDS); + throw (new QException("Failed " + SENTINEL)); + }).when(runtime).getClient(); + try(var executor = Executors.newSingleThreadExecutor()) + { + var first = executor.submit(() -> + { + QContext.init(instance, new QSystemUserSession()); + try + { + return (runtime.getStatusCache().get(runtime, Instant.now().plusSeconds(2))); + } + finally + { + QContext.clear(); + } + }); + try + { + assertThat(entered.await(2, TimeUnit.SECONDS)).isTrue(); + Instant started = Instant.now(); + assertThatThrownBy(() -> runtime.getStatusCache().get(runtime, started.plusMillis(100))).isInstanceOf(QException.class); + assertThat(Instant.now()).isBefore(started.plusSeconds(1)); + } + finally + { + release.countDown(); + } + first.get(3, TimeUnit.SECONDS); + } + assertThatThrownBy(() -> runtime.getStatusCache().get(runtime, Instant.now().minusSeconds(1))).isInstanceOf(QException.class); + } + + + + /******************************************************************************* + ** A short health deadline bounds the refresh wait as well as lock contention. + *******************************************************************************/ + @Test + void refreshWaitRespectsDeadline() throws Exception + { + CountDownLatch release = new CountDownLatch(1); + doReturn(originalRuntime.getStatusCache()).when(runtime).getStatusCache(); + org.mockito.Mockito.doAnswer(invocation -> + { + release.await(5, TimeUnit.SECONDS); + throw (new QException("Failed " + SENTINEL)); + }).when(runtime).getClient(); + try + { + HealthCheckMetaData check = check("opensearch"); + Instant started = Instant.now(); + HealthCheckOutcome outcome = check.getCheckInstance().run(input().withDeadline(started.plusMillis(100))); + assertThat(Instant.now()).isBefore(started.plusSeconds(1)); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN); + assertClean(outcome); + } + finally + { + release.countDown(); + } + } + + + + /******************************************************************************* + ** Supply a deterministic snapshot without creating a network transport. + *******************************************************************************/ + private void snapshot(State state, Boolean reachable, String color, Boolean paused, Integer pending, Integer exhausted, Boolean truncated, List rows) throws Exception + { + when(cache.get(eq(runtime), any(Instant.class))).thenReturn(new QuickSearchStatus(state, reachable, color, 1, 2, 0, paused, rows, + pending, exhausted, truncated, false, null, SENTINEL, Instant.now(), "/unused")); + } + + + + /******************************************************************************* + ** A row whose identity is deliberately mixed case. + *******************************************************************************/ + private IndexRow row(Boolean enabled, String state, Long documents, Long source, String error, Integer realTimeErrors) + { + return (new IndexRow(TEST_ENTITY_TABLE, enabled, state, documents, source, Instant.now(), Instant.now(), error, realTimeErrors)); + } + + + + /******************************************************************************* + ** Resolve through the actual core registry, including supplier validation. + *******************************************************************************/ + private HealthCheckMetaData check(String suffix) throws Exception + { + return (HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance).stream() + .filter(check -> check.getName().equals("quick_search." + suffix)).findFirst().orElseThrow()); + } + + + + /******************************************************************************* + ** Run one registered check with a bounded input. + *******************************************************************************/ + private HealthCheckOutcome run(String suffix) throws Exception + { + return (check(suffix).getCheckInstance().run(input())); + } + + + + /******************************************************************************* + ** The runner always supplies an instance and absolute deadline. + *******************************************************************************/ + private HealthCheckInput input() + { + return (new HealthCheckInput().withQInstance(instance).withDeadline(Instant.now().plusSeconds(5))); + } + + + + /******************************************************************************* + ** Assert the raw public boundary, without relying on core's later redaction. + *******************************************************************************/ + private void assertClean(HealthCheckOutcome outcome) + { + assertThat(outcome.getMessage()).doesNotContain(SENTINEL); + assertThat(outcome.getDetails().toJson()).doesNotContain(SENTINEL); + } +} -- 2.54.0 From 2cde433c355b891d4ea683475c1f39f43f789b6e Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 13:39:30 -0500 Subject: [PATCH 3/3] docs: quick search health checks Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +++++- README.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e139d4a..1626edd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,13 @@ All notable changes to this project are documented here. The format follows Keep ## [Unreleased] -Requires QQQ 4.1.0-RC.6 or later (the spec-managed UI API extension contract). +Requires QQQ's central health SPI (currently local `4.1.0-SNAPSHOT`); the POM's released QQQ pin remains unchanged pending release. ### Fixed - Real-time indexing failures now increase the table's error count by the number of failed events. Successful basepulls and full reindexes reset the count. Counter update failures are logged without failing the source write. ### Added +- Central health checks `quick_search.opensearch`, `quick_search.backlog` and the dynamic `quick_search.index.` supplier, all SEARCH/DEPENDENCY checks every 60 seconds. Reuses cached diagnostics with deadline-bounded waits and safe count/flag details. Includes registration, state, deadline and credential-sentinel tests. - Supported host status API (`api.QuickSearchStatusService.get()`), immutable status values, and `QuickSearchApi` home navigation constants (issue #50/F10). Permission-checked, bounded diagnostics refreshes share a short per-runtime cache and reuse saved validation counts. New spec-managed `GET /qqq/v1/qbit/quick-search/status` (`getQuickSearchStatus`) with committed OpenAPI fragment and strict HTTP/contract tests. RC.6 requires the default `/qbit/quick-search` prefix; `/quick-search` is not permitted by core. Existing routes and wire shapes are unchanged. - Record developer panel: `QuickSearchDeveloperPanelProvider` (id `quickSearch`) adds a "Quick Search" panel to the developer view of every record of an indexed table (`/app/{table}/{id}/dev` in the Next UI), for users who may read Quick Search Index. It shows the table's index row (enabled, status, record count, last basepull and full reindex, searchable fields), the latest index run, the record's indexed document compared field by field with the database (Inspect Search Document's comparison; needs `debugPermissionRules`), the record's failed index events, the Inspect Search Document and Reindex in Search actions the user may run, and links to the admin app and the index row. An unreachable OpenSearch shows a warning in the panel. Registered by `QuickSearchDeveloperPanelsMetaDataProducer`. @@ -19,6 +20,9 @@ Requires QQQ 4.1.0-RC.6 or later (the spec-managed UI API extension contract). - `QuickSearchResult.subtitle` and `displayValues` (from `SUBTITLE` and `DISPLAY` fields); document objects `fieldValuesWord`, `fieldValuesEnglish`, `fieldValuesExact`, added to existing indexes' mapping at startup. - Possible-value sources `quick_search_eligible_table`, `quick_search_indexable_field`, `quick_search_timestamp_field`, `quick_search_field_role`, `quick_search_field_analyzer`, `quick_search_documents_choice`, `quick_search_config_source`; the join `quick_search_index_join_quick_search_index_field` and the Searchable Fields widget on the index record. Config tables are audited at field level. +### Removed +- None for central health integration; the status API, diagnostics widget and developer panel are retained. + ### Changed - Distinguish protected fields from fields that simply cannot be searched. Boolean, blob and hidden fields no longer trigger result rebuilding or erase record labels. Labels are suppressed only when their source fields are protected; unprotected hidden primary keys and timestamps remain usable. - Exclude PII/PHI-classified fields (including table defaults) and field-security-locked fields everywhere Quick Search indexes or displays values. Configuration rejects protected fields; code defaults drop them with one warning per table. Search and diagnostics omit protected values from older documents. Rebuild existing indexes after upgrading or changing classification metadata. Uses QQQ 4.1.0-RC.6; no dependency changes. diff --git a/README.md b/README.md index 6c8afb0..d70d86c 100644 --- a/README.md +++ b/README.md @@ -347,6 +347,34 @@ configured metadata prefix. `QuickSearchApi.APP_NAME` (`quickSearchAdmin`) and navigation. Keep detailed runs, failed events, validation and operator tools in the qbit's own home. [kof22 migration mapping](docs/STATUS-API-HOST-MIGRATION.md). +## Health checks + +The QBit producer registers these checks with QQQ's central health registry. All use +component `quick_search`, category `SEARCH`, a 60-second interval and a 5-second timeout. + +| Name | Kind | Interval | Reports | +| --- | --- | --- | --- | +| `quick_search.opensearch` | DEPENDENCY | 60s | Cluster reachability and colour: red is DOWN, yellow is DEGRADED, disabled is UNKNOWN. | +| `quick_search.backlog` | DEPENDENCY | 60s | DEGRADED for any exhausted events, more than 100 pending events, or paused indexing. | +| `quick_search.index.` | DEPENDENCY | 60s | One check per persisted index row; non-active status, count mismatch, last error or real-time errors are DEGRADED. Disabled indexes are UNKNOWN. | + +Index names come from the row's table name, normalized to lowercase health identifiers. +The supplier refreshes the set through core's normal discovery cycle. Checks reuse the +five-second status cache and saved validation counts, with waits bounded by the runner's +deadline. Incomplete measurements never become zero; indexes absent from the bounded +100-row status snapshot report UNKNOWN. Backlog truncation reports DEGRADED. +Details contain counts, timestamps and flags, never stored error text or credentials. + +The status endpoint, diagnostics widget and record developer panel remain available. +This integration requires the central health SPI currently installed as QQQ +`4.1.0-SNAPSHOT`; the POM's released QQQ version is unchanged. For local offline verification: + +```powershell +./mvnw.cmd -o verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT -Denforcer.skipRules=requireReleaseDeps +``` + +The final flag exempts only the release-dependency rule for this unreleased core API. + ## Classified data Quick Search excludes fields classified as **PII** or **PHI**, including fields -- 2.54.0