From 09787bfe227962277031cb1a2d8eb9b9e1f7a0bd Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 13:39:43 -0500 Subject: [PATCH 1/5] fix: set forwarded UI API headers instead of appending (QQQ 4.1 develop compatibility) Co-Authored-By: Claude Opus 5.5 --- .../middleware/javalin/specs/invoke/HeaderedUiApiInvoker.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qbit-assistant/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/invoke/HeaderedUiApiInvoker.java b/qbit-assistant/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/invoke/HeaderedUiApiInvoker.java index f52a076..518ed5e 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/invoke/HeaderedUiApiInvoker.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qqq/middleware/javalin/specs/invoke/HeaderedUiApiInvoker.java @@ -83,7 +83,7 @@ public class HeaderedUiApiInvoker extends UiApiInvoker { if(!RESERVED_HEADERS.contains(name.toLowerCase(Locale.ROOT))) { - builder.header(name, value); + builder.setHeader(name, value); } }); return (builder.build()); -- 2.54.0 From 8e47d9a6f5617c3f0ff78e2eac3a87b52e0cc04c Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 13:39:44 -0500 Subject: [PATCH 2/5] feat(health): deadline-aware connection test with shared cost-cap accounting Co-Authored-By: Claude Opus 5.5 --- .../assistant/engine/ConnectionTest.java | 72 +++++++++++++++++-- .../assistant/engine/ProviderService.java | 12 +++- .../assistant/persistence/AssistantStore.java | 15 +++- .../qbits/assistant/rag/EmbeddingService.java | 20 +++++- 4 files changed, 106 insertions(+), 13 deletions(-) diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ConnectionTest.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ConnectionTest.java index affa0cf..3f70c16 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ConnectionTest.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ConnectionTest.java @@ -18,9 +18,11 @@ package com.kingsrook.qbits.assistant.engine; import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import com.kingsrook.qbits.assistant.AssistantRuntime; +import com.kingsrook.qbits.assistant.AssistantSecurity; import com.kingsrook.qbits.assistant.model.AssistantModel; import com.kingsrook.qbits.assistant.model.AssistantProvider; import com.kingsrook.qbits.assistant.provider.RegistryModelProvider; @@ -34,6 +36,7 @@ import com.kingsrook.qbits.assistant.spi.ModelRequest; import com.kingsrook.qbits.assistant.spi.ModelResponse; import com.kingsrook.qbits.assistant.spi.StreamHandle; import com.kingsrook.qqq.backend.core.actions.ai.DataClassification; +import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -124,6 +127,16 @@ public final class ConnectionTest ** Probe health, then make and account for a minimal completion. *******************************************************************************/ private static Result call(AssistantRuntime runtime, ProviderService.Active active, String userId) + { + return (call(runtime, active, userId, Instant.now().plusSeconds(runtime.getConfig().getProviderTestTimeoutSeconds()), true)); + } + + + + /******************************************************************************* + ** Share the minimal call and TEST accounting with central model health. + *******************************************************************************/ + private static Result call(AssistantRuntime runtime, ProviderService.Active active, String userId, Instant deadline, Boolean probe) { List messages = new ArrayList<>(); messages.add(ModelMessage.user(PING_PROMPT)); @@ -136,19 +149,27 @@ public final class ConnectionTest UsageRecorder.Call usageCall = new UsageRecorder.Call(userId, null, null, active, UsageRecorder.PURPOSE_TEST); ModelPricing pricing = runtime.getProviderService().pricing(active, active.model()); long started = System.nanoTime(); + StreamHandle handle = null; try { - AssistantProviderType type = runtime.getConfig().getProviderTypes().get(active.providerType()); - HealthResult health = type.checkHealth(runtime.getProviderService().settings(active.row())); - if(Boolean.FALSE.equals(health.healthy())) + if(probe) { - throw (new ModelProviderException(ModelProviderException.Kind.NETWORK, null, health.message())); + AssistantProviderType type = runtime.getConfig().getProviderTypes().get(active.providerType()); + HealthResult health = type.checkHealth(runtime.getProviderService().settings(active.row())); + if(Boolean.FALSE.equals(health.healthy())) + { + throw (new ModelProviderException(ModelProviderException.Kind.NETWORK, null, health.message())); + } } - StreamHandle handle = active.provider().stream(request, text -> + if(!Instant.now().isBefore(deadline) || Thread.currentThread().isInterrupted()) + { + return (new Result(STATUS_ERROR, null, null, "The test deadline passed.")); + } + handle = active.provider().stream(request, text -> { // the reply text is not used }); - ModelResponse response = handle.await(Duration.ofSeconds(runtime.getConfig().getProviderTestTimeoutSeconds())); + ModelResponse response = handle.await(Duration.between(Instant.now(), deadline)); long latency = (System.nanoTime() - started) / 1_000_000; UsageRecorder.save(runtime.getStore(), UsageRecorder.row(usageCall, response, null, latency, pricing)); if(ModelResponse.STOP_CANCELLED.equals(response.getStopReason())) @@ -170,8 +191,45 @@ public final class ConnectionTest catch(InterruptedException e) { Thread.currentThread().interrupt(); + UsageRecorder.save(runtime.getStore(), UsageRecorder.row(usageCall, null, + new ModelProviderException(ModelProviderException.Kind.TIMEOUT, null, "The test was interrupted."), + (System.nanoTime() - started) / 1_000_000, pricing)); return (new Result(STATUS_ERROR, null, null, "The test was interrupted.")); } + finally + { + if(handle != null) + { + handle.cancel(); + } + } + } + + + + /******************************************************************************* + ** Central health uses the same probe without changing manual lastTest fields. + ** Purpose is a string, so health calls reuse TEST rather than adding an enum. + *******************************************************************************/ + public static Result checkModel(AssistantRuntime runtime, AssistantModel model, Instant deadline) throws QException + { + if(!Instant.now().isBefore(deadline) || Thread.currentThread().isInterrupted()) + { + return (new Result(STATUS_ERROR, null, null, "The test deadline passed.")); + } + if(!model.isChat()) + { + return (testEmbedding(runtime, model)); + } + AssistantProvider row = runtime.getStore().getProvider(model.getProviderId()); + var type = runtime.getConfig().getProviderTypes().get(row.getProviderType()); + var settings = runtime.getProviderService().settings(row); + int seconds = (int) Math.max(1, Math.min(30, Duration.between(Instant.now(), deadline).toSeconds())); + settings.withConnectTimeoutSeconds(Math.min(settings.getConnectTimeoutSeconds(), seconds)) + .withRequestTimeoutSeconds(Math.min(settings.getRequestTimeoutSeconds(), seconds)); + var provider = new RegistryModelProvider(type.create(settings), model); + return (call(runtime, new ProviderService.Active(row, row.getProviderType(), model.getModelId(), provider), + AssistantSecurity.userId(QContext.getQSession()), deadline, false)); } @@ -246,7 +304,7 @@ public final class ConnectionTest try { new EmbeddingService(runtime).embed(model.getId(), List.of("Connection test."), - DataClassification.PUBLIC, "", null, null); + DataClassification.PUBLIC, "", null, null, UsageRecorder.PURPOSE_TEST); return (new Result(STATUS_OK, (int) ((System.nanoTime() - started) / 1_000_000), model.getModelId(), null)); } catch(QException e) diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java index 9618395..d135322 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java @@ -481,13 +481,23 @@ public class ProviderService ** micro-dollars, from the usage rows the session can see. *******************************************************************************/ public long monthToDateCostMicros(Integer providerId) throws QException + { + return (monthToDateCostMicros(providerId, false)); + } + + + + /******************************************************************************* + ** Shared accounting permits health-runner caps without granting owner access. + *******************************************************************************/ + public long monthToDateCostMicros(Integer providerId, Boolean sharedAccounting) throws QException { if(providerId == null) { return (0); } Instant monthStart = LocalDate.now(clock.withZone(ZoneOffset.UTC)).withDayOfMonth(1).atStartOfDay().toInstant(ZoneOffset.UTC); - return (store.sumUsageCostSince(providerId, monthStart)); + return (store.sumUsageCostSince(providerId, monthStart, sharedAccounting)); } diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/persistence/AssistantStore.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/persistence/AssistantStore.java index 712a31c..43b9b1f 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/persistence/AssistantStore.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/persistence/AssistantStore.java @@ -739,11 +739,22 @@ public class AssistantStore ** The cost (micro-dollars) of a provider's calls since a time. *******************************************************************************/ public long sumUsageCostSince(Integer providerId, Instant since) throws QException + { + return (sumUsageCostSince(providerId, since, false)); + } + + + + /******************************************************************************* + ** Automated health uses the existing private accounting projection so a cap + ** includes all users, without reading their owner-locked usage records. + *******************************************************************************/ + public long sumUsageCostSince(Integer providerId, Instant since, Boolean sharedAccounting) throws QException { long total = 0; - for(QRecord record : queryRecords(AssistantUsage.TABLE_NAME, new QQueryFilter() + for(QRecord record : queryRecords(sharedAccounting ? AssistantBudgetUsage.TABLE_NAME : AssistantUsage.TABLE_NAME, new QQueryFilter() .withCriteria("providerId", QCriteriaOperator.EQUALS, providerId) - .withCriteria("createDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, since), MAX_SUM_ROWS)) + .withCriteria(sharedAccounting ? "occurredAt" : "createDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, since), MAX_SUM_ROWS)) { Long cost = record.getValueLong("costMicros"); total += cost == null ? 0 : cost; diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/rag/EmbeddingService.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/rag/EmbeddingService.java index 1d3eb68..d078dc5 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/rag/EmbeddingService.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/rag/EmbeddingService.java @@ -64,11 +64,21 @@ public class EmbeddingService ** including failures, and updates the existing daily budget rollups. *******************************************************************************/ public List embed(Integer modelId, List texts, DataClassification classification, String tables, Integer conversationId, Integer turnId) throws QException + { + return (embed(modelId, texts, classification, tables, conversationId, turnId, UsageRecorder.PURPOSE_EMBED)); + } + + + + /******************************************************************************* + ** Connection diagnostics use TEST accounting with the same embedding guards. + *******************************************************************************/ + public List embed(Integer modelId, List texts, DataClassification classification, String tables, Integer conversationId, Integer turnId, String purpose) throws QException { List vectors = new ArrayList<>(); for(Integer offset = 0; offset < texts.size(); offset += 64) { - vectors.addAll(batch(modelId, texts.subList(offset, Math.min(offset + 64, texts.size())), classification, tables, conversationId, turnId)); + vectors.addAll(batch(modelId, texts.subList(offset, Math.min(offset + 64, texts.size())), classification, tables, conversationId, turnId, purpose)); } return (vectors); } @@ -104,7 +114,7 @@ public class EmbeddingService /******************************************************************************* ** Never downgrade an embedding call into a different vector space. *******************************************************************************/ - private List batch(Integer modelId, List texts, DataClassification classification, String tables, Integer conversationId, Integer turnId) throws QException + private List batch(Integer modelId, List texts, DataClassification classification, String tables, Integer conversationId, Integer turnId, String purpose) throws QException { String userId = AssistantSecurity.userId(QContext.getQSession()); ScheduledTurn.checkCost(runtime, conversationId, turnId); @@ -140,6 +150,10 @@ public class EmbeddingService ModelProviderException error = null; try { + if(Thread.currentThread().isInterrupted()) + { + throw (new ModelProviderException(ModelProviderException.Kind.TIMEOUT, null, "Embedding request interrupted.")); + } EmbeddingResponse embedded = active.provider().embed(new EmbeddingRequest(active.model(), texts)); response = new ModelResponse().withModel(active.model()).withRequestId(embedded.requestId()).withUsage(embedded.usage()); if(embedded.vectors().size() != texts.size()) @@ -160,7 +174,7 @@ public class EmbeddingService gate.release(); } var pricing = runtime.getProviderService().pricing(active, active.model()); - runtime.getStore().insert(UsageRecorder.row(new UsageRecorder.Call(userId, conversationId, turnId, active, UsageRecorder.PURPOSE_EMBED), + runtime.getStore().insert(UsageRecorder.row(new UsageRecorder.Call(userId, conversationId, turnId, active, purpose), response, error, (System.nanoTime() - started) / 1_000_000, pricing)); var usage = response == null ? (error == null ? null : error.getPartialUsage()) : response.getUsage(); if(usage != null) -- 2.54.0 From 969a7b910178433ac00ac7e978b51961a0e354c7 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 13:39:45 -0500 Subject: [PATCH 3/5] feat(health): register provider and model checks; breaker reuses the central result Co-Authored-By: Claude Opus 5.5 --- .../assistant/AssistantQBitProducer.java | 4 + .../assistant/engine/ProviderHealth.java | 17 + .../health/AssistantHealthChecks.java | 279 ++++++++ .../health/AssistantHealthWidget.java | 79 +++ .../health/AssistantHealthChecksTest.java | 617 ++++++++++++++++++ 5 files changed, 996 insertions(+) create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthWidget.java create mode 100644 qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitProducer.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitProducer.java index 101b4e9..ce7b659 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitProducer.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitProducer.java @@ -23,6 +23,8 @@ import java.util.List; import java.util.Properties; import com.kingsrook.qbits.assistant.api.AssistantRoutes; import com.kingsrook.qbits.assistant.developer.AssistantDeveloperPanels; +import com.kingsrook.qbits.assistant.health.AssistantHealthChecks; +import com.kingsrook.qbits.assistant.health.AssistantHealthWidget; import com.kingsrook.qbits.assistant.home.AssistantAccessWidget; import com.kingsrook.qbits.assistant.home.AssistantHome; import com.kingsrook.qbits.assistant.metadata.AssistantChannelWall; @@ -198,6 +200,7 @@ public class AssistantQBitProducer implements QBitMetaDataProducer probe = new FutureTask<>(() -> runtime.getConfig().getProviderTypes().get(row.getProviderType()).checkHealth(settings)); Thread.ofVirtual().name("assistant-health").start(probe); diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java new file mode 100644 index 0000000..9b1590c --- /dev/null +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java @@ -0,0 +1,279 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.health; + + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.assistant.AssistantRuntime; +import com.kingsrook.qbits.assistant.AssistantSecurity; +import com.kingsrook.qbits.assistant.engine.ConnectionTest; +import com.kingsrook.qbits.assistant.engine.ProviderService; +import com.kingsrook.qbits.assistant.model.AssistantModel; +import com.kingsrook.qbits.assistant.model.AssistantProvider; +import com.kingsrook.qbits.assistant.spi.HealthResult; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCategory; +import com.kingsrook.qqq.backend.core.health.HealthCheck; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthCheckKind; +import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; +import com.kingsrook.qqq.backend.core.health.HealthCheckSupplier; +import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.health.HealthDetails; +import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; + + +/******************************************************************************* + ** Central health adapters reuse the existing probes without exposing their text. + *******************************************************************************/ +public final class AssistantHealthChecks +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private AssistantHealthChecks() + { + } + + + + /******************************************************************************* + ** Register dynamic namespaces once, even if production is repeated. + *******************************************************************************/ + public static void register(QInstance instance) + { + HealthChecksMetaData registry = HealthChecksMetaData.ofOrWithNew(instance); + register(registry, "assistant.provider", Providers.class); + register(registry, "assistant.model", Models.class); + } + + + + /******************************************************************************* + ** Suppliers carry the component; individual definitions also declare it. + *******************************************************************************/ + private static void register(HealthChecksMetaData registry, String prefix, Class supplier) + { + if(registry.getSuppliers().stream().noneMatch(existing -> prefix.equals(existing.getNamePrefix()))) + { + registry.withSupplier(new HealthCheckSupplierMetaData().withNamePrefix(prefix).withComponent("assistant").withSupplier(new QCodeReference(supplier))); + } + } + + + + /******************************************************************************* + ** Provider and model ids are integer primary keys, hence already lowercase. + *******************************************************************************/ + private static HealthCheckMetaData definition(String name, Integer interval, Integer timeout, HealthCheck check) + { + return (new HealthCheckMetaData().withName(name).withLabel(name).withComponent("assistant") + .withKind(HealthCheckKind.DEPENDENCY).withCategory(HealthCategory.AI) + .withIntervalSeconds(interval).withTimeoutMs(timeout).withCheckInstance(check)); + } + + + + /******************************************************************************* + ** One definition per enabled provider; no I/O to providers during resolution. + *******************************************************************************/ + public static class Providers implements HealthCheckSupplier + { + /*************************************************************************** + ** Re-resolve rows when the central registry refreshes its suppliers. + ***************************************************************************/ + @Override + public List supply(QInstance instance) throws QException + { + AssistantRuntime runtime = AssistantRuntime.of(instance); + List checks = new ArrayList<>(); + for(AssistantProvider row : runtime.getStore().listProviders()) + { + if(Boolean.TRUE.equals(row.getEnabled())) + { + checks.add(definition("assistant.provider." + row.getId(), "LOCAL".equals(runtime.getProviderService().location(row)) ? 60 : 300, 10000, + input -> bounded(input, () -> provider(runtime, row.getId(), input)))); + } + } + return (checks); + } + } + + + + /******************************************************************************* + ** One definition per enabled model, including models of disabled providers. + *******************************************************************************/ + public static class Models implements HealthCheckSupplier + { + /*************************************************************************** + ** Eligibility is checked again when a scheduled check actually executes. + ***************************************************************************/ + @Override + public List supply(QInstance instance) throws QException + { + AssistantRuntime runtime = AssistantRuntime.of(instance); + List checks = new ArrayList<>(); + for(AssistantModel row : runtime.getStore().listModels()) + { + if(Boolean.TRUE.equals(row.getEnabled())) + { + checks.add(definition("assistant.model." + row.getId(), 21600, 30000, + input -> bounded(input, () -> model(runtime, row.getId(), input)))); + } + } + return (checks); + } + } + + + + /******************************************************************************* + ** Clamp transport timeouts and bound even an adapter that ignores them. + ** Keep the caller's session; never create or elevate a health-check session. + *******************************************************************************/ + private static HealthCheckOutcome bounded(HealthCheckInput input, Callable call) + { + long remaining = Duration.between(Instant.now(), input.getDeadline()).toMillis(); + if(remaining <= 0) + { + return (HealthCheckOutcome.down("Health check timed out")); + } + var session = QContext.getQSession(); + FutureTask task = new FutureTask<>(() -> + { + QContext.init(input.getQInstance(), session); + try + { + return (call.call()); + } + finally + { + QContext.clear(); + } + }); + Thread.ofVirtual().name("assistant-central-health").start(task); + try + { + return (task.get(remaining, TimeUnit.MILLISECONDS)); + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + return (HealthCheckOutcome.down("Health check interrupted")); + } + catch(java.util.concurrent.TimeoutException e) + { + return (HealthCheckOutcome.down("Health check timed out")); + } + catch(java.util.concurrent.ExecutionException e) + { + return (HealthCheckOutcome.down("Health check failed")); + } + finally + { + task.cancel(true); + } + } + + + + /******************************************************************************* + ** Unsupported probes may read recent traffic-produced breaker state only. + *******************************************************************************/ + private static HealthCheckOutcome provider(AssistantRuntime runtime, Integer id, HealthCheckInput input) throws QException + { + AssistantProvider row = runtime.getStore().getProvider(id); + if(row == null || !Boolean.TRUE.equals(row.getEnabled())) + { + return (HealthCheckOutcome.unknown("Provider disabled or unavailable")); + } + var type = runtime.getConfig().getProviderTypes().get(row.getProviderType()); + if(type == null) + { + return (HealthCheckOutcome.unknown("Provider type unavailable")); + } + var settings = runtime.getProviderService().settings(row); + int seconds = (int) Math.max(1, Math.min(10, Duration.between(Instant.now(), input.getDeadline()).toSeconds())); + settings.withConnectTimeoutSeconds(Math.min(settings.getConnectTimeoutSeconds(), seconds)) + .withRequestTimeoutSeconds(Math.min(settings.getRequestTimeoutSeconds(), seconds)); + HealthResult result = type.checkHealth(settings); + HealthCheckOutcome outcome; + if(result.healthy() == null) + { + Instant checked = row.getHealthCheckedAt(); + Instant now = runtime.getClock().instant(); + if(checked != null && !checked.isAfter(now) && checked.plusSeconds(600).isAfter(now) && List.of("UP", "DOWN", "DEGRADED").contains(row.getHealthState() == null ? "" : row.getHealthState())) + { + outcome = new HealthCheckOutcome().withStatus(HealthStatus.valueOf(row.getHealthState())).withMessage("Recent circuit-breaker state"); + } + else + { + outcome = HealthCheckOutcome.unknown("health probe not supported"); + } + } + else + { + outcome = result.healthy() ? HealthCheckOutcome.up("Provider reachable") : HealthCheckOutcome.down("Provider health probe failed"); + } + return (outcome.withLatencyMs(result.latencyMs()).withDetails(new HealthDetails().withValue("providerId", id))); + } + + + + /******************************************************************************* + ** Reuse the minimal connection call and its accounting, without lastTest writes. + *******************************************************************************/ + private static HealthCheckOutcome model(AssistantRuntime runtime, Integer id, HealthCheckInput input) throws QException + { + AssistantModel row = runtime.getStore().listModels().stream().filter(model -> id.equals(model.getId())).findFirst().orElse(null); + if(row == null || !Boolean.TRUE.equals(row.getEnabled())) + { + return (HealthCheckOutcome.unknown("Model disabled or unavailable")); + } + AssistantProvider provider = runtime.getStore().getProvider(row.getProviderId()); + if(provider == null || !Boolean.TRUE.equals(provider.getEnabled())) + { + return (HealthCheckOutcome.unknown("Provider disabled or unavailable")); + } + Long cap = ProviderService.capMicros(provider); + if(cap != null && runtime.getProviderService().monthToDateCostMicros(provider.getId(), true) >= cap) + { + return (HealthCheckOutcome.unknown("skipped: cost cap")); + } + if(runtime.getPolicyService().current(true).killSwitch()) + { + return (HealthCheckOutcome.unknown("skipped: kill switch")); + } + AssistantSecurity.ownerKey(QContext.getQSession(), runtime.getConfig()); + ConnectionTest.Result result = ConnectionTest.checkModel(runtime, row, input.getDeadline()); + return ((result.ok() ? HealthCheckOutcome.up("Model connection succeeded") : HealthCheckOutcome.down("Model connection failed")) + .withDetails(new HealthDetails().withValue("modelId", id).withValue("providerId", provider.getId())) + .withLatencyMs(result.latencyMs() == null ? null : result.latencyMs().longValue())); + } +} diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthWidget.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthWidget.java new file mode 100644 index 0000000..0f513ea --- /dev/null +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthWidget.java @@ -0,0 +1,79 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.health; + + +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.assistant.metadata.AssistantMetaDataBuilder; +import com.kingsrook.qbits.assistant.tools.FilterLinkBuilder; +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.health.HealthRunner; +import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; + + +/******************************************************************************* + ** A single generic status card links to the central checks, without probing. + *******************************************************************************/ +public class AssistantHealthWidget extends AbstractWidgetRenderer +{ + public static final String NAME = "assistantHealth"; + + + + /******************************************************************************* + ** No HTML, history or per-provider check list on the Assistant home. + *******************************************************************************/ + public static QWidgetMetaData widget() + { + return (new QWidgetMetaData().withName(NAME).withLabel("Assistant health") + .withType(WidgetType.STATISTICS.getType()).withCodeReference(new QCodeReference(AssistantHealthWidget.class)) + .withGridColumns(12).withPermissionRules(AssistantMetaDataBuilder.adminRules())); + } + + + + /******************************************************************************* + ** Summarize the runner's latest results; full history lives in System Health. + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + var instance = QContext.getQInstance(); + HealthRunner runner = HealthRunner.getForInstance(instance); + HealthStatus status = HealthStatus.UNKNOWN; + if(runner != null) + { + status = HealthStatus.aggregate(HealthChecksMetaData.ofOrWithNew(instance).resolveAll(instance).stream() + .filter(check -> "assistant".equals(check.getComponent())) + .map(check -> runner.getLastOutcome(check.getName())) + .map(outcome -> outcome == null ? HealthStatus.UNKNOWN : outcome.getStatus()).toArray(HealthStatus[]::new)); + } + return (new RenderWidgetOutput(new StatisticsData().withCount(status.name()) + .withCountContext("System Health") + .withCountURL(FilterLinkBuilder.href("health_check", List.of(Map.of("fieldName", "component", "operator", "EQUALS", "values", List.of("assistant"))))))); + } +} diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java new file mode 100644 index 0000000..d8c3d06 --- /dev/null +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java @@ -0,0 +1,617 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.health; + + +import java.math.BigDecimal; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.assistant.AssistantQBitConfig; +import com.kingsrook.qbits.assistant.engine.ProviderHealth; +import com.kingsrook.qbits.assistant.engine.ProviderService; +import com.kingsrook.qbits.assistant.engine.UsageRecorder; +import com.kingsrook.qbits.assistant.model.AssistantModel; +import com.kingsrook.qbits.assistant.model.AssistantPolicy; +import com.kingsrook.qbits.assistant.model.AssistantProvider; +import com.kingsrook.qbits.assistant.model.AssistantUsage; +import com.kingsrook.qbits.assistant.provider.fake.ScriptedModelProvider; +import com.kingsrook.qbits.assistant.spi.AssistantModelProvider; +import com.kingsrook.qbits.assistant.spi.AssistantProviderType; +import com.kingsrook.qbits.assistant.spi.EmbeddingRequest; +import com.kingsrook.qbits.assistant.spi.EmbeddingResponse; +import com.kingsrook.qbits.assistant.spi.HealthResult; +import com.kingsrook.qbits.assistant.spi.ModelProviderException; +import com.kingsrook.qbits.assistant.spi.ModelUsage; +import com.kingsrook.qbits.assistant.spi.ProviderSettings; +import com.kingsrook.qbits.assistant.testing.AssistantTestFixture; +import com.kingsrook.qbits.assistant.testing.BaseAssistantTest; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.health.HealthCategory; +import com.kingsrook.qqq.backend.core.health.HealthCheckInput; +import com.kingsrook.qqq.backend.core.health.HealthCheckKind; +import com.kingsrook.qqq.backend.core.health.HealthCheckMetaData; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; +import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; +import com.kingsrook.qqq.backend.core.health.HealthRunner; +import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData; +import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSession; +import com.kingsrook.qqq.backend.core.model.session.QUser; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; + + +/******************************************************************************* + ** Registration, deadlines, billing and secret safety through the memory backend. + *******************************************************************************/ +class AssistantHealthChecksTest extends BaseAssistantTest +{ + private static final String SECRET = "S3NT1NEL-secret"; + private final FakeType type = new FakeType(); + private Boolean healthy = true; + private Boolean failProbe = false; + private Boolean failCreate = false; + private Boolean failEmbedding = false; + private Integer probes = 0; + private Integer embeddings = 0; + private ProviderSettings lastSettings; + private CountDownLatch probeWait; + private final CountDownLatch interrupted = new CountDownLatch(1); + + + + /******************************************************************************* + ** Resolve a poisoned secret through the normal key-reference mechanism. + *******************************************************************************/ + @Override + protected void customizeConfig(AssistantQBitConfig config) + { + provider = new EmbeddingProvider(); + config.withModelProvider(provider).withProviderType(type).withAllowedApiKeyEnvNames(Set.of("HEALTH_TEST_KEY")) + .withSecretResolver(name -> SECRET); + } + + + + /******************************************************************************* + ** An adapter with no network and controllable provider failures. + *******************************************************************************/ + class FakeType implements AssistantProviderType + { + /*************************************************************************** + ** Fake provider contract. + ***************************************************************************/ + @Override + public String getType() + { + return ("health-fake"); + } + + + + /*************************************************************************** + ** Fake provider contract. + ***************************************************************************/ + @Override + public String getLabel() + { + return ("Health fake"); + } + + + + /*************************************************************************** + ** Fake provider contract. + ***************************************************************************/ + @Override + public List getModels() + { + return (List.of("model")); + } + + + + /*************************************************************************** + ** Fake provider contract. + ***************************************************************************/ + @Override + public String getDefaultApiKeyEnvName() + { + return ("HEALTH_TEST_KEY"); + } + + + + /*************************************************************************** + ** Fake provider contract. + ***************************************************************************/ + @Override + public String getDefaultBaseUrl() + { + return ("https://health.invalid"); + } + + + + /*************************************************************************** + ** Fake provider contract. + ***************************************************************************/ + @Override + public String getDefaultLocation(String kind) + { + return ("LOCAL"); + } + + + + /*************************************************************************** + ** Reuse the existing scripted completion fake. + ***************************************************************************/ + @Override + public AssistantModelProvider create(String key, String url) + { + assertThat(key).isEqualTo(SECRET); + if(failCreate) + { + throw (new IllegalStateException(SECRET)); + } + return (provider); + } + + + + /*************************************************************************** + ** Verify bounded settings and exercise hostile exception messages. + ***************************************************************************/ + @Override + public HealthResult checkHealth(ProviderSettings settings) + { + probes++; + lastSettings = settings; + assertThat(settings.getApiKey()).isEqualTo(SECRET); + if(failProbe) + { + throw (new IllegalStateException(SECRET)); + } + if(probeWait != null) + { + try + { + probeWait.await(); + } + catch(InterruptedException e) + { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + } + return (new HealthResult(healthy, 7L, SECRET)); + } + } + + + + /******************************************************************************* + ** Keep embedding tests on the existing scripted-provider fixture as well. + *******************************************************************************/ + class EmbeddingProvider extends ScriptedModelProvider + { + /*************************************************************************** + ** Minimal embedding result; no external service is contacted. + ***************************************************************************/ + @Override + public EmbeddingResponse embed(EmbeddingRequest request) throws ModelProviderException + { + embeddings++; + if(failEmbedding) + { + throw (new ModelProviderException(ModelProviderException.Kind.NETWORK, null, SECRET)); + } + return (new EmbeddingResponse(List.of(new float[] { 1, 0 }), new ModelUsage().withInputTokens(2L), "fake")); + } + } + + + + /******************************************************************************* + ** Set up real settings records in the existing admin fixture session. + *******************************************************************************/ + private AssistantProvider row() throws Exception + { + QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN)); + return (config.getRuntime().getStore().insert(new AssistantProvider().withName("Health test") + .withProviderType(type.getType()).withEnabled(true))); + } + + + + /******************************************************************************* + ** Enabled chat model with a stable integer id. + *******************************************************************************/ + private AssistantModel model(AssistantProvider row) throws Exception + { + return (config.getRuntime().getStore().insert(new AssistantModel().withProviderId(row.getId()).withModelId("model").withEnabled(true))); + } + + + + /******************************************************************************* + ** Exercise the producer-installed suppliers, not a separately constructed check. + *******************************************************************************/ + private HealthCheckMetaData check(String name) throws Exception + { + return (HealthChecksMetaData.ofOrWithNew(qInstance).resolveAll(qInstance).stream() + .filter(check -> name.equals(check.getName())).findFirst().orElseThrow()); + } + + + + /******************************************************************************* + ** Raw outcomes must be clean; core's sanitize method is package-private. + *******************************************************************************/ + private HealthCheckOutcome run(HealthCheckMetaData check) throws Exception + { + HealthCheckOutcome outcome = check.getCheckInstance().run(new HealthCheckInput().withQInstance(qInstance).withDeadline(Instant.now().plusSeconds(5))); + assertThat(outcome.getMessage()).doesNotContain(SECRET, "health.invalid", "Connection test.", "\n"); + assertThat(outcome.getDetails().toJson()).doesNotContain(SECRET, "health.invalid", "Connection test."); + return (outcome); + } + + + + /******************************************************************************* + ** Both suppliers are installed by production, refresh enabled sets and use AI. + *******************************************************************************/ + @Test + void registration() throws Exception + { + AssistantProvider local = row(); + AssistantProvider cloud = config.getRuntime().getStore().insert(new AssistantProvider().withName("Cloud") + .withProviderType(type.getType()).withEnabled(true).withLocation("CLOUD")); + AssistantModel model = model(local); + model(cloud); + config.getRuntime().getStore().insert(new AssistantProvider().withName("Disabled").withProviderType(type.getType()).withEnabled(false)); + config.getRuntime().getStore().insert(new AssistantModel().withProviderId(local.getId()).withModelId("disabled").withEnabled(false)); + var registry = HealthChecksMetaData.ofOrWithNew(qInstance); + AssistantHealthChecks.register(qInstance); + assertThat(registry.getSuppliers()).hasSize(2).allSatisfy(supplier -> assertThat(supplier.getComponent()).isEqualTo("assistant")); + assertThat(registry.resolveAll(qInstance)).hasSize(4).allSatisfy(check -> + { + assertThat(check.getKind()).isEqualTo(HealthCheckKind.DEPENDENCY); + assertThat(check.getCategory()).isEqualTo(HealthCategory.AI); + assertThat(check.getComponent()).isEqualTo("assistant"); + assertThat(check.getName()).matches("[a-z0-9]+([._-][a-z0-9]+)*"); + }); + assertThat(check("assistant.provider." + local.getId()).getIntervalSeconds()).isEqualTo(60); + assertThat(check("assistant.provider." + cloud.getId()).getIntervalSeconds()).isEqualTo(300); + var modelCheck = check("assistant.model." + model.getId()); + assertThat(modelCheck.getIntervalSeconds()).isEqualTo(21600); + assertThat(modelCheck.getTimeoutMs()).isEqualTo(30000); + assertThat(check("assistant.provider." + local.getId()).getTimeoutMs()).isEqualTo(10000); + config.getRuntime().getStore().updateFields(AssistantModel.TABLE_NAME, new QRecord().withValue("id", model.getId()).withValue("enabled", false)); + assertThat(registry.resolveAll(qInstance)).hasSize(3); + assertThat(run(modelCheck).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + } + + + + /******************************************************************************* + ** HealthResult messages and thrown adapter messages never enter outcomes. + *******************************************************************************/ + @Test + void providerStatesAndNoSecrets() throws Exception + { + AssistantProvider row = row(); + var check = check("assistant.provider." + row.getId()); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.UP); + assertThat(lastSettings.getRequestTimeoutSeconds()).isLessThanOrEqualTo(5); + healthy = false; + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DOWN); + failProbe = true; + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DOWN); + failProbe = false; + healthy = null; + assertThat(run(check).getMessage()).isEqualTo("health probe not supported"); + for(String state : List.of("UP", "DOWN", "DEGRADED")) + { + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", row.getId()) + .withValue("healthState", state).withValue("healthCheckedAt", Instant.now().minusSeconds(30))); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.valueOf(state)); + } + for(Instant time : List.of(Instant.now().minusSeconds(600), Instant.now().plusSeconds(30))) + { + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", row.getId()).withValue("healthCheckedAt", time)); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + } + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", row.getId()).withValue("enabled", false)); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + } + + + + /******************************************************************************* + ** Missing adapter/configuration is safe and does not dispatch a completion. + *******************************************************************************/ + @Test + void missingAdapterAndSecret() throws Exception + { + AssistantProvider row = row(); + var check = check("assistant.provider." + row.getId()); + config.withSecretResolver(name -> null); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DOWN); + config.getProviderTypes().remove(type.getType()); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + } + + + + /******************************************************************************* + ** Successful and failed completions use TEST and preserve manual test history. + *******************************************************************************/ + @Test + void modelAccountingAndNoSecrets() throws Exception + { + AssistantModel model = model(row()); + Instant previous = Instant.now().minusSeconds(1000); + config.getRuntime().getStore().updateFields(AssistantModel.TABLE_NAME, new QRecord().withValue("id", model.getId()) + .withValue("lastTestAt", previous).withValue("lastTestStatus", "OK")); + var check = check("assistant.model." + model.getId()); + provider.thenText(SECRET); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.UP); + assertThat(provider.getRequests().getFirst().getMaxTokens()).isEqualTo(32); + assertThat(probes).isZero(); + provider.thenError(ModelProviderException.Kind.NETWORK, SECRET); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DOWN); + var usage = config.getRuntime().getStore().queryRecords(AssistantUsage.TABLE_NAME, new QQueryFilter(), 10); + assertThat(usage).hasSize(2).allSatisfy(record -> assertThat(record.getValueString("purpose")).isEqualTo(UsageRecorder.PURPOSE_TEST)); + assertThat(config.getRuntime().getStore().listModels().getFirst().getLastTestAt()).isEqualTo(previous); + failCreate = true; + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DOWN); + } + + + + /******************************************************************************* + ** Monthly cap skips even on-demand checks, before any billable request. + *******************************************************************************/ + @Test + void modelCostCapAndDisabledProvider() throws Exception + { + AssistantProvider row = row(); + AssistantModel model = model(row); + var check = check("assistant.model." + model.getId()); + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", row.getId()).withValue("monthlyCostCapUsd", BigDecimal.ZERO)); + var outcome = check.getCheckInstance().run(new HealthCheckInput().withQInstance(qInstance).withDeadline(Instant.now().plusSeconds(5)).withOnDemand(true)); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.UNKNOWN); + assertThat(outcome.getMessage()).isEqualTo("skipped: cost cap"); + assertThat(provider.getRequests()).isEmpty(); + assertThat(config.getRuntime().getStore().queryRecords(AssistantUsage.TABLE_NAME, new QQueryFilter(), 10)).isEmpty(); + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", row.getId()).withValue("enabled", false)); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.UNKNOWN); + } + + + + /******************************************************************************* + ** The real runner identity records only its own usage and sees shared caps. + *******************************************************************************/ + @Test + void automatedSessionAccountingAndSharedCap() throws Exception + { + AssistantProvider row = row(); + AssistantModel model = model(row); + var check = check("assistant.model." + model.getId()); + QContext.init(qInstance, new QSession().withUser(new QUser().withIdReference("health_runner")) + .withSecurityKeyValues(Map.of()).withPermissions("health.read", "health.write")); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.UP); + assertThat(QContext.getQSession().getSecurityKeyValues()).containsOnlyKeys(config.getOwnerSecurityKeyType()); + assertThat(QContext.getQSession().hasPermission("assistant.admin")).isFalse(); + var usage = config.getRuntime().getStore().queryRecords(AssistantUsage.TABLE_NAME, new QQueryFilter(), 10); + assertThat(usage).hasSize(1); + assertThat(usage.getFirst().getValueString("userId")).isEqualTo("health_runner"); + var healthSession = QContext.getQSession(); + QContext.init(qInstance, AssistantTestFixture.ownerSession(AssistantTestFixture.ADMIN, config)); + config.getRuntime().getStore().insert(new AssistantUsage().withUserId(AssistantTestFixture.EDITOR).withProviderId(row.getId()) + .withCostMicros(1_000_000L).withCreateDate(Instant.now()).withPurpose("TURN").withStatus("OK")); + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", row.getId()).withValue("monthlyCostCapUsd", BigDecimal.ONE)); + QContext.init(qInstance, healthSession); + assertThat(config.getRuntime().getProviderService().monthToDateCostMicros(row.getId())).isZero(); + assertThat(run(check).getMessage()).isEqualTo("skipped: cost cap"); + assertThat(provider.getRequests()).hasSize(1); + } + + + + /******************************************************************************* + ** The kill switch is read fresh before every scheduled model call. + *******************************************************************************/ + @Test + void modelKillSwitch() throws Exception + { + var check = check("assistant.model." + model(row()).getId()); + config.getRuntime().getStore().insert(new AssistantPolicy().withName("Stop").withKillSwitch(true)); + assertThat(run(check).getMessage()).isEqualTo("skipped: kill switch"); + assertThat(provider.getRequests()).isEmpty(); + } + + + + /******************************************************************************* + ** Embedding models reuse ConnectionTest's embedding path, never chat. + *******************************************************************************/ + @Test + void embeddingAccountingAndNoSecrets() throws Exception + { + AssistantProvider row = row(); + AssistantModel model = config.getRuntime().getStore().insert(new AssistantModel().withProviderId(row.getId()).withModelId("embedding") + .withModelKind("EMBEDDING").withEnabled(true)); + var check = check("assistant.model." + model.getId()); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.UP); + failEmbedding = true; + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(embeddings).isEqualTo(2); + assertThat(provider.getRequests()).isEmpty(); + assertThat(config.getRuntime().getStore().queryRecords(AssistantUsage.TABLE_NAME, new QQueryFilter(), 10)) + .hasSize(2).allSatisfy(record -> assertThat(record.getValueString("purpose")).isEqualTo("TEST")); + } + + + + /******************************************************************************* + ** Expired input performs no work; a stuck existing probe is interrupted. + *******************************************************************************/ + @Test + void providerDeadline() throws Exception + { + var check = check("assistant.provider." + row().getId()); + assertThat(check.getCheckInstance().run(new HealthCheckInput().withQInstance(qInstance).withDeadline(Instant.now().minusSeconds(1))).getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(probes).isZero(); + probeWait = new CountDownLatch(1); + assertTimeoutPreemptively(Duration.ofSeconds(3), () -> + { + QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN)); + try + { + var outcome = check.getCheckInstance().run(new HealthCheckInput().withQInstance(qInstance).withDeadline(Instant.now().plusMillis(200))); + assertThat(outcome.getMessage()).isEqualTo("Health check timed out"); + assertThat(interrupted.await(1, TimeUnit.SECONDS)).isTrue(); + } + finally + { + QContext.clear(); + } + }); + } + + + + /******************************************************************************* + ** A slow completion cannot outlive the caller's deadline. + *******************************************************************************/ + @Test + void modelDeadline() throws Exception + { + var check = check("assistant.model." + model(row()).getId()); + provider.withDelay(Duration.ofSeconds(10)).thenText("OK"); + assertTimeoutPreemptively(Duration.ofSeconds(3), () -> + { + QContext.init(qInstance, AssistantTestFixture.sessionFor(AssistantTestFixture.ADMIN)); + try + { + var outcome = check.getCheckInstance().run(new HealthCheckInput().withQInstance(qInstance).withDeadline(Instant.now().plusMillis(300))); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN); + } + finally + { + QContext.clear(); + } + }); + } + + + + /******************************************************************************* + ** Drive the real runner cache without its scheduler or health-table writes. + *******************************************************************************/ + @SuppressWarnings("unchecked") + private HealthRunner cached(String name, HealthCheckOutcome outcome, Instant at) throws Exception + { + HealthRunner runner = new HealthRunner(qInstance, "test", Clock.fixed(at, ZoneOffset.UTC)); + var cache = HealthRunner.class.getDeclaredMethod("cacheOutcome", String.class, HealthCheckOutcome.class); + cache.setAccessible(true); + cache.invoke(runner, name, outcome); + var runners = HealthRunner.class.getDeclaredField("RUNNERS"); + runners.setAccessible(true); + ((Map) runners.get(null)).put(qInstance, runner); + return (runner); + } + + + + /******************************************************************************* + ** Fresh central results avoid duplicate local probes; stale/unknown fall back. + *******************************************************************************/ + @Test + void breakerUsesCentralResults() throws Exception + { + AssistantProvider row = row(); + var active = new ProviderService.Active(row, type.getType(), "model", provider); + ProviderHealth breaker = new ProviderHealth(config.getRuntime()); + String name = "assistant.provider." + row.getId(); + for(HealthStatus status : List.of(HealthStatus.UP, HealthStatus.DEGRADED, HealthStatus.DOWN)) + { + try(var central = cached(name, new HealthCheckOutcome().withStatus(status), Instant.now().minusSeconds(1))) + { + if(status == HealthStatus.DOWN) + { + assertThatThrownBy(() -> breaker.beforeCall(active)).isInstanceOf(ModelProviderException.class); + } + else + { + breaker.beforeCall(active); + } + assertThat(probes).isZero(); + } + } + try(var central = cached(name, HealthCheckOutcome.down("Down"), Instant.now().minusSeconds(60))) + { + breaker.beforeCall(active); + assertThat(probes).isEqualTo(1); + } + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", row.getId()).withValue("healthCheckedAt", null)); + try(var central = cached(name, HealthCheckOutcome.unknown("Unsupported"), Instant.now())) + { + breaker.beforeCall(active); + assertThat(probes).isEqualTo(2); + } + } + + + + /******************************************************************************* + ** The replacement card is generic and links to filtered central health. + *******************************************************************************/ + @Test + void centralHealthCard() throws Exception + { + AssistantProvider row = row(); + assertThat(qInstance.getWidget(AssistantHealthWidget.NAME).getType()).isEqualTo(WidgetType.STATISTICS.getType()); + var renderer = new AssistantHealthWidget(); + StatisticsData data = (StatisticsData) renderer.render(new RenderWidgetInput()).getWidgetData(); + assertThat(data.getCount()).isEqualTo("UNKNOWN"); + assertThat(data.getCountURL()).startsWith("/app/").contains("health_check", "assistant"); + try(var central = cached("assistant.provider." + row.getId(), HealthCheckOutcome.up("Reachable"), Instant.now())) + { + data = (StatisticsData) renderer.render(new RenderWidgetInput()).getWidgetData(); + assertThat(data.getCount()).isEqualTo("UP"); + } + } +} -- 2.54.0 From 303cf9a148e21135dd57901c359fcaa14b79ffc4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Wed, 14 Oct 2026 13:39:46 -0500 Subject: [PATCH 4/5] refactor(assistant): home shows the central health status instead of its own provider health view Co-Authored-By: Claude Opus 5.5 --- .../qbits/assistant/home/AssistantHome.java | 2 +- .../assistant/home/AssistantHomeView.java | 81 +++---------------- .../metadata/AssistantMetaDataBuilder.java | 3 +- .../assistant/AssistantQBitProducerTest.java | 2 +- .../assistant/engine/RoutingFallbackTest.java | 2 +- .../settings/AssistantSettingsTest.java | 2 +- 6 files changed, 19 insertions(+), 73 deletions(-) diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/home/AssistantHome.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/home/AssistantHome.java index d0ddda2..8e9e772 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/home/AssistantHome.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/home/AssistantHome.java @@ -32,7 +32,7 @@ import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; /******************************************************************************* ** The Assistant app home: one full-width HTML widget, the single owner of the - ** assistant's health, usage and cost data (no other dashboard repeats it). + ** assistant's configuration, usage and cost data. Health lives in System Health. *******************************************************************************/ public final class AssistantHome { diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/home/AssistantHomeView.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/home/AssistantHomeView.java index af42f07..0adf147 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/home/AssistantHomeView.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/home/AssistantHomeView.java @@ -17,16 +17,6 @@ package com.kingsrook.qbits.assistant.home; -import com.kingsrook.qbits.assistant.engine.BudgetService; -import com.kingsrook.qbits.assistant.metadata.AssistantRecordScreens; -import com.kingsrook.qbits.assistant.model.AssistantBudget; -import com.kingsrook.qbits.assistant.model.AssistantBudgetOverride; -import com.kingsrook.qbits.assistant.model.AssistantConversation; -import com.kingsrook.qbits.assistant.model.AssistantModelPrice; -import com.kingsrook.qbits.assistant.model.AssistantProvider; -import com.kingsrook.qbits.assistant.model.AssistantToolCall; -import com.kingsrook.qbits.assistant.model.AssistantUsage; -import com.kingsrook.qqq.backend.core.model.data.QRecord; import java.math.BigDecimal; import java.math.RoundingMode; import java.text.NumberFormat; @@ -38,6 +28,16 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; +import com.kingsrook.qbits.assistant.engine.BudgetService; +import com.kingsrook.qbits.assistant.metadata.AssistantRecordScreens; +import com.kingsrook.qbits.assistant.model.AssistantBudget; +import com.kingsrook.qbits.assistant.model.AssistantBudgetOverride; +import com.kingsrook.qbits.assistant.model.AssistantConversation; +import com.kingsrook.qbits.assistant.model.AssistantModelPrice; +import com.kingsrook.qbits.assistant.model.AssistantProvider; +import com.kingsrook.qbits.assistant.model.AssistantToolCall; +import com.kingsrook.qbits.assistant.model.AssistantUsage; +import com.kingsrook.qqq.backend.core.model.data.QRecord; /******************************************************************************* @@ -112,44 +112,17 @@ public final class AssistantHomeView /******************************************************************************* - ** The status row: is the assistant answering, with what, at what cost. + ** Configuration and usage; health status belongs to the central card. *******************************************************************************/ private static void status(StringBuilder html, AssistantHomeData data, String prefix) { - open(html, "Health"); + open(html, "Configuration and usage"); AssistantHomeData.ProviderHealth active = data.getProviders().stream().filter(AssistantHomeData.ProviderHealth::active).findFirst().orElse(null); - - html.append("
"); - if(data.isKillSwitch()) - { - html.append(chip(RED, "Turned off by the kill switch")); - } - else if(data.getProviders().isEmpty()) - { - html.append(chip(GREY, "No provider configured here (the application's built-in provider is used, if any)")); - } - else if(active == null) - { - html.append(chip(RED, "No provider is turned on")); - } - else if(!active.keyPresent() || !active.problems().isEmpty()) - { - html.append(chip(RED, "Default provider cannot be used")); - } - else if("ERROR".equals(active.row().getLastTestStatus())) - { - html.append(chip(AMBER, "Last connection test failed")); - } - else - { - html.append(chip(GREEN, "Answering")); - } - html.append("
"); + html.append("
"); cell(html, "Provider", active == null ? "None" : active.row().getName() + " (" + nvl(active.row().getModel(), "default model") + ")"); cell(html, "HIPAA mode", Boolean.TRUE.equals(data.getHipaaMode()) ? "On" : "Off"); cell(html, "PHI retention", data.getHipaaRetentionDays() == null ? "Normal retention" : data.getHipaaRetentionDays() + " days"); cell(html, "API key", active == null ? "Not applicable" : active.keyStatus()); - cell(html, "Last test", active == null ? "Not applicable" : lastTest(active.row())); Integer rate = data.errorRatePercent7d(); cell(html, "Error rate, 7 days", rate == null ? "No requests yet" : rate + "% of " + number(data.getCalls7d()) + " requests"); cell(html, "Cost this month", money(data.getMonthCostMicros()) + (data.getInstanceMonthlyCapMicros() == null ? "" : " of " + money(data.getInstanceMonthlyCapMicros()) + " limit")); @@ -187,17 +160,8 @@ public final class AssistantHomeView { html.append(chip(GREY, "Off")); } - if(row.getLastTestStatus() != null) - { - html.append(chip("OK".equals(row.getLastTestStatus()) ? GREEN : RED, "OK".equals(row.getLastTestStatus()) ? "Last test passed" : "Last test failed")); - } html.append("
"); cell(html, "Type", nvl(p.typeLabel(), "Not set")); - cell(html, "Health", nvl(row.getHealthState(), "Unknown")); - if(row.getDownUntil() != null) - { - cell(html, "Down until", when(row.getDownUntil())); - } cell(html, "Kind", AssistantRecordScreens.providerKindLabel(p.kind())); cell(html, "Location", "LOCAL".equals(p.location()) ? "Local" : "Cloud"); if(Boolean.TRUE.equals(row.getHipaaEligible())) @@ -207,7 +171,6 @@ public final class AssistantHomeView cell(html, "Enabled models", String.valueOf(p.enabledModels())); cell(html, "Model", nvl(row.getModel(), "Default model")); cell(html, "API key", p.keyStatus()); - cell(html, "Last test", lastTest(row)); cell(html, "Cost this month", money(p.monthCostMicros()) + (p.capMicros() == null ? ", no cap" : " of " + money(p.capMicros()) + " cap")); List limits = new ArrayList<>(); if(row.getMaxOutputTokensPerRequest() != null) @@ -455,24 +418,6 @@ public final class AssistantHomeView - /******************************************************************************* - ** A provider's last test in words. - *******************************************************************************/ - static String lastTest(AssistantProvider row) - { - if(row.getLastTestAt() == null) - { - return ("Not tested yet: run Test connection"); - } - if("OK".equals(row.getLastTestStatus())) - { - return ("Passed " + WHEN.format(row.getLastTestAt()) + ", " + nvl(row.getLastTestModel(), "model") + " in " + row.getLastTestLatencyMs() + " ms"); - } - return ("Failed " + WHEN.format(row.getLastTestAt()) + ": " + nvl(row.getLastTestError(), "no detail")); - } - - - /******************************************************************************* ** Micro-dollars as dollars ("$0.00", "<$0.01" for a little, "$12.34"). *******************************************************************************/ diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantMetaDataBuilder.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantMetaDataBuilder.java index c270244..657f7bc 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantMetaDataBuilder.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantMetaDataBuilder.java @@ -20,6 +20,7 @@ package com.kingsrook.qbits.assistant.metadata; import java.util.ArrayList; import java.util.List; import com.kingsrook.qbits.assistant.AssistantQBitConfig; +import com.kingsrook.qbits.assistant.health.AssistantHealthWidget; import com.kingsrook.qbits.assistant.home.AssistantAccessWidget; import com.kingsrook.qbits.assistant.home.AssistantHome; import com.kingsrook.qbits.assistant.model.AssistantBudget; @@ -383,7 +384,7 @@ public class AssistantMetaDataBuilder .withLabel("Assistant") .withIcon(new QIcon("smart_toy")) .withPermissionRules(adminRules()) - .withWidgets(List.of(AssistantHome.WIDGET_NAME, AssistantAccessWidget.WIDGET_NAME)); + .withWidgets(List.of(AssistantHealthWidget.NAME, AssistantHome.WIDGET_NAME, AssistantAccessWidget.WIDGET_NAME)); for(String name : NAV_TABLES) { tables.stream().filter(t -> t.getName().equals(name)).findFirst().ifPresent(app::withChild); diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java index df30caf..2cc2f97 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java @@ -91,7 +91,7 @@ class AssistantQBitProducerTest extends BaseAssistantTest ////////////////////////////////////////////////////////////////////// assertThat(qInstance.getApp(AssistantMetaDataBuilder.APP_NAME).getChildren()).extracting(c -> c.getName()) .containsExactly("assistant_scheduled_task", "assistant_rag_source", "assistant_budget", "assistant_provider", "assistant_profile", "assistant_skill", "assistant_data_access", "assistant_conversation", "assistant_model_price", "assistant_policy"); - assertThat(qInstance.getApp(AssistantMetaDataBuilder.APP_NAME).getWidgets()).containsExactly("assistantHome", "assistantAccess"); + assertThat(qInstance.getApp(AssistantMetaDataBuilder.APP_NAME).getWidgets()).containsExactly("assistantHealth", "assistantHome", "assistantAccess"); assertThat(qInstance.getProcess(AssistantMetaDataBuilder.RETENTION_PROCESS_NAME).getTableName()).isEqualTo("assistant_conversation"); assertThat(qInstance.getProcess(AssistantMetaDataBuilder.RETENTION_PROCESS_NAME)).isNotNull(); assertThat(qInstance.getProcess(AssistantMetaDataBuilder.RETENTION_PROCESS_NAME).getSchedule()).isNull(); diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/RoutingFallbackTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/RoutingFallbackTest.java index 2244982..132fed4 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/RoutingFallbackTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/RoutingFallbackTest.java @@ -676,7 +676,7 @@ class RoutingFallbackTest extends BaseAssistantTest assertThat(script("a").getRequests()).isEmpty(); assertThat(store().getProvider(a.getProviderId()).getHealthState()).isEqualTo("DOWN"); String html = AssistantHomeView.render(AssistantHomeData.load(config.getRuntime()), ""); - assertThat(html).contains("DOWN", "Down until", "0 fallbacks"); + assertThat(html).contains("0 fallbacks").doesNotContain("Down until", "Last test failed"); } diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/settings/AssistantSettingsTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/settings/AssistantSettingsTest.java index 7acfdbc..8bcb83b 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/settings/AssistantSettingsTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/settings/AssistantSettingsTest.java @@ -561,7 +561,7 @@ class AssistantSettingsTest extends BaseAssistantTest assertThat(data.getMonthCostMicros()).isZero(); String html = render(); - assertThat(html).contains("Answering").contains("Fake").contains("In use").contains("Last test passed").contains("…wxyz") + assertThat(html).contains("Configuration and usage").contains("Fake").contains("In use").doesNotContain("Answering", "Last test passed").contains("…wxyz") .contains("of $50.00 cap").contains("list_tables").contains("Model call #").contains("25% of 4 requests").doesNotContain(KEY).doesNotContain(" Date: Wed, 14 Oct 2026 13:39:46 -0500 Subject: [PATCH 5/5] docs: assistant health checks Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 14 ++++++++++++++ README.md | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbb3d2f..6859287 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## Unreleased - Central health + +### Added + +- Dynamic `assistant.provider.` and `assistant.model.` dependency checks in component `assistant`, category AI, reusing provider diagnostics and minimal connection calls with deadlines, cost-cap/kill-switch skips and TEST usage accounting. +- LOCAL breaker reuse of central provider outcomes younger than 60 seconds, with its existing lazy probe as fallback. +- Generic Assistant health status card linking to filtered System Health; registration, status, accounting, deadline and sentinel-secret tests. +- Offline Checkstyle configuration using a Java 21-aware engine and both existing Apache header forms; normalize existing import order and spacing. +- Snapshot compatibility for the header shim: setting extra headers avoids duplicates now that core forwards them too. + +### Removed + +- The home's independent answering/last-test health indicators and per-provider health/down-until display. Central health owns status and history. Manual Test connection actions, `lastTest*` fields and the circuit breaker remain. + ## Unreleased - X1/Y1 - Add default-off `withRecordHistory(true)` host opt-in: remove only `audit*` on RC.6, or invoke core #1193 audit-read/write-deny support when available. Preserve user permissions and all other channel denials; kof22 does not opt in. diff --git a/README.md b/README.md index 031dc73..a01d185 100644 --- a/README.md +++ b/README.md @@ -192,12 +192,42 @@ Admins set up providers on the **Providers** screen (`assistant_provider`), a st **Test connection** is a record action on a provider (select one or more). It is a standard streamed ETL: Review lists what will be called and anything wrong with the row (missing key, disallowed variable or URL) before any call. Result has one line per provider: `connected to claude-sonnet-5-5 in 812 ms`, or the error with its kind and HTTP status. It makes one short live call (a 1-sentence prompt, at most 32 output tokens, effort low), saves the result on the row, and records the call in `assistant_usage` with purpose `TEST`. +## Health checks + +The producer registers dynamic central checks in component `assistant`, category `AI`: + +| Name | Kind | Interval | Timeout | +| --- | --- | --- | --- | +| `assistant.provider.` | DEPENDENCY | 300s; 60s for LOCAL providers | 10s | +| `assistant.model.` | DEPENDENCY | 21600s (6 hours) | 30s | + +Suppliers include enabled provider/model rows and refresh with the central registry. +Provider checks reuse `AssistantProviderType.checkHealth`. Unsupported probes use +circuit-breaker state only when less than ten minutes old, otherwise return UNKNOWN. +Model checks reuse `ConnectionTest` (embeddings for embedding models), respect the +provider's monthly cost cap and the kill switch, and leave manual `lastTest*` fields +unchanged. A capped check returns UNKNOWN, `skipped: cost cap`, even on demand. +The automated runner records usage under its own owner key. Provider caps use the +existing shared accounting projection without access to other users' usage records. +Usage purpose is a string, so health diagnostics reuse `TEST` (not a new `HEALTH` enum). +All checks honor their deadlines; messages and details omit provider error text, +credentials, endpoints, prompts and completions. + +The LOCAL circuit breaker consults central results less than 60 seconds old before +its lazy probe. UP/DEGRADED avoids a duplicate probe, DOWN rejects the call, and +UNKNOWN or stale/missing results retain the existing probe fallback. The breaker +and manual Test connection actions remain available. + +This integration requires the unreleased core health SPI. For local verification, +use `./mvnw -o verify -Dqqq.snapshot.version=4.1.0-SNAPSHOT`; the POM's released QQQ +version remains unchanged. + ## The app home -The **Assistant** app's home is the one place for the assistant's health, usage and cost (no other dashboard repeats it). All of it comes from the tables below, read as the viewing admin; an empty table gives an empty state, never sample data. +The **Assistant** app's home shows configuration, usage and cost read as the viewing admin. A generic status card summarizes the central runner's latest results and links to System Health filtered to `assistant`; it shows UNKNOWN before results are available. Health history and check lists live in System Health. -- **Health:** is it answering (kill switch, default provider, key present, last test), the 7-day error rate of model calls, cost and requests this month against the instance's monthly limit. -- **Providers:** one row per provider: type, model, masked key status, last test, cost this month against its cap, limits, and any problem with its settings. +- **Configuration and usage:** default provider, key status, the 7-day error rate of model calls, cost and requests this month against the instance's monthly limit. +- **Providers:** one row per provider: type, model, masked key status, cost this month against its cap, limits, and any problem with its settings. - **Usage, last 30 days:** bars of model requests per day, totals (conversations, requests, tokens in and out, cost) and the last 7 days in words. - **Costs this month:** month to date against the policy limit and each provider cap, then by model. Cost is tokens times the model's price from **Model prices**. - **By user, this month**, **Top tools** (calls, failures, average time) and **Recent failures** (model calls and tool calls, linked). @@ -616,8 +646,8 @@ never automatically move to cloud. Provider state/signatures are removed when switching, preserving neutral tool history. Each attempted call is audited as usage. NETWORK/TIMEOUT marks a provider DOWN for 60 seconds; success marks it UP. Local -health is probed when unknown/stale with a two-second timeout. Provider cards show -health and down-until, failures show fallback totals, and turn records show the +health uses recent central results or a lazy probe with a two-second timeout. The +central health card links to System Health; failures show fallback totals, and turn records show the route and final destination. See [UPGRADE.md](UPGRADE.md) for the additive schema. ## Scheduled tasks -- 2.54.0