From cfad90eaeeafe68c7d93c298aaee6079a4051dc4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 16 Oct 2026 11:43:58 -0500 Subject: [PATCH] fix(assistant): optional unreachable providers degrade health; add provider setup state An enabled provider that is not the default and not the only route for any configured routing (enabled profile simple/complex model, budget downgrade, knowledge source embedding, with no fallback on another usable provider) is optional. When it is unreachable its provider and model checks now report DEGRADED with a message naming where requests go, instead of turning the whole "assistant" component DOWN (central health rolls up worst-of; WARNING severity does not cap DOWN). The default provider and sole routes stay DOWN. Failed provider probes carry reachable=false so the circuit breaker still refuses the provider when its central outcome is only DEGRADED. New assistant_provider.setup_state (Active / Setting up, empty = Active): a provider that is Setting up is never routed to (default, profile models, fallbacks, budget downgrades, embeddings) and gets no provider or model health checks. Additive guarded DDL in docs/schema-postgres.sql; see UPGRADE.md. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 + UPGRADE.md | 14 ++ docs/schema-postgres.sql | 6 + .../assistant/engine/ProviderHealth.java | 24 ++- .../assistant/engine/ProviderService.java | 11 +- .../health/AssistantHealthChecks.java | 183 ++++++++++++++++- .../metadata/AssistantRecordScreens.java | 5 +- .../metadata/AssistantSettingsCustomizer.java | 8 + .../assistant/model/AssistantProvider.java | 57 ++++++ .../main/resources/help/qbit-assistant.yaml | 9 + .../health/AssistantHealthChecksTest.java | 190 ++++++++++++++++++ 11 files changed, 499 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18fc876..688af47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased - Optional provider health + +- An enabled provider that is not the default and not the only route for any configured routing (an enabled profile's simple/complex model, a budget downgrade model or a knowledge source's embedding model, with no fallback on another usable provider) is optional: when it is unreachable, its `assistant.provider.` and `assistant.model.` checks report DEGRADED with a message naming the provider requests use instead, so one unfinished local server no longer turns the whole `assistant` component DOWN. The default provider and sole routes still report DOWN. A failed provider probe still carries `reachable=false`, so the circuit breaker keeps refusing the provider. +- New `assistant_provider.setup_state` (Setup state: Active / Setting up, empty counts as Active). A provider that is Setting up is never routed to (default, profile models, fallbacks, budget downgrades, embeddings) and gets no provider or model health checks; Test connection and Discover models still work. Apply the Provider setup state DDL (UPGRADE.md). + ## Unreleased - Fail soft (UI API extension 1.5.0) - Every message part, tool render and presentation is validated before it is stored or streamed, and again when stored parts and events are read: all render types (records, record, count, widget, history, pending_action, bulk_load_dry_run) and presentation blocks always carry their lists. A render that cannot be repaired is replaced by an error notice ("This result could not be displayed."); unknown part and render types pass through. Problems are logged with the type and a fixed code only. diff --git a/UPGRADE.md b/UPGRADE.md index 38fcce6..b8d2206 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,5 +1,19 @@ # Upgrading qbit-assistant +## Provider setup state (Unreleased) + +Apply the **Provider setup state** block at the end of `docs/schema-postgres.sql` +**before** deploying: the provider table now reads `setup_state`, so the new jar +fails on `assistant_provider` without it. It adds a nullable `VARCHAR(16)` with +default `'ACTIVE'` and a check for `ACTIVE` / `SETTING_UP`; existing rows behave as +before. It is `IF NOT EXISTS` and safe to re-run. + +To stop an unfinished provider from being routed to or health-checked, edit it in +Assistant > Assistant Provider (Provider section) and set **Setup state** to **Setting up** (SQL: +`UPDATE assistant_provider SET setup_state = 'SETTING_UP' WHERE id = ;`). +Enabled providers that are neither the default nor the only route for a configured +model now report DEGRADED, not DOWN, when unreachable. No configuration change. + ## Fail soft (Unreleased, UI API extension assistant 1.5.0) Apply the **Fail soft** block at the end of `docs/schema-postgres.sql` before diff --git a/docs/schema-postgres.sql b/docs/schema-postgres.sql index 965aa2b..f628eb4 100644 --- a/docs/schema-postgres.sql +++ b/docs/schema-postgres.sql @@ -744,3 +744,9 @@ CREATE INDEX IF NOT EXISTS assistant_client_error_create_date ON assistant_clien CREATE INDEX IF NOT EXISTS assistant_client_error_conversation ON assistant_client_error (conversation_id); CREATE INDEX IF NOT EXISTS assistant_client_error_message ON assistant_client_error (message_id); CREATE INDEX IF NOT EXISTS assistant_client_error_turn ON assistant_client_error (turn_id); + +-- Provider setup state: SETTING_UP marks a provider an admin is still building. The +-- assistant never routes to it and its provider and model health checks do not run. +-- Existing rows (and NULL) behave as ACTIVE, exactly as before. Safe to re-run. +ALTER TABLE assistant_provider ADD COLUMN IF NOT EXISTS setup_state VARCHAR(16) DEFAULT 'ACTIVE' + CONSTRAINT ck_assistant_provider_setup_state CHECK (setup_state IN ('ACTIVE', 'SETTING_UP')); diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderHealth.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderHealth.java index 54a4fe3..7eb643e 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderHealth.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderHealth.java @@ -27,6 +27,7 @@ import com.kingsrook.qbits.assistant.spi.HealthResult; import com.kingsrook.qbits.assistant.spi.ModelProviderException; import com.kingsrook.qbits.assistant.spi.ProviderSettings; import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.health.HealthCheckOutcome; import com.kingsrook.qqq.backend.core.health.HealthRunner; import com.kingsrook.qqq.backend.core.health.HealthStatus; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -37,6 +38,12 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; *******************************************************************************/ public class ProviderHealth { + /******************************************************************************* + ** Health detail set to false when a provider probe failed, even when the + ** outcome is only DEGRADED because the provider is optional. + *******************************************************************************/ + public static final String DETAIL_REACHABLE = "reachable"; + private final AssistantRuntime runtime; @@ -65,6 +72,10 @@ public class ProviderHealth { throw (new ModelProviderException(ModelProviderException.Kind.AUTHENTICATION, null, "Provider is disabled.")); } + if(row.isSettingUp()) + { + throw (new ModelProviderException(ModelProviderException.Kind.AUTHENTICATION, null, "Provider is not set up yet.")); + } if(isDown(row)) { throw (new ModelProviderException(ModelProviderException.Kind.OVERLOADED, null, "Provider circuit is open.")); @@ -80,7 +91,7 @@ public class ProviderHealth var outcome = runner == null ? null : runner.getLastOutcome(checkName); if(checked != null && !checked.isAfter(now) && checked.plusSeconds(60).isAfter(now) && outcome != null) { - if(outcome.getStatus() == HealthStatus.DOWN) + if(outcome.getStatus() == HealthStatus.DOWN || isUnreachable(outcome)) { throw (new ModelProviderException(ModelProviderException.Kind.NETWORK, null, "Provider health check failed.")); } @@ -158,4 +169,15 @@ public class ProviderHealth .withValue("healthState", state).withValue("healthCheckedAt", runtime.getClock().instant()).withValue("downUntil", downUntil)); runtime.getProviderService().invalidate(); } + + + + /******************************************************************************* + ** A central outcome that says the provider could not be reached, whatever + ** its status (an optional provider's failed probe is DEGRADED, not DOWN). + *******************************************************************************/ + public static boolean isUnreachable(HealthCheckOutcome outcome) + { + return (outcome != null && outcome.getDetails() != null && Boolean.FALSE.equals(outcome.getDetails().getValues().get(DETAIL_REACHABLE))); + } } diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java index d135322..564131f 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ProviderService.java @@ -158,14 +158,15 @@ public class ProviderService /******************************************************************************* - ** The default enabled provider row, or null. + ** The default usable (enabled and set up) provider row, else the first + ** usable one, or null. *******************************************************************************/ public AssistantProvider activeRow() throws QException { AssistantProvider first = null; for(AssistantProvider row : rows()) { - if(Boolean.TRUE.equals(row.getEnabled())) + if(row.isUsable()) { if(Boolean.TRUE.equals(row.getIsDefault())) { @@ -782,7 +783,7 @@ public class ProviderService return (null); } AssistantProvider row = store.getProvider(model.getProviderId()); - if(row == null || !Boolean.TRUE.equals(row.getEnabled()) || ((requiresHipaa || Boolean.TRUE.equals(config.getHipaaMode())) && !Boolean.TRUE.equals(row.getHipaaEligible()))) + if(row == null || !row.isUsable() || ((requiresHipaa || Boolean.TRUE.equals(config.getHipaaMode())) && !Boolean.TRUE.equals(row.getHipaaEligible()))) { return (null); } @@ -854,9 +855,9 @@ public class ProviderService throw (new ProviderSetupException("Select an enabled EMBEDDING model for this knowledge source.")); } AssistantProvider row = store.getProvider(model.getProviderId()); - if(row == null || !Boolean.TRUE.equals(row.getEnabled())) + if(row == null || !row.isUsable()) { - throw (new ProviderSetupException("The embedding provider is disabled or unavailable.")); + throw (new ProviderSetupException("The embedding provider is disabled, not set up yet or unavailable.")); } if((requiresHipaa || Boolean.TRUE.equals(config.getHipaaMode())) && !Boolean.TRUE.equals(row.getHipaaEligible())) { diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java index a9a5527..857506a 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecks.java @@ -20,16 +20,26 @@ package com.kingsrook.qbits.assistant.health; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.stream.Collectors; import com.kingsrook.qbits.assistant.AssistantRuntime; import com.kingsrook.qbits.assistant.AssistantSecurity; import com.kingsrook.qbits.assistant.engine.ConnectionTest; +import com.kingsrook.qbits.assistant.engine.ProviderHealth; import com.kingsrook.qbits.assistant.engine.ProviderService; +import com.kingsrook.qbits.assistant.model.AssistantBudget; import com.kingsrook.qbits.assistant.model.AssistantModel; +import com.kingsrook.qbits.assistant.model.AssistantProfile; import com.kingsrook.qbits.assistant.model.AssistantProvider; +import com.kingsrook.qbits.assistant.model.AssistantRagSource; import com.kingsrook.qbits.assistant.spi.HealthResult; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; @@ -44,6 +54,8 @@ import com.kingsrook.qqq.backend.core.health.HealthCheckSupplierMetaData; import com.kingsrook.qqq.backend.core.health.HealthChecksMetaData; import com.kingsrook.qqq.backend.core.health.HealthDetails; import com.kingsrook.qqq.backend.core.health.HealthStatus; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; @@ -54,6 +66,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; public final class AssistantHealthChecks { public static final String CLIENT_ERRORS = "assistant.client-errors"; + public static final String NOT_SET_UP = "Not set up yet; checks are skipped"; + + private static final QLogger LOG = QLogger.getLogger(AssistantHealthChecks.class); /******************************************************************************* ** Utility class. @@ -129,7 +144,8 @@ public final class AssistantHealthChecks /******************************************************************************* - ** One definition per enabled provider; no I/O to providers during resolution. + ** One definition per enabled provider that is set up; no I/O to providers + ** during resolution. *******************************************************************************/ public static class Providers implements HealthCheckSupplier { @@ -143,10 +159,10 @@ public final class AssistantHealthChecks List checks = new ArrayList<>(); for(AssistantProvider row : runtime.getStore().listProviders()) { - if(Boolean.TRUE.equals(row.getEnabled())) + if(row.isUsable()) { checks.add(definition("assistant.provider." + row.getId(), "LOCAL".equals(runtime.getProviderService().location(row)) ? 60 : 300, 10000, - input -> bounded(input, () -> provider(runtime, row.getId(), input)))); + input -> optional(runtime, row.getId(), bounded(input, () -> provider(runtime, row.getId(), input)), false))); } } return (checks); @@ -156,7 +172,8 @@ public final class AssistantHealthChecks /******************************************************************************* - ** One definition per enabled model, including models of disabled providers. + ** One definition per enabled model, including models of disabled providers, + ** but none for a provider that is still being set up. *******************************************************************************/ public static class Models implements HealthCheckSupplier { @@ -168,12 +185,13 @@ public final class AssistantHealthChecks { AssistantRuntime runtime = AssistantRuntime.of(instance); List checks = new ArrayList<>(); + Set settingUp = runtime.getStore().listProviders().stream().filter(AssistantProvider::isSettingUp).map(AssistantProvider::getId).collect(Collectors.toSet()); for(AssistantModel row : runtime.getStore().listModels()) { - if(Boolean.TRUE.equals(row.getEnabled())) + if(Boolean.TRUE.equals(row.getEnabled()) && !settingUp.contains(row.getProviderId())) { checks.add(definition("assistant.model." + row.getId(), 21600, 30000, - input -> bounded(input, () -> model(runtime, row.getId(), input)))); + input -> optional(runtime, row.getProviderId(), bounded(input, () -> model(runtime, row.getId(), input)), true))); } } return (checks); @@ -242,6 +260,10 @@ public final class AssistantHealthChecks { return (HealthCheckOutcome.unknown("Provider disabled or unavailable")); } + if(row.isSettingUp()) + { + return (HealthCheckOutcome.up(NOT_SET_UP).withDetails(new HealthDetails().withValue("providerId", id))); + } var type = runtime.getConfig().getProviderTypes().get(row.getProviderType()); if(type == null) { @@ -290,6 +312,10 @@ public final class AssistantHealthChecks { return (HealthCheckOutcome.unknown("Provider disabled or unavailable")); } + if(provider.isSettingUp()) + { + return (HealthCheckOutcome.up(NOT_SET_UP).withDetails(new HealthDetails().withValue("modelId", id).withValue("providerId", provider.getId()))); + } Long cap = ProviderService.capMicros(provider); if(cap != null && runtime.getProviderService().monthToDateCostMicros(provider.getId(), true) >= cap) { @@ -305,4 +331,149 @@ public final class AssistantHealthChecks .withDetails(new HealthDetails().withValue("modelId", id).withValue("providerId", provider.getId())) .withLatencyMs(result.latencyMs() == null ? null : result.latencyMs().longValue())); } + + + + /******************************************************************************* + ** A failed provider (or model) check is DOWN only when the provider matters: + ** it is the default, or the only route for a configured profile model, budget + ** downgrade or knowledge source. Any other enabled provider is optional, so its + ** failure is DEGRADED with a message that says where requests go instead. + ** Provider failures also carry reachable=false, so the circuit breaker still + ** refuses the provider. If the routes cannot be read, the outcome stays DOWN. + *******************************************************************************/ + static HealthCheckOutcome optional(AssistantRuntime runtime, Integer providerId, HealthCheckOutcome outcome, boolean model) + { + if(outcome == null || outcome.getStatus() != HealthStatus.DOWN) + { + return (outcome); + } + HealthDetails details = outcome.getDetails() == null ? new HealthDetails() : outcome.getDetails(); + outcome.withDetails(details); + if(!model) + { + details.withValue(ProviderHealth.DETAIL_REACHABLE, false); + } + try + { + AssistantProvider row = providerId == null ? null : runtime.getStore().getProvider(providerId); + if(row == null || !row.isUsable()) + { + return (outcome); + } + String required = requiredBecause(runtime, row); + if(required != null) + { + details.withValue("required", required); + return (outcome); + } + AssistantProvider active = runtime.getProviderService().activeRow(); + String instead = active == null ? "the other providers" : "'" + active.getName() + "'"; + String what = model ? "Model connection failed; provider '" + row.getName() + "' is optional" : "Provider '" + row.getName() + "' is unreachable; it is optional"; + details.withValue("optional", true); + return (outcome.withStatus(HealthStatus.DEGRADED).withMessage(what + " (not the default and not the only route), so requests use " + instead)); + } + catch(Exception e) + { + LOG.warn("Could not tell whether an assistant provider is optional; reporting it as down", e); + return (outcome); + } + } + + + + /******************************************************************************* + ** Why a provider is required (null when it is optional): it is the provider + ** the assistant uses by default, or one of its enabled models is assigned to + ** a routing profile, a budget downgrade or a knowledge source and that model + ** has no fallback on another usable provider. + *******************************************************************************/ + static String requiredBecause(AssistantRuntime runtime, AssistantProvider row) throws QException + { + AssistantProvider active = runtime.getProviderService().activeRow(); + if(active == null || Objects.equals(active.getId(), row.getId())) + { + return ("default provider"); + } + + Map models = runtime.getStore().listModels().stream().collect(Collectors.toMap(AssistantModel::getId, Function.identity(), (a, b) -> a)); + Map providers = runtime.getStore().listProviders().stream().collect(Collectors.toMap(AssistantProvider::getId, Function.identity(), (a, b) -> a)); + Set routed = new HashSet<>(); + for(AssistantProfile profile : runtime.getStore().query(AssistantProfile.TABLE_NAME, new QQueryFilter(), AssistantProfile.class)) + { + if(!Boolean.FALSE.equals(profile.getEnabled()) && profile.getRoutingMode() != null && !"OFF".equals(profile.getRoutingMode())) + { + routed.add(profile.getSimpleModelId()); + routed.add(profile.getComplexModelId()); + } + } + for(AssistantBudget budget : runtime.getStore().budgets()) + { + routed.add(budget.getDowngradeModelId()); + } + for(Integer id : routed) + { + AssistantModel model = id == null ? null : models.get(id); + if(onProvider(model, row) && !hasFallback(runtime, model, row, models, providers)) + { + return ("only route for a configured model"); + } + } + for(AssistantRagSource source : runtime.getStore().query(AssistantRagSource.TABLE_NAME, new QQueryFilter(), AssistantRagSource.class)) + { + if(!Boolean.FALSE.equals(source.getEnabled()) && source.getEmbeddingModelId() != null && onProvider(models.get(source.getEmbeddingModelId()), row)) + { + return ("only route for a knowledge source"); + } + } + return (null); + } + + + + /******************************************************************************* + ** An enabled model of this provider. + *******************************************************************************/ + private static boolean onProvider(AssistantModel model, AssistantProvider row) + { + return (model != null && Boolean.TRUE.equals(model.getEnabled()) && Objects.equals(model.getProviderId(), row.getId())); + } + + + + /******************************************************************************* + ** The fallback FallbackChain would try: the explicit chain when the model has + ** one, else (local providers only) another local provider of the same type. + ** Only enabled models of other usable providers count. + *******************************************************************************/ + private static boolean hasFallback(AssistantRuntime runtime, AssistantModel model, AssistantProvider row, Map models, Map providers) + { + if(model.getFallbackModelId() != null) + { + Set seen = new HashSet<>(); + Integer next = model.getFallbackModelId(); + while(next != null && seen.add(next)) + { + AssistantModel candidate = models.get(next); + if(candidate == null) + { + return (false); + } + AssistantProvider owner = providers.get(candidate.getProviderId()); + if(Boolean.TRUE.equals(candidate.getEnabled()) && owner != null && owner.isUsable() && !Objects.equals(owner.getId(), row.getId())) + { + return (true); + } + next = candidate.getFallbackModelId(); + } + return (false); + } + if(!"LOCAL".equals(runtime.getProviderService().location(row))) + { + return (false); + } + return (providers.values().stream().anyMatch(other -> other.isUsable() && !Objects.equals(other.getId(), row.getId()) + && "LOCAL".equals(runtime.getProviderService().location(other)) && Objects.equals(other.getProviderType(), row.getProviderType()) + && models.values().stream().anyMatch(candidate -> Boolean.TRUE.equals(candidate.getEnabled()) && Objects.equals(candidate.getProviderId(), other.getId())))); + } } diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantRecordScreens.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantRecordScreens.java index b1fa06d..3ec8ae7 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantRecordScreens.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantRecordScreens.java @@ -333,6 +333,7 @@ public final class AssistantRecordScreens settingsSource("assistant_price_model", modelValues, true, false), settingsSource("assistant_provider_kind", kinds, false, false), enumSource("assistant_provider_location", List.of(new QPossibleValue<>("LOCAL", "Local"), new QPossibleValue<>("CLOUD", "Cloud"))), + enumSource("assistant_provider_setup_state", List.of(new QPossibleValue<>("ACTIVE", "Active"), new QPossibleValue<>("SETTING_UP", "Setting up"))), enumSource("assistant_routing_mode", List.of(new QPossibleValue<>("OFF", "Off"), new QPossibleValue<>("HEURISTIC", "Heuristic"), new QPossibleValue<>("CLASSIFIER", "Classifier"))), enumSource("assistant_model_kind", List.of(new QPossibleValue<>("CHAT", "Chat"), new QPossibleValue<>("EMBEDDING", "Embedding"))), enumSource("assistant_model_source", List.of(new QPossibleValue<>("DISCOVERED", "Discovered"), new QPossibleValue<>("MANUAL", "Manual"))), @@ -396,6 +397,7 @@ public final class AssistantRecordScreens pvs(table, "model", MODEL_PVS_NAME, "Model"); pvs(table, "kind", "assistant_provider_kind", "Kind"); pvs(table, "location", "assistant_provider_location", "Location"); + pvs(table, "setupState", "assistant_provider_setup_state", "Setup state"); table.getField("hipaaEligible").withLabel("HIPAA-eligible"); table.getField("apiVersion").withLabel("API version"); table.getField("connectTimeoutSeconds").withLabel("Connect timeout (s)"); @@ -414,6 +416,7 @@ public final class AssistantRecordScreens help(table, "providerType", "Which kind of provider this is. Only the types installed on this server are offered."); help(table, "model", "The model answers come from. Empty uses the policy's (or the application's) main model."); help(table, "enabled", "Off: the assistant never uses this provider."); + help(table, "setupState", "Setting up: the provider is still being built, so the assistant never routes to it and its health checks do not run. Use Active once it works."); help(table, "isDefault", "The assistant uses the default enabled provider (else the first enabled one). Marking one default unmarks the others."); help(table, "apiKeyEnvName", "The NAME of the environment variable that holds the API key (e.g. ANTHROPIC_API_KEY), never the key itself. " + "The key stays in the server's secret store; only allowed variable names are accepted. Empty uses the type's default variable. Bedrock and Vertex require this field to be empty and use server credentials. To replace the key, change it in the secret store."); @@ -424,7 +427,7 @@ public final class AssistantRecordScreens help(table, "maxConcurrentRequests", "How many model calls may run at the same time on this server. Empty: no limit."); table.getField(ProviderService.KEY_STATUS_FIELD).withLabel("API Key").withHelpContent(new QHelpContent("Whether the key variable is set on this server, with the key's last four characters. The key itself is never shown or returned.")); sections(table, - section("provider", "Provider", "settings_input_component", Tier.T1, "id", "name", "providerType", "model", "enabled", "isDefault"), + section("provider", "Provider", "settings_input_component", Tier.T1, "id", "name", "providerType", "model", "enabled", "setupState", "isDefault"), section("connection", "Connection", "key", Tier.T2, "apiKeyEnvName", ProviderService.KEY_STATUS_FIELD, "baseUrl", "tlsCaBundle", "cloudProject"), section("limits", "Limits", "speed", Tier.T2, "monthlyCostCapUsd", "maxOutputTokensPerRequest", "maxConcurrentRequests"), section("registry", "Provider settings", "settings", Tier.T2, "kind", "location", "hipaaEligible", "region", "apiVersion", "connectTimeoutSeconds", "requestTimeoutSeconds"), diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantSettingsCustomizer.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantSettingsCustomizer.java index db53105..b69cd0d 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantSettingsCustomizer.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/metadata/AssistantSettingsCustomizer.java @@ -137,6 +137,14 @@ public class AssistantSettingsCustomizer implements TableCustomizerInterface { record.setValue("location", service.location(provider)); } + if(merged.getValue("setupState") == null) + { + record.setValue("setupState", AssistantProvider.SETUP_STATE_ACTIVE); + } + else if(!List.of(AssistantProvider.SETUP_STATE_ACTIVE, AssistantProvider.SETUP_STATE_SETTING_UP).contains(merged.getValueString("setupState"))) + { + record.addError(new BadInputStatusMessage("Setup state must be Active or Setting up.")); + } for(String field : List.of("hipaaEligible", "connectTimeoutSeconds", "requestTimeoutSeconds")) { if(merged.getValue(field) == null) diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/model/AssistantProvider.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/model/AssistantProvider.java index 26f493e..12ae296 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/model/AssistantProvider.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/model/AssistantProvider.java @@ -71,6 +71,12 @@ public class AssistantProvider extends QRecordEntity public static final String TABLE_NAME = "assistant_provider"; + public static final String SETUP_STATE_ACTIVE = "ACTIVE"; + public static final String SETUP_STATE_SETTING_UP = "SETTING_UP"; + + @QField(maxLength = 16) + private String setupState; + @QField(isEditable = false, maxLength = 8) private String healthState; @@ -1218,4 +1224,55 @@ public class AssistantProvider extends QRecordEntity this.downUntil = downUntil; return (this); } + + + /******************************************************************************* + ** Get setupState: ACTIVE (or empty) when in use, SETTING_UP while an admin + ** is still building it (never routed to, no health checks). + *******************************************************************************/ + public String getSetupState() + { + return (this.setupState); + } + + + + /******************************************************************************* + ** Set setupState. + *******************************************************************************/ + public void setSetupState(String setupState) + { + this.setupState = setupState; + } + + + + /******************************************************************************* + ** With setupState. + *******************************************************************************/ + public AssistantProvider withSetupState(String setupState) + { + this.setupState = setupState; + return (this); + } + + + + /******************************************************************************* + ** Whether an admin marked this provider as not set up yet. + *******************************************************************************/ + public boolean isSettingUp() + { + return (SETUP_STATE_SETTING_UP.equals(setupState)); + } + + + + /******************************************************************************* + ** Enabled and set up: the only rows the assistant routes to or health-checks. + *******************************************************************************/ + public boolean isUsable() + { + return (Boolean.TRUE.equals(enabled) && !isSettingUp()); + } } diff --git a/qbit-assistant/src/main/resources/help/qbit-assistant.yaml b/qbit-assistant/src/main/resources/help/qbit-assistant.yaml index 211127e..c99bb7d 100644 --- a/qbit-assistant/src/main/resources/help/qbit-assistant.yaml +++ b/qbit-assistant/src/main/resources/help/qbit-assistant.yaml @@ -562,6 +562,10 @@ entries: summary: LOCAL for a server on your own network, or CLOUD for an outside service. content: | Local providers are probed for health before use. Empty uses the usual value for the kind: Ollama, LM Studio, vLLM and llama.cpp are local; everything else is cloud. +- key: table:assistant_provider;field:setupState + summary: Setting up while you are still building this provider; Active once it works. Empty counts as Active. + content: | + While a provider is Setting up the assistant never sends questions to it, it is not the default even if marked so, and its provider and model health checks do not run, so an unfinished server does not show the assistant as down. Test connection and Discover models still work, so you can check it before you switch it to Active. - key: table:assistant_provider;field:region summary: The cloud region to use; only needed for AWS Bedrock and Google Vertex AI. content: | @@ -1397,6 +1401,11 @@ entries: content: | - **Local**: runs on your network. Its models cost 0 unless priced, and it is the preferred automatic backup. - **Cloud**: a hosted service. Costs follow the price list and the monthly cost cap. +- key: possibleValueSource:assistant_provider_setup_state;slot:help + summary: Whether a provider is in use or still being set up. + content: | + - **Active**: in use. The assistant routes to it and health-checks it. + - **Setting up**: not used yet. No routing and no health checks until you switch it to Active. - key: possibleValueSource:assistant_routing_mode;slot:help summary: How a profile chooses between its simple and complex model for each question. content: | diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java index 017e8e9..467ed31 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/health/AssistantHealthChecksTest.java @@ -31,8 +31,10 @@ import com.kingsrook.qbits.assistant.AssistantQBitConfig; import com.kingsrook.qbits.assistant.engine.ProviderHealth; import com.kingsrook.qbits.assistant.engine.ProviderService; import com.kingsrook.qbits.assistant.engine.UsageRecorder; +import com.kingsrook.qbits.assistant.model.AssistantBudget; import com.kingsrook.qbits.assistant.model.AssistantModel; import com.kingsrook.qbits.assistant.model.AssistantPolicy; +import com.kingsrook.qbits.assistant.model.AssistantProfile; import com.kingsrook.qbits.assistant.model.AssistantProvider; import com.kingsrook.qbits.assistant.model.AssistantUsage; import com.kingsrook.qbits.assistant.provider.fake.ScriptedModelProvider; @@ -625,4 +627,192 @@ class AssistantHealthChecksTest extends BaseAssistantTest assertThat(data.getCount()).isEqualTo("Degraded"); } } + + + /******************************************************************************* + ** A second enabled provider, not the default. + *******************************************************************************/ + private AssistantProvider optionalRow(AssistantProvider defaultRow) throws Exception + { + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", defaultRow.getId()).withValue("isDefault", true).withValue("name", "Claude")); + AssistantProvider optional = config.getRuntime().getStore().insert(new AssistantProvider().withName("lore (local GPU)") + .withProviderType(type.getType()).withEnabled(true).withIsDefault(false)); + config.getRuntime().getProviderService().invalidate(); + return (optional); + } + + + + /******************************************************************************* + ** An unreachable optional provider is DEGRADED and names the default; the + ** default stays DOWN, and the breaker still refuses the optional provider. + *******************************************************************************/ + @Test + void optionalProviderIsDegradedDefaultIsDown() throws Exception + { + AssistantProvider defaultRow = row(); + AssistantProvider lore = optionalRow(defaultRow); + healthy = false; + + HealthCheckOutcome outcome = run(check("assistant.provider." + lore.getId())); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DEGRADED); + assertThat(outcome.getMessage()).isEqualTo("Provider 'lore (local GPU)' is unreachable; it is optional (not the default and not the only route), so requests use 'Claude'"); + assertThat(outcome.getDetails().getValues()).containsEntry("reachable", false).containsEntry("optional", true).containsEntry("providerId", lore.getId()); + assertThat(ProviderHealth.isUnreachable(outcome)).isTrue(); + + HealthCheckOutcome required = run(check("assistant.provider." + defaultRow.getId())); + assertThat(required.getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(required.getDetails().getValues()).containsEntry("required", "default provider").containsEntry("reachable", false); + + healthy = true; + assertThat(run(check("assistant.provider." + lore.getId())).getStatus()).isEqualTo(HealthStatus.UP); + assertThat(ProviderHealth.isUnreachable(HealthCheckOutcome.degraded("Slow"))).isFalse(); + assertThat(ProviderHealth.isUnreachable(null)).isFalse(); + + //////////////////////////////////////////////////////////////////// + // a fresh DEGRADED-but-unreachable central result opens the breaker // + //////////////////////////////////////////////////////////////////// + var active = new ProviderService.Active(lore, type.getType(), "model", provider); + try(var central = cached("assistant.provider." + lore.getId(), outcome, Instant.now().minusSeconds(1))) + { + assertThatThrownBy(() -> new ProviderHealth(config.getRuntime()).beforeCall(active)).isInstanceOf(ModelProviderException.class); + assertThat(probes).isEqualTo(3); + } + } + + + + /******************************************************************************* + ** Model checks follow their provider: DEGRADED when optional. + *******************************************************************************/ + @Test + void optionalProviderModelIsDegraded() throws Exception + { + AssistantProvider defaultRow = row(); + AssistantProvider lore = optionalRow(defaultRow); + AssistantModel model = model(lore); + provider.thenError(ModelProviderException.Kind.NETWORK, SECRET); + HealthCheckOutcome outcome = run(check("assistant.model." + model.getId())); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DEGRADED); + assertThat(outcome.getMessage()).startsWith("Model connection failed; provider 'lore (local GPU)' is optional").endsWith("so requests use 'Claude'"); + assertThat(outcome.getDetails().getValues()).doesNotContainKey("reachable").containsEntry("modelId", model.getId()); + + AssistantModel defaultModel = model(defaultRow); + provider.thenError(ModelProviderException.Kind.NETWORK, SECRET); + assertThat(run(check("assistant.model." + defaultModel.getId())).getStatus()).isEqualTo(HealthStatus.DOWN); + } + + + + /******************************************************************************* + ** A provider that is the only route for routing or a budget downgrade stays + ** DOWN; an explicit fallback on another provider makes it optional again. + *******************************************************************************/ + @Test + void soleRouteStaysDown() throws Exception + { + AssistantProvider defaultRow = row(); + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", defaultRow.getId()).withValue("location", "CLOUD")); + AssistantProvider lore = optionalRow(defaultRow); + AssistantModel loreModel = model(lore); + AssistantModel defaultModel = model(defaultRow); + healthy = false; + var check = check("assistant.provider." + lore.getId()); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DEGRADED); + + AssistantProfile profile = config.getRuntime().getStore().insert(new AssistantProfile().withName("routing").withDisplayName("Assistant").withEnabled(true) + .withRoutingMode("HEURISTIC").withSimpleModelId(loreModel.getId()).withComplexModelId(defaultModel.getId())); + HealthCheckOutcome outcome = run(check); + assertThat(outcome.getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(outcome.getDetails().getValues()).containsEntry("required", "only route for a configured model"); + + config.getRuntime().getStore().updateFields(AssistantModel.TABLE_NAME, new QRecord().withValue("id", loreModel.getId()).withValue("fallbackModelId", defaultModel.getId())); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DEGRADED); + + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", defaultRow.getId()).withValue("setupState", AssistantProvider.SETUP_STATE_SETTING_UP)); + config.getRuntime().getProviderService().invalidate(); + assertThat(run(check).getStatus()).as("lore is the only usable provider left, hence the default").isEqualTo(HealthStatus.DOWN); + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", defaultRow.getId()).withValue("setupState", AssistantProvider.SETUP_STATE_ACTIVE)); + config.getRuntime().getProviderService().invalidate(); + + config.getRuntime().getStore().updateFields(AssistantModel.TABLE_NAME, new QRecord().withValue("id", loreModel.getId()).withValue("fallbackModelId", null)); + config.getRuntime().getStore().updateFields(AssistantProfile.TABLE_NAME, new QRecord().withValue("id", profile.getId()).withValue("routingMode", "OFF")); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DEGRADED); + + config.getRuntime().getStore().insert(new AssistantBudget().withName("Downgrade").withScopeType("INSTANCE").withPeriod("MONTH").withMetric("COST_USD") + .withLimitValue(BigDecimal.ONE).withEnabled(true).withDowngradeModelId(loreModel.getId())); + assertThat(run(check).getStatus()).isEqualTo(HealthStatus.DOWN); + } + + + + /******************************************************************************* + ** Another local provider of the same type is the automatic fallback. + *******************************************************************************/ + @Test + void localPoolIsAFallback() throws Exception + { + AssistantProvider defaultRow = row(); + AssistantProvider lore = optionalRow(defaultRow); + AssistantModel loreModel = model(lore); + model(defaultRow); + config.getRuntime().getStore().insert(new AssistantProfile().withName("routing").withDisplayName("Assistant").withEnabled(true) + .withRoutingMode("HEURISTIC").withSimpleModelId(loreModel.getId()).withComplexModelId(loreModel.getId())); + healthy = false; + assertThat(run(check("assistant.provider." + lore.getId())).getStatus()).isEqualTo(HealthStatus.DEGRADED); + } + + + + /******************************************************************************* + ** Setting up: no checks registered, no routing, and an already resolved check + ** reports UP without probing. + *******************************************************************************/ + @Test + void settingUpSkipsChecksAndRouting() throws Exception + { + AssistantProvider defaultRow = row(); + AssistantProvider lore = optionalRow(defaultRow); + AssistantModel model = model(lore); + assertThat(lore.getSetupState()).isEqualTo(AssistantProvider.SETUP_STATE_ACTIVE); + var providerCheck = check("assistant.provider." + lore.getId()); + var modelCheck = check("assistant.model." + model.getId()); + + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", lore.getId()).withValue("setupState", AssistantProvider.SETUP_STATE_SETTING_UP)); + config.getRuntime().getProviderService().invalidate(); + assertThat(HealthChecksMetaData.ofOrWithNew(qInstance).resolveAll(qInstance)).extracting(HealthCheckMetaData::getName) + .doesNotContain("assistant.provider." + lore.getId(), "assistant.model." + model.getId()) + .contains("assistant.provider." + defaultRow.getId()); + Integer before = probes; + assertThat(run(providerCheck).getMessage()).isEqualTo(AssistantHealthChecks.NOT_SET_UP); + assertThat(run(modelCheck).getStatus()).isEqualTo(HealthStatus.UP); + assertThat(probes).isEqualTo(before); + assertThat(provider.getRequests()).isEmpty(); + + assertThat(config.getRuntime().getProviderService().resolveModel(model.getId(), false)).isNull(); + assertThatThrownBy(() -> new ProviderHealth(config.getRuntime()).beforeCall(new ProviderService.Active(lore, type.getType(), "model", provider))) + .hasMessageContaining("not set up yet"); + + config.getRuntime().getStore().updateFields(AssistantProvider.TABLE_NAME, new QRecord().withValue("id", lore.getId()).withValue("isDefault", true)); + config.getRuntime().getProviderService().invalidate(); + assertThat(config.getRuntime().getProviderService().activeRow().getId()).as("a default that is setting up is skipped").isEqualTo(defaultRow.getId()); + + assertThatThrownBy(() -> config.getRuntime().getStore().insert(new AssistantProvider().withName("Bad").withProviderType(type.getType()).withEnabled(true).withSetupState("BOGUS"))) + .hasMessageContaining("Setup state must be Active or Setting up."); + } + + + + /******************************************************************************* + ** If routes cannot be read the failure stays DOWN; other statuses pass through. + *******************************************************************************/ + @Test + void optionalPassThrough() throws Exception + { + assertThat(AssistantHealthChecks.optional(config.getRuntime(), null, null, false)).isNull(); + HealthCheckOutcome up = HealthCheckOutcome.up("fine"); + assertThat(AssistantHealthChecks.optional(config.getRuntime(), 1, up, false)).isSameAs(up); + assertThat(AssistantHealthChecks.optional(config.getRuntime(), null, HealthCheckOutcome.down("x"), true).getStatus()).isEqualTo(HealthStatus.DOWN); + assertThat(AssistantHealthChecks.optional(null, 1, HealthCheckOutcome.down("x"), false).getStatus()).isEqualTo(HealthStatus.DOWN); + } } -- 2.54.0