From cb7d5d4d5a9dc558a7e4cf5e3874bf30fe71041f Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 19:30:01 -0500 Subject: [PATCH] refactor: budget alerts through core QAlerts (QQQ 4.1.0-RC.5), alert types declared Closes #28 (first half; the DataClassification and loopback gateway stand-ins go with RC.6). - Default QQQ is 4.1.0-RC.5. - BudgetAlerts calls QAlerts.raise(AlertSignal...) directly instead of binding by reflection. - The producer declares assistant.budget.warning, .downgraded and .stopped as QAlertTypeMetaData (assistant.admin, subject assistant_budget). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + README.md | 4 +- UPGRADE.md | 2 + docs/DESIGN.md | 6 +- pom.xml | 4 +- .../assistant/AssistantQBitProducer.java | 18 +++++ .../qbits/assistant/engine/BudgetAlerts.java | 66 ++---------------- .../qbits/assistant/engine/BudgetService.java | 35 ++-------- .../assistant/AssistantQBitProducerTest.java | 25 +++++++ .../assistant/engine/BudgetServiceTest.java | 67 ++++++++++++++++--- 10 files changed, 128 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9c2dc2..89d3889 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ ## Unreleased - P-d +- Require QQQ RC.5 and send budget alerts directly through core QAlerts, with admin-visible alert type metadata and core sink tests. + - Admin budgets, expiring overrides, deduplicated warning/downgrade/stop events. - Per-dispatch limits preserving PHI routing and existing caps. - Home budget cards, 30-day cost/token bars and provider breakdowns. diff --git a/README.md b/README.md index 4ed4566..9925821 100644 --- a/README.md +++ b/README.md @@ -385,7 +385,7 @@ Apache 2.0. ## Data classification and HIPAA mode -On QQQ 4.1.0-RC.4, classify application tables/fields in the host configuration: +On QQQ 4.1.0-RC.5, classify application tables/fields in the host configuration: ```java import com.kingsrook.qbits.assistant.classification.DataClassification; @@ -455,7 +455,7 @@ extra allowance or temporary unlimited use from the budget's Overrides child lis with a required reason and expiry; the session sets the grantor. Budget state changes appear once per scope/period/state in the Events child list, -as panel notices and, when the optional QQQ Alerts SPI is installed, bell alerts. +as panel notices and, through core QAlerts, bell alerts when an alerts sink is installed (logging otherwise). The Assistant home shows budgets, overrides, daily cost and tokens, and breakdowns by user, model, provider and tool. Existing `AssistantBudgets`/policy limits and provider monthly caps continue to apply independently of these rows and overrides. diff --git a/UPGRADE.md b/UPGRADE.md index f641636..646546a 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -28,6 +28,8 @@ All configuration remains assistant.admin-only, with AI exposure NONE. ## P-d: data-driven budgets +Requires QQQ 4.1.0-RC.5 for the core Alerts API and alert type metadata. + Create `assistant_budget`, `assistant_budget_override`, `assistant_budget_event` and `assistant_budget_usage` using the P-d DDL in [schema-postgres.sql](docs/schema-postgres.sql) before deploying. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 308d0e0..da6a2e9 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -387,8 +387,10 @@ AI exposure NONE is not enough, because the assistant acts as the user and an ad reason, session grantor and expiry; they never waive the other limit layers. - Start-turn checks use the existing 429 BUDGET path through PolicyService. Every dispatch, including strict retries, evaluates current usage and overrides. - Events deduplicate WARNED, DOWNGRADED and STOPPED; optional reflective QAlerts - signals accompany panel notices without a compile-time Alerts dependency. + Events deduplicate WARNED, DOWNGRADED and STOPPED. +- QQQ RC.5 core QAlerts sends budget signals alongside panel notices, with three + admin-visible alert types and a default logging sink. Resolution awaits active + alert/recurrence tracking so overrides cannot clear shared alerts incorrectly. - Explicit registry model selection leaves provider defaults untouched. Automatic downgrades prefer LOCAL, then price. PHI eligibility and destination provider caps still apply; a missing eligible target stops the turn. diff --git a/pom.xml b/pom.xml index dcc4cbb..b3d2499 100644 --- a/pom.xml +++ b/pom.xml @@ -193,7 +193,7 @@ - + @@ -205,7 +205,7 @@ true - 4.1.0-RC.4 + 4.1.0-RC.5 diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitProducer.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitProducer.java index 1949531..3324769 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitProducer.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/AssistantQBitProducer.java @@ -28,6 +28,9 @@ import com.kingsrook.qbits.assistant.metadata.AssistantRecordScreens; import com.kingsrook.qbits.assistant.processes.DiscoverModelsProcess; import com.kingsrook.qbits.assistant.processes.TestConnectionProcess; import com.kingsrook.qbits.assistant.processes.TestModelConnectionProcess; +import com.kingsrook.qqq.backend.core.alerts.AlertSeverity; +import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData; +import com.kingsrook.qbits.assistant.model.AssistantBudget; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.logging.QLogger; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; @@ -155,6 +158,9 @@ public class AssistantQBitProducer implements QBitMetaDataProducer signalType = Class.forName(ALERT_SIGNAL, true, loader); - Object signal = signalType.getConstructor(String.class).newInstance(type); - signal = signalType.getMethod("withDedupKey", String.class).invoke(signal, dedupKey); - Class severityType = Class.forName(SEVERITY, true, loader); - @SuppressWarnings({"unchecked", "rawtypes"}) - Object level = Enum.valueOf((Class) severityType.asSubclass(Enum.class), severity); - signal = signalType.getMethod("withSeverity", severityType).invoke(signal, level); - signal = signalType.getMethod("withTitle", String.class).invoke(signal, title); - signal = signalType.getMethod("withSummary", String.class).invoke(signal, summary); - Method raise = Class.forName(QALERTS, true, loader).getMethod("raise", signalType); - raise.invoke(null, signal); - } - catch(Exception | LinkageError e) - { - LOG.info("Could not raise a assistant budget alert through the Alerts SPI", logPair("type", type), logPair("reason", e.getClass().getSimpleName())); - } - } - - - } diff --git a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/BudgetService.java b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/BudgetService.java index 1879686..137246e 100644 --- a/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/BudgetService.java +++ b/qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/BudgetService.java @@ -24,6 +24,7 @@ import com.kingsrook.qbits.assistant.model.AssistantBudgetUsage; import com.kingsrook.qbits.assistant.model.AssistantModel; import com.kingsrook.qbits.assistant.model.AssistantUsage; import com.kingsrook.qbits.assistant.persistence.AssistantStore; +import com.kingsrook.qqq.backend.core.alerts.AlertSeverity; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; @@ -48,21 +49,6 @@ public class BudgetService private final AssistantStore store; private final ProviderService providers; private final Clock clock; - private final AlertRaiser alerts; - - - - /******************************************************************************* - ** Injectable alert seam; production uses the optional Alerts producer SPI. - *******************************************************************************/ - @FunctionalInterface - public interface AlertRaiser - { - /******************************************************************************* - ** Attempt to raise one deduplicated transition. - *******************************************************************************/ - void raise(String type, String key, String severity, String title, String summary); - } @@ -104,21 +90,10 @@ public class BudgetService ** Production constructor. *******************************************************************************/ public BudgetService(AssistantStore store, ProviderService providers, Clock clock) - { - this(store, providers, clock, BudgetAlerts::raise); - } - - - - /******************************************************************************* - ** Constructor with an alert seam for tests. - *******************************************************************************/ - public BudgetService(AssistantStore store, ProviderService providers, Clock clock, AlertRaiser alerts) { this.store = store; this.providers = providers; this.clock = clock; - this.alerts = alerts; } @@ -318,6 +293,10 @@ public class BudgetService /******************************************************************************* ** The persisted unique key is also the warning/alert deduplication record. + ** Resolution on period rollover or override recovery needs an active-alert + ** lifecycle: these immutable events cannot re-raise a resolved state within + ** the same period, and user-specific overrides must not resolve shared alerts. + ** Track active keys and recurrence before calling QAlerts.resolve here. *******************************************************************************/ private boolean transition(Snapshot snapshot, String state, String userId, String message) throws QException { @@ -333,8 +312,8 @@ public class BudgetService case "DOWNGRADED" -> "downgraded"; default -> "stopped"; }; - alerts.raise(type, snapshot.budget().getId() + ":" + snapshot.scopeKey() + ":" + snapshot.periodStart(), - "STOPPED".equals(state) ? "ERROR" : "WARNING", "Assistant budget " + snapshot.budget().getName(), message); + BudgetAlerts.raise(type, snapshot.budget().getId() + ":" + snapshot.scopeKey() + ":" + snapshot.periodStart(), + "STOPPED".equals(state) ? AlertSeverity.ERROR : AlertSeverity.WARNING, "Assistant budget " + snapshot.budget().getName(), message); return (true); } diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java index a4b19c8..719e8a1 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/AssistantQBitProducerTest.java @@ -26,6 +26,9 @@ import com.kingsrook.qbits.assistant.testing.AssistantTestFixture; import com.kingsrook.qbits.assistant.testing.BaseAssistantTest; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionCheckResult; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; +import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData; +import com.kingsrook.qqq.backend.core.alerts.AlertSeverity; +import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException; @@ -103,6 +106,28 @@ class AssistantQBitProducerTest extends BaseAssistantTest + /******************************************************************************* + ** Budget types are discoverable through core with admin-only visibility. + *******************************************************************************/ + @Test + void budgetAlertTypes() + { + var types = AlertsInstanceMetaData.of(qInstance).getAlertTypes(); + assertThat(types).containsOnlyKeys("assistant.budget.warning", "assistant.budget.downgraded", "assistant.budget.stopped"); + for(QAlertTypeMetaData type : types.values()) + { + assertThat(type.getLabel()).isNotBlank(); + assertThat(type.getDescription()).isNotBlank(); + assertThat(type.getDefaultSeverity()).isEqualTo(type.getName().endsWith("stopped") ? AlertSeverity.ERROR : AlertSeverity.WARNING); + assertThat(type.getHowToFix()).isEqualTo("Raise the budget or grant an override in Assistant → Budgets"); + assertThat(type.getSourceQBitName()).isEqualTo("assistant"); + assertThat(type.getVisibilityPermission()).isEqualTo("assistant.admin"); + assertThat(type.getSubjectTableName()).isEqualTo("assistant_budget"); + } + } + + + /******************************************************************************* ** The admin app and tables: visible only with assistant.admin. *******************************************************************************/ diff --git a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/BudgetServiceTest.java b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/BudgetServiceTest.java index 8c6a9b4..beb7757 100644 --- a/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/BudgetServiceTest.java +++ b/qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/engine/BudgetServiceTest.java @@ -35,6 +35,13 @@ import com.kingsrook.qbits.assistant.provider.AssistantProviderType; import com.kingsrook.qbits.assistant.provider.ModelUsage; import com.kingsrook.qbits.assistant.testing.AssistantTestFixture; import com.kingsrook.qbits.assistant.testing.BaseAssistantTest; +import com.kingsrook.qqq.backend.core.alerts.AlertSignal; +import com.kingsrook.qqq.backend.core.alerts.AlertSeverity; +import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData; +import com.kingsrook.qqq.backend.core.alerts.LoggingAlertSink; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReferenceWithProperties; +import com.kingsrook.qqq.backend.core.model.metadata.code.InitializableViaCodeReference; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -172,12 +179,45 @@ class BudgetServiceTest extends BaseAssistantTest /******************************************************************************* - ** Fixed-clock evaluator and alert seam. + ** Fixed-clock evaluator with the production core sink configuration. *******************************************************************************/ - private BudgetService service(List alerts) + private BudgetService service(List alerts) { - return (new BudgetService(store(), config.getRuntime().getProviderService(), Clock.fixed(NOW, ZoneOffset.UTC), - (type, key, severity, title, summary) -> alerts.add(type + ":" + severity))); + AlertsInstanceMetaData.ofOrWithNew(qInstance).withSink(new QCodeReferenceWithProperties(TestAlertSink.class, Map.of("signals", (ArrayList) alerts))); + return (new BudgetService(store(), config.getRuntime().getProviderService(), Clock.fixed(NOW, ZoneOffset.UTC))); + } + + + + /******************************************************************************* + ** Core loads a separate sink per instance; captured signals stay test-local. + *******************************************************************************/ + public static class TestAlertSink extends LoggingAlertSink implements InitializableViaCodeReference + { + private List signals; + + + + /******************************************************************************* + ** Receive this test's capture list from the configured code reference. + *******************************************************************************/ + @Override + @SuppressWarnings("unchecked") + public void initialize(QCodeReference codeReference) + { + signals = (List) ((QCodeReferenceWithProperties) codeReference).getProperties().get("signals"); + } + + + + /******************************************************************************* + ** Capture the signal delivered through core. + *******************************************************************************/ + @Override + public void raise(AlertSignal signal) + { + signals.add(signal); + } } @@ -188,15 +228,22 @@ class BudgetServiceTest extends BaseAssistantTest @Test void warningBoundaryAndDedup() throws Exception { - budget(); + AssistantBudget budget = budget(); usage(790_000, NOW); - List alerts = new ArrayList<>(); + List alerts = new ArrayList<>(); BudgetService service = service(alerts); assertThat(service.evaluate(USER, null, false, false).notices()).isEmpty(); usage(10_000, NOW); assertThat(service.evaluate(USER, null, false, false).notices()).singleElement().asString().contains("80%", "Per user daily"); assertThat(service.evaluate(USER, null, false, false).notices()).isEmpty(); - assertThat(alerts).containsExactly("assistant.budget.warning:WARNING"); + assertThat(alerts).singleElement().satisfies(signal -> + { + assertThat(signal.getAlertTypeName()).isEqualTo("assistant.budget.warning"); + assertThat(signal.getDedupKey()).isEqualTo(budget.getId() + ":USER:" + USER + ":2026-10-13T00:00:00Z"); + assertThat(signal.getSeverity()).isEqualTo(AlertSeverity.WARNING); + assertThat(signal.getTitle()).isEqualTo("Assistant budget Per user daily"); + assertThat(signal.getSummary()).contains("80%", "Per user daily"); + }); assertThat(store().query(AssistantBudgetEvent.TABLE_NAME, new QQueryFilter(), AssistantBudgetEvent.class)).singleElement() .extracting(AssistantBudgetEvent::getState).isEqualTo("WARNED"); } @@ -215,10 +262,14 @@ class BudgetServiceTest extends BaseAssistantTest AssistantModel cheap = store().insert(new AssistantModel().withProviderId(local.getId()).withModelId("cheap").withEnabled(true).withInputUsdPerMTok(BigDecimal.ONE)); store().insert(new AssistantModel().withProviderId(cloud.getId()).withModelId("main").withEnabled(true).withInputUsdPerMTok(BigDecimal.ZERO)); usage(1_000_000, NOW); - BudgetService service = service(new ArrayList<>()); + List alerts = new ArrayList<>(); + BudgetService service = service(alerts); ProviderService.Active active = config.getRuntime().getProviderService().resolve("main"); assertThat(service.evaluate(USER, active, false, false).active().providerId()).isEqualTo(local.getId()); assertThat(service.evaluate(USER, active, true, false).stopped()).isTrue(); + assertThat(alerts).extracting(AlertSignal::getAlertTypeName).containsExactly("assistant.budget.downgraded", "assistant.budget.stopped"); + assertThat(alerts).extracting(AlertSignal::getSeverity).containsExactly(AlertSeverity.WARNING, AlertSeverity.ERROR); + assertThat(alerts).extracting(AlertSignal::getDedupKey).containsOnly(budget.getId() + ":USER:" + USER + ":2026-10-13T00:00:00Z"); store().updateFields(AssistantBudget.TABLE_NAME, new QRecord().withValue("id", budget.getId()).withValue("downgradeModelId", cheap.getId())); assertThat(service.evaluate(USER, active, false, false).active().model()).isEqualTo("cheap"); } -- 2.54.0