From 8d869f65cd57347f24cb9ba419f67bebb3d20ffb Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 17:17:39 -0500 Subject: [PATCH 1/3] feat: provider and model registry, OpenAI-compatible adapter, discovery and health (P-a) - assistant_provider gains kind, location, hipaa_eligible, region, api version, timeouts, a CA bundle reference and discovery status. - New assistant_model table (child list on the provider): capabilities, prices per million tokens, enabled flag, last test. - Provider SPI: ProviderSettings, kinds, discovery and health as default methods; create(apiKey, baseUrl) implementations keep working. - New module qbit-assistant-provider-openai: OpenAI, Azure OpenAI, OpenRouter, Ollama, LM Studio, vLLM and llama.cpp over JDK HTTP, with streaming, tool calls, usage, timeouts, cancel, strict only where supported (and the existing fallback on refusal), keyless local endpoints and per-provider private CA trust. - Record actions: Discover models (provider), Test connection (model); provider Test connection checks health first. - Mock-server contract tests, a golden tool round trip, TLS tests and a config-driven live check (skipped unless configured). Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 + AGENTS.md | 1 + CHANGELOG.md | 4 + CLAUDE.md | 1 + README.md | 58 ++ UPGRADE.md | 67 ++ docs/DESIGN.md | 24 + docs/schema-postgres.sql | 47 +- pom.xml | 6 + qbit-assistant-provider-openai/pom.xml | 128 +++ .../openai/OpenAiCompatibleModelProvider.java | 421 ++++++++++ .../openai/OpenAiCompatibleProviderType.java | 199 +++++ .../assistant/provider/openai/OpenAiHttp.java | 267 ++++++ .../provider/openai/OpenAiMapping.java | 190 +++++ .../openai/LiveOpenAiCompatibleTest.java | 77 ++ .../provider/openai/MockOpenAiServer.java | 183 +++++ .../provider/openai/OpenAiContractTest.java | 461 +++++++++++ .../provider/openai/OpenAiGoldenTurnTest.java | 87 ++ .../provider/openai/OpenAiTlsTest.java | 216 +++++ .../src/test/resources/tls/README.md | 23 + .../src/test/resources/tls/ca.pem | 19 + .../src/test/resources/tls/server.p12 | Bin 0 -> 4424 bytes .../qbits/assistant/AssistantQBitConfig.java | 58 +- .../assistant/AssistantQBitProducer.java | 7 + .../assistant/engine/ConnectionTest.java | 90 +- .../assistant/engine/ModelDiscovery.java | 118 +++ .../assistant/engine/ProviderService.java | 205 ++++- .../qbits/assistant/engine/TurnEngine.java | 2 +- .../assistant/home/AssistantHomeData.java | 6 +- .../assistant/home/AssistantHomeView.java | 7 + .../metadata/AssistantMetaDataBuilder.java | 5 + .../metadata/AssistantRecordScreens.java | 38 +- .../metadata/AssistantSettingsCustomizer.java | 40 +- .../assistant/model/AssistantMessage.java | 2 +- .../qbits/assistant/model/AssistantModel.java | 767 ++++++++++++++++++ .../assistant/model/AssistantProvider.java | 380 ++++++++- .../qbits/assistant/model/AssistantTurn.java | 2 +- .../qbits/assistant/model/AssistantUsage.java | 2 +- .../assistant/persistence/AssistantStore.java | 12 + .../processes/DiscoverModelsProcess.java | 216 +++++ .../processes/TestConnectionProcess.java | 5 +- .../processes/TestModelConnectionProcess.java | 225 +++++ .../provider/AssistantProviderType.java | 72 ++ .../assistant/provider/DiscoveredModel.java | 25 + .../assistant/provider/HealthResult.java | 32 + .../assistant/provider/ModelCapabilities.java | 104 +++ .../assistant/provider/ModelRequest.java | 37 + .../assistant/provider/ProviderSettings.java | 330 ++++++++ .../provider/RegistryModelProvider.java | 97 +++ .../assistant/provider/TrustedCaBundle.java | 153 ++++ .../api/AssistantRouteProviderTest.java | 7 +- .../engine/EngineComponentsTest.java | 17 +- .../provider/TrustedCaBundleTest.java | 67 ++ .../settings/ProviderRegistryTest.java | 432 ++++++++++ 54 files changed, 5980 insertions(+), 62 deletions(-) create mode 100644 qbit-assistant-provider-openai/pom.xml create mode 100644 qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiCompatibleModelProvider.java create mode 100644 qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiCompatibleProviderType.java create mode 100644 qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiHttp.java create mode 100644 qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiMapping.java create mode 100644 qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/LiveOpenAiCompatibleTest.java create mode 100644 qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/MockOpenAiServer.java create mode 100644 qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiContractTest.java create mode 100644 qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiGoldenTurnTest.java create mode 100644 qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiTlsTest.java create mode 100644 qbit-assistant-provider-openai/src/test/resources/tls/README.md create mode 100644 qbit-assistant-provider-openai/src/test/resources/tls/ca.pem create mode 100644 qbit-assistant-provider-openai/src/test/resources/tls/server.p12 create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/engine/ModelDiscovery.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/model/AssistantModel.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/processes/DiscoverModelsProcess.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/processes/TestModelConnectionProcess.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/provider/DiscoveredModel.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/provider/HealthResult.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/provider/ProviderSettings.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/provider/RegistryModelProvider.java create mode 100644 qbit-assistant/src/main/java/com/kingsrook/qbits/assistant/provider/TrustedCaBundle.java create mode 100644 qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/provider/TrustedCaBundleTest.java create mode 100644 qbit-assistant/src/test/java/com/kingsrook/qbits/assistant/settings/ProviderRegistryTest.java diff --git a/.gitignore b/.gitignore index a971850..087d6d7 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,6 @@ dependency-reduced-pom.xml /.env.local /.cache/ .claude/settings.local.json + +# Runtime output from every provider module +log/ diff --git a/AGENTS.md b/AGENTS.md index 9b0a0f0..865adc0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,7 @@ Always use the wrapper: Maven 3.10 cannot read the parent POM. The committed def - `processes/`: `AssistantRetentionStep` and `TestConnectionProcess` (record action on `assistant_provider`). - `engine/ProviderService` (active provider from `assistant_provider` rows, key references, prices), `UsageRecorder` (one `assistant_usage` row per model call), `ConnectionTest`. - `qbit-assistant-provider-anthropic`: `AnthropicModelProvider` and `AnthropicMapping` (anthropic-java streaming). +- `qbit-assistant-provider-openai`: OpenAI-compatible Chat Completions, model discovery and health using JDK HTTP and Jackson. ## Invariants (do not break) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aa3ebf..0e1e9d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project are documented here. The format follows Keep ## [0.1.0] - Unreleased +- Add named, server-only CA bundles for HTTPS providers, additive JVM trust and sanitized TLS diagnostics; tolerate repeated/missing streamed tool identities and invalid arguments, with a golden tool round-trip test. +- Add the provider/model registry, per-model capabilities and prices, keyless local providers, discovery and health, and provider/model connection tests. +- Add the optional JDK HTTP OpenAI-compatible adapter for OpenAI, Azure, OpenRouter, Ollama, LM Studio, vLLM and llama.cpp, with strict-schema safeguards and localhost contract tests. + Phase 0/1 of the QQQ Assistant: read-only Q&A and navigation. Requires QQQ 4.1 with: - the AI tool kernel (`qqq-backend-core` `actions.ai`, QRun-IO/qqq#1078) - the developer-panel contract (QRun-IO/qqq#1076). diff --git a/CLAUDE.md b/CLAUDE.md index 85cd556..3e23bd6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,7 @@ Always use the wrapper: Maven 3.10 cannot read the parent POM. The committed def - `processes/`: `AssistantRetentionStep` and `TestConnectionProcess` (record action on `assistant_provider`). - `engine/ProviderService` (active provider from `assistant_provider` rows, key references, prices), `UsageRecorder` (one `assistant_usage` row per model call), `ConnectionTest`. - `qbit-assistant-provider-anthropic`: `AnthropicModelProvider` and `AnthropicMapping` (anthropic-java streaming). +- `qbit-assistant-provider-openai`: OpenAI-compatible Chat Completions, model discovery and health using JDK HTTP and Jackson. ## Invariants (do not break) diff --git a/README.md b/README.md index bc91aad..85d2e82 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,64 @@ The **live** evals run the same golden tasks against Claude. They are skipped un ANTHROPIC_API_KEY=... ./mvnw verify -Dassistant.live=true -Dtest.excludedGroups= -Dgroups=live -Djacoco.skip=true -pl qbit-assistant-provider-anthropic -am ``` +## Providers + +Add `com.kingsrook.qbits:qbit-assistant-provider-openai` at the same version as +the QBit and register the adapter alongside any Anthropic adapter: + +```java +config.withProviderType(new OpenAiCompatibleProviderType()); +``` + +`OpenAiCompatibleProviderType` is in +`com.kingsrook.qbits.assistant.provider.openai`. Its installed type id is +`openai-compatible`; provider rows choose OPENAI, AZURE_OPENAI, OPENROUTER, +OLLAMA, LM_STUDIO, VLLM, LLAMA_CPP or OPENAI_COMPATIBLE as their kind. + +- Local kinds use their localhost endpoint by default and need no key. Leave + `apiKeyEnvName` empty. An auth proxy can use an explicitly allowed key env name. +- Cloud defaults use `OPENAI_API_KEY`, `AZURE_OPENAI_API_KEY` or + `OPENROUTER_API_KEY`. Values stay in the host's secret store. +- An override must match an entry in `config.withAllowedBaseUrls(Set.of(...))`. + Azure and OPENAI_COMPATIBLE have no default URL and always require an entry. + Redirects are not followed. Azure uses the deployment model id and defaults + `apiVersion` to `2024-10-21`. +- **Discover models** adds a provider's Models child list; existing edits survive. + Large cloud catalogs (over 20 models) start disabled. **Test connection** is + available on both providers and models and records a TEST usage row. +- A selected registered model must be enabled. With no model selected, the first + enabled registry model wins; providers with no model rows retain legacy selection. + Registry prices override type/model prices, then configured prices. Unpriced + local models cost zero. Capabilities can be adjusted per registered model. + +For Ollama behind a TLS proxy with a private CA and bearer authentication: + +```java +config.withProviderType(new OpenAiCompatibleProviderType()) + .withAllowedBaseUrls(Set.of("https://llm.example.lan:8443/v1")) + .withAllowedApiKeyEnvNames(Set.of("LOCAL_LLM_API_KEY")) + .withTrustedCaBundleFile("local-llm", Path.of("/etc/app/llm-root-ca.pem")); +``` + +Set `LOCAL_LLM_API_KEY` in the server environment from your secret store. In the +provider record choose kind `OLLAMA`, base URL `https://llm.example.lan:8443/v1`, +`apiKeyEnvName = LOCAL_LLM_API_KEY`, and `tlsCaBundle = local-llm`. Only the names +are stored. `withTrustedCaBundle("local-llm", pem)` also accepts one or more PEM +certificates directly at startup. Invalid PEM fails configuration immediately. +The named bundle adds to the JVM roots; hostname verification stays enabled. +Leave `tlsCaBundle` empty to use JVM trust alone. Restart after changing a CA file. + +Default tests use localhost mocks. To exercise an existing development server, +set `ASSISTANT_LIVE_OPENAI_BASE_URL` and `ASSISTANT_LIVE_OPENAI_MODEL`, optionally +`ASSISTANT_LIVE_OPENAI_KIND` (default OLLAMA), `ASSISTANT_LIVE_OPENAI_KEY_ENV` +(the variable name containing the key), and `ASSISTANT_LIVE_OPENAI_CA_FILE` +(a PEM bundle path). Unset or unreachable endpoints, including TLS failures, skip +the live test. Run: + +```sh +./mvnw verify -pl qbit-assistant-provider-openai -am -Dtest.excludedGroups= -Dgroups=live -Djacoco.skip=true +``` + ## License Apache 2.0. diff --git a/UPGRADE.md b/UPGRADE.md index 618fefc..5fcc2b5 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -118,3 +118,70 @@ Database (see `docs/schema-postgres.sql`): Runtime: the tools call this server's own `/qqq/v1` on `127.0.0.1` and the port the first assistant request arrived on. If the server is not reachable that way (a different bind address, a TLS-only listener), set `withUiApiBaseUrl(...)`. Grant `assistant.admin` holders the new tables through the qbit's existing admin permission (no new permission names). Behavior: nothing is writable until an admin adds a READ_WRITE data access row; before that the assistant reads exactly what it did before, now through the UI API. + +## P-a: provider and model registry + +Before starting the new version, database-backed hosts add the provider columns +and model sub-table below. Memory-backed hosts need no migration. Existing rows +derive kind and location from the adapter and keep their model behavior until +models are registered. Keys remain environment-variable references. + +```sql +ALTER TABLE assistant_provider + ADD COLUMN kind VARCHAR(30), + ADD COLUMN tls_ca_bundle VARCHAR(100), + ADD COLUMN location VARCHAR(10), + ADD COLUMN hipaa_eligible BOOLEAN DEFAULT FALSE, + ADD COLUMN region VARCHAR(50), + ADD COLUMN api_version VARCHAR(30), + ADD COLUMN connect_timeout_seconds INTEGER DEFAULT 10 CHECK (connect_timeout_seconds BETWEEN 1 AND 600), + ADD COLUMN request_timeout_seconds INTEGER DEFAULT 120 CHECK (request_timeout_seconds BETWEEN 1 AND 600), + ADD COLUMN last_discovery_at TIMESTAMP WITH TIME ZONE, + ADD COLUMN last_discovery_status VARCHAR(16), + ADD COLUMN last_discovery_error VARCHAR(500); +ALTER TABLE assistant_provider ALTER COLUMN model TYPE VARCHAR(200); +ALTER TABLE assistant_provider ALTER COLUMN last_test_model TYPE VARCHAR(200); +ALTER TABLE assistant_usage ALTER COLUMN model TYPE VARCHAR(200); +ALTER TABLE assistant_message ALTER COLUMN model TYPE VARCHAR(200); +ALTER TABLE assistant_turn ALTER COLUMN model TYPE VARCHAR(200); + +CREATE TABLE assistant_model +( + id SERIAL PRIMARY KEY, + provider_id INTEGER NOT NULL REFERENCES assistant_provider(id), + model_id VARCHAR(200) NOT NULL, + display_name VARCHAR(200), + enabled BOOLEAN, + supports_tools BOOLEAN, + supports_vision BOOLEAN, + supports_streaming BOOLEAN, + supports_reasoning_effort BOOLEAN, + context_window_tokens INTEGER, + max_output_tokens INTEGER, + input_usd_per_m_tok NUMERIC(12, 4), + output_usd_per_m_tok NUMERIC(12, 4), + source VARCHAR(16), + discovered_at TIMESTAMP WITH TIME ZONE, + last_test_at TIMESTAMP WITH TIME ZONE, + last_test_status VARCHAR(16), + last_test_latency_ms INTEGER, + last_test_error VARCHAR(500), + create_date TIMESTAMP WITH TIME ZONE, + modify_date TIMESTAMP WITH TIME ZONE, + UNIQUE (provider_id, model_id) +); +``` + +Register the optional `qbit-assistant-provider-openai` module and +`OpenAiCompatibleProviderType` to offer its kinds. Azure and generic endpoints +require an explicit `allowedBaseUrls` entry; local defaults may be keyless. +Discover models preserves existing rows. Cloud catalogs larger than 20 are added +disabled; enable the models to offer before leaving the provider's model empty. +HIPAA eligibility records an administrator's fact; P-a does not implement PHI routing. + +Custom CA bundles are server configuration, never database content. Register them +with `withTrustedCaBundle(name, pem)` or `withTrustedCaBundleFile(name, path)` at +startup, then set the HTTPS provider row's `tlsCaBundle` to that name. Empty +references retain JVM default trust. The bundle adds trust without replacing JVM +roots; hostname verification remains enabled. If the P-a columns were already +applied, add only `tls_ca_bundle VARCHAR(100)` to `assistant_provider`. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index ad12145..536700d 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -309,6 +309,30 @@ AI exposure NONE is not enough, because the assistant acts as the user and an ad - **scheduled assistant tasks**: recurring prompts that run as a designated user (with that user's stored credentials through the UI API, never a system session) and post results as notifications or records. 6. Everything goes through the `/qqq/v1` UI API, as the user, with the guarantees above. +### P-a as built + +- Per-provider `tlsCaBundle` references startup-parsed, transient server CA bundles. OpenAI-compatible chat, health and discovery trust JVM roots plus the bundle, retain hostname verification, and report sanitized NETWORK errors with a trust hint. +- `assistant_provider` records kind, location, HIPAA eligibility, region, API + version, timeouts and discovery status. `assistant_model` is its admin-only + child list: enablement, capabilities, prices and test status, unique per provider/model. +- `ProviderSettings`, discovery and health extend the provider SPI through default + methods. Legacy `create(apiKey, baseUrl)` implementations keep working. +- The OpenAI-compatible module uses JDK HTTP and Jackson, Chat Completions with + SSE or single JSON replies, Azure deployment URLs and api-key auth, and keyless + local endpoints. Keys and URL overrides pass the existing allowlists first. +- Strict tools require explicit opt-in, OpenAI or Azure, and recursively strict + object schemas. A strict/schema/grammar 400 signals the existing one-retry fallback. +- Discovery only adds missing models; it preserves edits and disables catalogs + larger than 20 for cloud providers. Health uses a short models request. Provider + tests probe health then complete; model tests can probe a disabled model before + enabling it. Completion tests record TEST usage. +- Registry prices and capabilities override adapter/type defaults. Empty provider + model selects the first enabled registry model; an explicitly disabled selection + is a setup error. Rows without models retain the prior behavior. +- Localhost contract tests cover dialects, fragmented parallel tool calls, usage, + errors, timeouts, cancellation, discovery and a golden tool round trip. The live + smoke is environment-driven and excluded by default. + ### Phases (each a green PR) | | Scope | Depends on | |---|---|---| diff --git a/docs/schema-postgres.sql b/docs/schema-postgres.sql index 8ed61b9..f37661c 100644 --- a/docs/schema-postgres.sql +++ b/docs/schema-postgres.sql @@ -30,7 +30,7 @@ CREATE TABLE assistant_message status VARCHAR(16), turn_id INTEGER, seq INTEGER, - model VARCHAR(64), + model VARCHAR(200), parts_json TEXT, create_date TIMESTAMP WITH TIME ZONE, modify_date TIMESTAMP WITH TIME ZONE @@ -49,7 +49,7 @@ CREATE TABLE assistant_turn stop_reason VARCHAR(64), error_code VARCHAR(32), cancel_reason VARCHAR(64), - model VARCHAR(64), + model VARCHAR(200), started_at TIMESTAMP WITH TIME ZONE, ended_at TIMESTAMP WITH TIME ZONE, tool_call_count INTEGER, @@ -173,10 +173,21 @@ CREATE TABLE assistant_provider id SERIAL PRIMARY KEY, name VARCHAR(100) NOT NULL UNIQUE, provider_type VARCHAR(50) NOT NULL, - model VARCHAR(100), + kind VARCHAR(30), + location VARCHAR(10), + hipaa_eligible BOOLEAN DEFAULT FALSE, + region VARCHAR(50), + api_version VARCHAR(30), + connect_timeout_seconds INTEGER DEFAULT 10 CHECK (connect_timeout_seconds BETWEEN 1 AND 600), + request_timeout_seconds INTEGER DEFAULT 120 CHECK (request_timeout_seconds BETWEEN 1 AND 600), + last_discovery_at TIMESTAMP WITH TIME ZONE, + last_discovery_status VARCHAR(16), + last_discovery_error VARCHAR(500), + model VARCHAR(200), enabled BOOLEAN, is_default BOOLEAN, base_url VARCHAR(255), + tls_ca_bundle VARCHAR(100), api_key_env_name VARCHAR(100), api_key_status VARCHAR(200), monthly_cost_cap_usd NUMERIC(12, 2), @@ -185,12 +196,38 @@ CREATE TABLE assistant_provider last_test_at TIMESTAMP WITH TIME ZONE, last_test_status VARCHAR(16), last_test_latency_ms INTEGER, - last_test_model VARCHAR(100), + last_test_model VARCHAR(200), last_test_error VARCHAR(500), create_date TIMESTAMP WITH TIME ZONE, modify_date TIMESTAMP WITH TIME ZONE ); +CREATE TABLE assistant_model +( + id SERIAL PRIMARY KEY, + provider_id INTEGER NOT NULL REFERENCES assistant_provider(id), + model_id VARCHAR(200) NOT NULL, + display_name VARCHAR(200), + enabled BOOLEAN, + supports_tools BOOLEAN, + supports_vision BOOLEAN, + supports_streaming BOOLEAN, + supports_reasoning_effort BOOLEAN, + context_window_tokens INTEGER, + max_output_tokens INTEGER, + input_usd_per_m_tok NUMERIC(12, 4), + output_usd_per_m_tok NUMERIC(12, 4), + source VARCHAR(16), + discovered_at TIMESTAMP WITH TIME ZONE, + last_test_at TIMESTAMP WITH TIME ZONE, + last_test_status VARCHAR(16), + last_test_latency_ms INTEGER, + last_test_error VARCHAR(500), + create_date TIMESTAMP WITH TIME ZONE, + modify_date TIMESTAMP WITH TIME ZONE, + UNIQUE (provider_id, model_id) +); + CREATE TABLE assistant_model_price ( id SERIAL PRIMARY KEY, @@ -215,7 +252,7 @@ CREATE TABLE assistant_usage turn_id INTEGER, provider_id INTEGER, provider_type VARCHAR(50), - model VARCHAR(100), + model VARCHAR(200), purpose VARCHAR(16), status VARCHAR(16), error_kind VARCHAR(32), diff --git a/pom.xml b/pom.xml index 6e823ea..132c592 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ qbit-assistant qbit-assistant-provider-anthropic + qbit-assistant-provider-openai @@ -101,6 +102,11 @@ ${revision} test-jar + + com.kingsrook.qbits + qbit-assistant-provider-openai + ${revision} + com.anthropic anthropic-java diff --git a/qbit-assistant-provider-openai/pom.xml b/qbit-assistant-provider-openai/pom.xml new file mode 100644 index 0000000..b99bee1 --- /dev/null +++ b/qbit-assistant-provider-openai/pom.xml @@ -0,0 +1,128 @@ + + + + + 4.0.0 + + + com.kingsrook.qbits + qbit-assistant-parent + ${revision} + + + qbit-assistant-provider-openai + jar + + QBit Assistant - OpenAI-compatible provider + OpenAI-compatible Chat Completions provider using the JDK HTTP client + https://github.com/QRun-IO/qbit-assistant + + + + com.kingsrook.qbits + qbit-assistant + + + com.kingsrook.qqq + qqq-backend-core + + + com.fasterxml.jackson.core + jackson-databind + + + + com.kingsrook.qbits + qbit-assistant + test-jar + test + + + com.kingsrook.qqq + qqq-middleware-javalin + test + + + org.junit.jupiter + junit-jupiter + test + + + org.assertj + assertj-core + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + ${test.excludedGroups} + true + + + + org.jacoco + jacoco-maven-plugin + + + prepare-agent + + prepare-agent + + + + report + verify + + report + + + + check + + check + + + + + BUNDLE + + + INSTRUCTION + COVEREDRATIO + 0.70 + + + CLASS + COVEREDRATIO + 0.90 + + + + + + + + + + + diff --git a/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiCompatibleModelProvider.java b/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiCompatibleModelProvider.java new file mode 100644 index 0000000..ac0ae2e --- /dev/null +++ b/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiCompatibleModelProvider.java @@ -0,0 +1,421 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.provider.openai; + + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.kingsrook.qbits.assistant.provider.AssistantModelProvider; +import com.kingsrook.qbits.assistant.provider.CompletableStreamHandle; +import com.kingsrook.qbits.assistant.provider.ModelCapabilities; +import com.kingsrook.qbits.assistant.provider.ModelEventListener; +import com.kingsrook.qbits.assistant.provider.ModelPart; +import com.kingsrook.qbits.assistant.provider.ModelProviderException; +import com.kingsrook.qbits.assistant.provider.ModelRequest; +import com.kingsrook.qbits.assistant.provider.ModelResponse; +import com.kingsrook.qbits.assistant.provider.ModelUsage; +import com.kingsrook.qbits.assistant.provider.ProviderSettings; +import com.kingsrook.qbits.assistant.provider.StreamHandle; + + +/******************************************************************************* + ** JDK HTTP Chat Completions adapter. Only visible content is emitted or stored; + ** reasoning deltas are ignored. Each call owns its cancellation and read guard. + *******************************************************************************/ +public class OpenAiCompatibleModelProvider implements AssistantModelProvider +{ + private final OpenAiHttp http; + private final boolean local; + + + + /******************************************************************************* + ** Build a client with resolved settings. + *******************************************************************************/ + public OpenAiCompatibleModelProvider(ProviderSettings settings) + { + http = new OpenAiHttp(settings); + local = "LOCAL".equals(new OpenAiCompatibleProviderType().getDefaultLocation(http.settings.getKind())); + } + + + + /******************************************************************************* + ** Adapter identity. + *******************************************************************************/ + @Override + public String getName() + { + return ("openai-compatible"); + } + + + + /******************************************************************************* + ** Conservative defaults; registry rows can override model-specific facts. + *******************************************************************************/ + @Override + public ModelCapabilities getCapabilities(String modelId) + { + return (new ModelCapabilities().withTools(true).withParallelToolCalls(true).withPromptCaching(false).withThinking(false) + .withMaxContextTokens(local ? 32_000 : 128_000).withStreaming(true).withVision(false).withReasoningEffort(false)); + } + + + + /******************************************************************************* + ** Interrupting the worker aborts the HTTP call, including a blocked body read. + *******************************************************************************/ + @Override + public StreamHandle stream(ModelRequest request, ModelEventListener listener) + { + CompletableStreamHandle handle = new CompletableStreamHandle(); + AtomicReference input = new AtomicReference<>(); + Thread worker = Thread.ofVirtual().unstarted(() -> run(request, listener, handle, input)); + handle.setOnCancel(() -> + { + worker.interrupt(); + close(input.get()); + }); + worker.start(); + return (handle); + } + + + + /******************************************************************************* + ** Guard the entire request and idle reads, including after HTTP headers. + *******************************************************************************/ + private void run(ModelRequest request, ModelEventListener listener, CompletableStreamHandle handle, AtomicReference input) + { + Thread worker = Thread.currentThread(); + AtomicLong lastRead = new AtomicLong(System.nanoTime()); + long started = lastRead.get(); + long timeout = TimeUnit.SECONDS.toNanos(http.settings.getRequestTimeoutSeconds()); + ScheduledExecutorService guard = Executors.newSingleThreadScheduledExecutor(Thread.ofVirtual().factory()); + Accumulator result = new Accumulator(request.getModel(), listener); + handle.setPartialUsage(() -> result.usage); + guard.scheduleAtFixedRate(() -> + { + long now = System.nanoTime(); + if(now - lastRead.get() >= timeout || now - started >= timeout) + { + handle.fail(new ModelProviderException(ModelProviderException.Kind.TIMEOUT, null, "The provider did not answer in time.").withPartialUsage(result.usage)); + worker.interrupt(); + close(input.get()); + } + }, 100, 100, TimeUnit.MILLISECONDS); + try + { + ModelCapabilities caps = request.getCapabilities() == null ? getCapabilities(request.getModel()) : request.getCapabilities(); + ObjectNode body = OpenAiMapping.request(request, http.settings.getKind(), caps); + boolean strict = false; + for(JsonNode tool : body.path("tools")) + { + strict |= tool.path("function").path("strict").asBoolean(); + } + HttpResponse response = http.client.send(http.request(request.getModel(), http.settings.getRequestTimeoutSeconds()) + .POST(java.net.http.HttpRequest.BodyPublishers.ofString(body.toString())).build(), HttpResponse.BodyHandlers.ofInputStream()); + input.set(response.body()); + result.requestId = OpenAiHttp.requestId(response); + if(handle.isCancelled()) + { + return; + } + if(response.statusCode() != 200) + { + throw (http.error(response.statusCode(), new String(response.body().readNBytes(64 * 1024), StandardCharsets.UTF_8), result.requestId, strict)); + } + if(Boolean.TRUE.equals(caps.getStreaming())) + { + readEvents(response.body(), lastRead, result, strict, handle); + } + else + { + result.accept(OpenAiHttp.JSON.readTree(response.body()), false, http, strict); + } + handle.complete(result.response()); + } + catch(ModelProviderException e) + { + handle.fail(e.withPartialUsage(result.usage)); + } + catch(HttpTimeoutException e) + { + handle.fail(new ModelProviderException(ModelProviderException.Kind.TIMEOUT, null, "The provider did not answer in time.").withPartialUsage(result.usage)); + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + if(!handle.isCancelled()) + { + handle.fail(new ModelProviderException(ModelProviderException.Kind.NETWORK, null, "The provider request was interrupted.")); + } + } + catch(IOException | RuntimeException e) + { + handle.fail(OpenAiHttp.transportError(e).withPartialUsage(result.usage)); + } + finally + { + guard.shutdownNow(); + close(input.get()); + } + } + + + + /******************************************************************************* + ** SSE data fields, CRLF, comments, multi-line data and an optional DONE marker. + *******************************************************************************/ + private void readEvents(InputStream stream, AtomicLong lastRead, Accumulator result, boolean strict, CompletableStreamHandle handle) throws IOException, ModelProviderException + { + BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8)); + StringBuilder data = new StringBuilder(); + String line; + while(!handle.isCancelled() && (line = reader.readLine()) != null) + { + lastRead.set(System.nanoTime()); + if(line.isEmpty()) + { + if(data.length() > 0) + { + if("[DONE]".equals(data.toString().trim())) + { + return; + } + result.accept(OpenAiHttp.JSON.readTree(data.toString()), true, http, strict); + data.setLength(0); + } + } + else if(line.startsWith("data:")) + { + if(data.length() > 0) + { + data.append('\n'); + } + data.append(line.substring(5).stripLeading()); + } + } + if(data.length() > 0 && !"[DONE]".equals(data.toString().trim())) + { + result.accept(OpenAiHttp.JSON.readTree(data.toString()), true, http, strict); + } + } + + + + /******************************************************************************* + ** Closing an already cancelled stream is harmless. + *******************************************************************************/ + private static void close(InputStream input) + { + if(input != null) + { + try + { + input.close(); + } + catch(IOException e) + { + // The aborted HTTP call may already have closed the stream. + } + } + } + + + + /******************************************************************************* + ** Tool arguments arrive as JSON fragments, ordered by call index. + *******************************************************************************/ + private static class Tool + { + private String id = ""; + private String name = ""; + private final StringBuilder arguments = new StringBuilder(); + private boolean announced; + } + + + + /******************************************************************************* + ** Shared mapping for stream chunks and one-shot responses. + *******************************************************************************/ + private static class Accumulator + { + private final ModelEventListener listener; + private final StringBuilder text = new StringBuilder(); + private final Map tools = new TreeMap<>(); + private volatile ModelUsage usage = new ModelUsage(); + private String model; + private String requestId; + private String stop = ModelResponse.STOP_END_TURN; + + + + /******************************************************************************* + ** One completion. + *******************************************************************************/ + Accumulator(String model, ModelEventListener listener) + { + this.model = model; + this.listener = listener; + } + + + + /******************************************************************************* + ** Ignore reasoning fields; retain only text, tools and billable usage. + *******************************************************************************/ + void accept(JsonNode chunk, boolean streaming, OpenAiHttp http, boolean strict) throws ModelProviderException + { + if(chunk.has("error")) + { + JsonNode error = chunk.path("error"); + int status = error.path("status").asInt(error.path("code").asInt(500)); + String type = error.path("type").asText(); + if(status == 500) + { + status = switch(type) + { + case "invalid_request_error" -> 400; + case "authentication_error" -> 401; + case "rate_limit_error" -> 429; + case "overloaded_error" -> 503; + default -> 500; + }; + } + throw (http.error(status, chunk.toString(), requestId, strict)); + } + model = chunk.path("model").asText(model); + if(chunk.hasNonNull("usage")) + { + JsonNode used = chunk.get("usage"); + long cached = used.path("prompt_tokens_details").path("cached_tokens").asLong(); + usage = new ModelUsage().withInputTokens(Math.max(0, used.path("prompt_tokens").asLong() - cached)) + .withOutputTokens(used.path("completion_tokens").asLong()).withCacheReadTokens(cached); + } + for(JsonNode choice : chunk.path("choices")) + { + if(choice.path("index").asInt() != 0) + { + continue; + } + JsonNode delta = choice.path(streaming ? "delta" : "message"); + String content = delta.path("content").asText(""); + if(!content.isEmpty()) + { + text.append(content); + listener.onTextDelta(content); + } + int position = 0; + for(JsonNode call : delta.path("tool_calls")) + { + Tool tool = tools.computeIfAbsent(call.path("index").asInt(position++), k -> new Tool()); + if(tool.id.isEmpty()) + { + tool.id = call.path("id").asText(""); + } + if(tool.name.isEmpty()) + { + tool.name = call.path("function").path("name").asText(""); + } + tool.arguments.append(call.path("function").path("arguments").asText("")); + if(!tool.announced && !tool.id.isEmpty() && !tool.name.isEmpty()) + { + listener.onToolUseStart(tool.id, tool.name); + tool.announced = true; + } + } + if(choice.hasNonNull("finish_reason")) + { + stop = switch(choice.get("finish_reason").asText()) + { + case "tool_calls", "function_call" -> ModelResponse.STOP_TOOL_USE; + case "length" -> ModelResponse.STOP_MAX_TOKENS; + case "content_filter" -> ModelResponse.STOP_REFUSAL; + default -> ModelResponse.STOP_END_TURN; + }; + } + } + } + + + + /******************************************************************************* + ** Parse complete argument objects only, preserving parallel call order. + *******************************************************************************/ + ModelResponse response() + { + List parts = new ArrayList<>(); + if(!text.isEmpty()) + { + parts.add(ModelPart.text(text.toString())); + } + for(Map.Entry entry : tools.entrySet()) + { + Tool tool = entry.getValue(); + if(tool.id.isEmpty()) + { + tool.id = "call_" + entry.getKey(); + } + if(!tool.announced) + { + listener.onToolUseStart(tool.id, tool.name); + tool.announced = true; + } + Map arguments; + try + { + arguments = tool.arguments.isEmpty() ? Map.of() : OpenAiHttp.JSON.readerFor(new TypeReference>() {}) + .with(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_TRAILING_TOKENS).readValue(tool.arguments.toString()); + if(arguments == null) + { + throw (new IOException("Tool arguments must be an object.")); + } + } + catch(IOException e) + { + arguments = Map.of("_invalid_arguments", tool.arguments.substring(0, Math.min(200, tool.arguments.length()))); + } + parts.add(ModelPart.toolUse(tool.id, tool.name, arguments)); + } + if(!tools.isEmpty() && ModelResponse.STOP_END_TURN.equals(stop)) + { + stop = ModelResponse.STOP_TOOL_USE; + } + return (new ModelResponse().withModel(model).withRequestId(requestId).withParts(parts).withUsage(usage).withStopReason(stop)); + } + } +} diff --git a/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiCompatibleProviderType.java b/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiCompatibleProviderType.java new file mode 100644 index 0000000..f06ce50 --- /dev/null +++ b/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiCompatibleProviderType.java @@ -0,0 +1,199 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.provider.openai; + + +import java.util.List; +import com.kingsrook.qbits.assistant.provider.AssistantModelProvider; +import com.kingsrook.qbits.assistant.provider.AssistantProviderType; +import com.kingsrook.qbits.assistant.provider.DiscoveredModel; +import com.kingsrook.qbits.assistant.provider.HealthResult; +import com.kingsrook.qbits.assistant.provider.ModelProviderException; +import com.kingsrook.qbits.assistant.provider.ProviderSettings; + + +/******************************************************************************* + ** One installed adapter for the Chat Completions dialects. + *******************************************************************************/ +public class OpenAiCompatibleProviderType implements AssistantProviderType +{ + /******************************************************************************* + ** Installed adapter id. + *******************************************************************************/ + @Override + public String getType() + { + return ("openai-compatible"); + } + + + + /******************************************************************************* + ** Settings label. + *******************************************************************************/ + @Override + public String getLabel() + { + return ("OpenAI-compatible (OpenAI, Azure, OpenRouter, Ollama, LM Studio, vLLM, llama.cpp)"); + } + + + + /******************************************************************************* + ** Discover endpoint-specific models instead of guessing ids. + *******************************************************************************/ + @Override + public List getModels() + { + return (List.of()); + } + + + + /******************************************************************************* + ** Extensible kind list. + *******************************************************************************/ + @Override + public List getKinds() + { + return (List.of("OPENAI", "AZURE_OPENAI", "OPENROUTER", "OLLAMA", "LM_STUDIO", "VLLM", "LLAMA_CPP", "OPENAI_COMPATIBLE")); + } + + + + /******************************************************************************* + ** The legacy factory uses the generic dialect. + *******************************************************************************/ + @Override + public String getDefaultApiKeyEnvName() + { + return (null); + } + + + + /******************************************************************************* + ** The generic dialect requires an explicit URL. + *******************************************************************************/ + @Override + public String getDefaultBaseUrl() + { + return (null); + } + + + + /******************************************************************************* + ** Built-in endpoints. Redirects are never followed. + *******************************************************************************/ + @Override + public String getDefaultBaseUrl(String kind) + { + return (switch(kind) + { + case "OPENAI" -> "https://api.openai.com/v1"; + case "OPENROUTER" -> "https://openrouter.ai/api/v1"; + case "OLLAMA" -> "http://localhost:11434/v1"; + case "LM_STUDIO" -> "http://localhost:1234/v1"; + case "VLLM" -> "http://localhost:8000/v1"; + case "LLAMA_CPP" -> "http://localhost:8080/v1"; + default -> null; + }); + } + + + + /******************************************************************************* + ** Default key references, never key values. + *******************************************************************************/ + @Override + public String getDefaultApiKeyEnvName(String kind) + { + return (switch(kind) + { + case "OPENAI" -> "OPENAI_API_KEY"; + case "AZURE_OPENAI" -> "AZURE_OPENAI_API_KEY"; + case "OPENROUTER" -> "OPENROUTER_API_KEY"; + default -> null; + }); + } + + + + /******************************************************************************* + ** Location is an admin-editable fact, with a kind-based default. + *******************************************************************************/ + @Override + public String getDefaultLocation(String kind) + { + return (List.of("OLLAMA", "LM_STUDIO", "VLLM", "LLAMA_CPP").contains(kind) ? "LOCAL" : "CLOUD"); + } + + + + /******************************************************************************* + ** Backward-compatible factory. + *******************************************************************************/ + @Override + public AssistantModelProvider create(String apiKey, String baseUrl) + { + return (create(new ProviderSettings().withKind("OPENAI_COMPATIBLE").withApiKey(apiKey).withBaseUrl(baseUrl))); + } + + + + /******************************************************************************* + ** Full connection settings. + *******************************************************************************/ + @Override + public AssistantModelProvider create(ProviderSettings settings) + { + return (new OpenAiCompatibleModelProvider(settings)); + } + + + + /******************************************************************************* + ** Only a models request, never a completion. + *******************************************************************************/ + @Override + public List discoverModels(ProviderSettings settings) throws ModelProviderException + { + return (new OpenAiHttp(settings).models(settings.getRequestTimeoutSeconds())); + } + + + + /******************************************************************************* + ** A short models probe with latency and count. + *******************************************************************************/ + @Override + public HealthResult checkHealth(ProviderSettings settings) + { + long start = System.nanoTime(); + try + { + List models = new OpenAiHttp(settings).models(Math.min(10, settings.getRequestTimeoutSeconds())); + return (new HealthResult(true, (System.nanoTime() - start) / 1_000_000, models.size() + " models available")); + } + catch(ModelProviderException e) + { + return (new HealthResult(false, (System.nanoTime() - start) / 1_000_000, + e.getKind() + (e.getHttpStatus() == null ? "" : " HTTP " + e.getHttpStatus()) + ": " + e.getMessage())); + } + } +} diff --git a/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiHttp.java b/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiHttp.java new file mode 100644 index 0000000..048cb94 --- /dev/null +++ b/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiHttp.java @@ -0,0 +1,267 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.provider.openai; + + +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import javax.net.ssl.SSLHandshakeException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.kingsrook.qbits.assistant.provider.DiscoveredModel; +import com.kingsrook.qbits.assistant.provider.ModelProviderException; +import com.kingsrook.qbits.assistant.provider.ProviderSettings; + + +/******************************************************************************* + ** Shared HTTP/authentication/error mapping. ProviderService guards URLs before + ** constructing this transport; redirects must not move keys to another host. + *******************************************************************************/ +final class OpenAiHttp +{ + static final ObjectMapper JSON = new ObjectMapper(); + final HttpClient client; + final ProviderSettings settings; + private final String base; + + + + /******************************************************************************* + ** Copy settings so callers cannot mutate a cached client's credentials. + *******************************************************************************/ + OpenAiHttp(ProviderSettings source) + { + settings = new ProviderSettings().withKind(source.getKind() == null ? "OPENAI_COMPATIBLE" : source.getKind()) + .withTlsCaBundle(source.getTlsCaBundle()).withSslContext(source.getSslContext()) + .withApiKey(source.getApiKey()).withBaseUrl(source.getBaseUrl()).withApiVersion(source.getApiVersion()).withRegion(source.getRegion()) + .withConnectTimeoutSeconds(source.getConnectTimeoutSeconds()).withRequestTimeoutSeconds(source.getRequestTimeoutSeconds()); + String url = settings.getBaseUrl() == null ? new OpenAiCompatibleProviderType().getDefaultBaseUrl(settings.getKind()) : settings.getBaseUrl(); + if(url == null) + { + throw (new IllegalArgumentException("A base URL is required for this provider kind.")); + } + base = url.replaceAll("/+$", ""); + HttpClient.Builder builder = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(settings.getConnectTimeoutSeconds())).followRedirects(HttpClient.Redirect.NEVER); + if(settings.getSslContext() != null) + { + builder.sslContext(settings.getSslContext()); + } + client = builder.build(); + } + + + + /******************************************************************************* + ** Azure uses deployment paths and a version query parameter. + *******************************************************************************/ + HttpRequest.Builder request(String model, Integer seconds) throws ModelProviderException + { + try + { + String path = model == null ? "/models" : "/chat/completions"; + if("AZURE_OPENAI".equals(settings.getKind())) + { + path = model == null ? "/openai/models" : "/openai/deployments/" + encode(model) + "/chat/completions"; + path += "?api-version=" + encode(settings.getApiVersion() == null || settings.getApiVersion().isBlank() ? "2024-10-21" : settings.getApiVersion()); + } + HttpRequest.Builder builder = HttpRequest.newBuilder(URI.create(base + path)).timeout(Duration.ofSeconds(seconds)).header("Content-Type", "application/json"); + if(settings.getApiKey() != null && !settings.getApiKey().isBlank()) + { + if("AZURE_OPENAI".equals(settings.getKind())) + { + builder.header("api-key", settings.getApiKey()); + } + else + { + builder.header("Authorization", "Bearer " + settings.getApiKey()); + } + } + return (builder); + } + catch(IllegalArgumentException e) + { + throw (new ModelProviderException(ModelProviderException.Kind.INVALID_REQUEST, null, "The provider connection settings are invalid.")); + } + } + + + + /******************************************************************************* + ** Models with optional server-specific context information. + *******************************************************************************/ + List models(Integer seconds) throws ModelProviderException + { + CompletableFuture> pending = client.sendAsync(request(null, seconds).GET().build(), HttpResponse.BodyHandlers.ofString()); + try + { + HttpResponse response = pending.get(seconds, TimeUnit.SECONDS); + if(response.statusCode() != 200) + { + throw (error(response.statusCode(), response.body(), requestId(response), false)); + } + JsonNode body = JSON.readTree(response.body()); + if(body.has("error")) + { + throw (error(500, response.body(), requestId(response), false)); + } + List result = new ArrayList<>(); + for(JsonNode model : body.path("data")) + { + String id = model.path("id").asText(); + if(id.isBlank()) + { + continue; + } + int context = model.path("context_length").asInt(model.path("max_model_len").asInt(model.path("meta").path("n_ctx_train").asInt())); + result.add(new DiscoveredModel(id, model.path("name").asText(id), context > 0 ? context : null, model.path("owned_by").asText(null))); + } + return (result); + } + catch(TimeoutException e) + { + throw (new ModelProviderException(ModelProviderException.Kind.TIMEOUT, null, "The provider did not answer in time.")); + } + catch(ExecutionException e) + { + throw (transportError(e)); + } + catch(InterruptedException e) + { + Thread.currentThread().interrupt(); + throw (new ModelProviderException(ModelProviderException.Kind.NETWORK, null, "The provider request was interrupted.")); + } + catch(IOException e) + { + throw (transportError(e)); + } + finally + { + pending.cancel(true); + client.close(); + } + } + + + + /******************************************************************************* + ** Sanitize errors, explicitly removing even short or unusual API keys. + *******************************************************************************/ + ModelProviderException error(int status, String body, String requestId, boolean strict) + { + String message = "The provider refused the request."; + String type = "openai-compatible"; + try + { + JsonNode error = JSON.readTree(body).path("error"); + message = error.path("message").asText(message); + type = error.path("type").asText(type); + } + catch(Exception e) + { + // Do not expose unstructured proxy pages or request bodies. + } + message = safe(message); + type = safe(type); + ModelProviderException.Kind kind = errorKind(status); + String lower = message == null ? "" : message.toLowerCase(Locale.ROOT); + return (new ModelProviderException(kind, status, message).withProviderDetails(type, message, safe(requestId)) + .withStrictSchemaRejected(strict && status == 400 && (lower.contains("strict") || lower.contains("schema") || lower.contains("grammar")))); + } + + + + /******************************************************************************* + ** Credential-free diagnostics. + *******************************************************************************/ + private String safe(String value) + { + String key = settings.getApiKey(); + return (ModelProviderException.sanitizeProviderMessage(value == null || key == null || key.isEmpty() ? value : value.replace(key, "[redacted]"), 300)); + } + + + + /******************************************************************************* + ** Common request-id headers. + *******************************************************************************/ + static String requestId(HttpResponse response) + { + return (response.headers().firstValue("x-request-id").orElse(response.headers().firstValue("apim-request-id").orElse(null))); + } + + + + /******************************************************************************* + ** Encode a single path segment or query value. + *******************************************************************************/ + private static String encode(String value) + { + return (URLEncoder.encode(value, StandardCharsets.UTF_8).replace("+", "%20")); + } + + + + /******************************************************************************* + ** Map the provider error category. + *******************************************************************************/ + private static ModelProviderException.Kind errorKind(int status) + { + return (switch(status) + { + case 401, 403 -> ModelProviderException.Kind.AUTHENTICATION; + case 429 -> ModelProviderException.Kind.RATE_LIMITED; + case 503, 529 -> ModelProviderException.Kind.OVERLOADED; + case 400, 404, 422 -> ModelProviderException.Kind.INVALID_REQUEST; + default -> status >= 500 ? ModelProviderException.Kind.SERVER : ModelProviderException.Kind.INVALID_REQUEST; + }); + } + + + /******************************************************************************* + ** Fixed diagnostics omit remote certificate details, URLs and credentials. + *******************************************************************************/ + static ModelProviderException transportError(Throwable error) + { + for(Throwable cause = error; cause != null; cause = cause.getCause()) + { + if(cause instanceof SSLHandshakeException) + { + return (new ModelProviderException(ModelProviderException.Kind.NETWORK, null, + "The provider TLS certificate is not trusted or does not match the host. Check tlsCaBundle and the HTTPS hostname.")); + } + if(cause instanceof HttpTimeoutException) + { + return (new ModelProviderException(ModelProviderException.Kind.TIMEOUT, null, "The provider did not answer in time.")); + } + } + return (new ModelProviderException(ModelProviderException.Kind.NETWORK, null, "Could not read the provider response.")); + } +} diff --git a/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiMapping.java b/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiMapping.java new file mode 100644 index 0000000..efc466f --- /dev/null +++ b/qbit-assistant-provider-openai/src/main/java/com/kingsrook/qbits/assistant/provider/openai/OpenAiMapping.java @@ -0,0 +1,190 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.provider.openai; + + +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.kingsrook.qbits.assistant.provider.ModelCapabilities; +import com.kingsrook.qbits.assistant.provider.ModelMessage; +import com.kingsrook.qbits.assistant.provider.ModelPart; +import com.kingsrook.qbits.assistant.provider.ModelRequest; +import com.kingsrook.qbits.assistant.provider.ModelToolDefinition; +import com.kingsrook.qbits.assistant.provider.SystemBlock; + + +/******************************************************************************* + ** Chat Completions request mapping and conservative strict-schema checks. + *******************************************************************************/ +final class OpenAiMapping +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private OpenAiMapping() + { + } + + + + /******************************************************************************* + ** Map only capabilities the target dialect supports. + *******************************************************************************/ + static ObjectNode request(ModelRequest request, String kind, ModelCapabilities capabilities) throws JsonProcessingException + { + ObjectNode body = OpenAiHttp.JSON.createObjectNode(); + boolean streaming = Boolean.TRUE.equals(capabilities.getStreaming()); + boolean nativeOpenAi = List.of("OPENAI", "AZURE_OPENAI").contains(kind); + body.put("model", request.getModel()); + body.put("stream", streaming); + if(streaming) + { + body.putObject("stream_options").put("include_usage", true); + } + if(request.getMaxTokens() != null) + { + body.put(nativeOpenAi ? "max_completion_tokens" : "max_tokens", request.getMaxTokens()); + } + if(Boolean.TRUE.equals(capabilities.getReasoningEffort()) && request.getEffort() != null) + { + String effort = request.getEffort().toLowerCase(Locale.ROOT); + body.put("reasoning_effort", List.of("low", "medium").contains(effort) ? effort : "high"); + } + ArrayNode messages = body.putArray("messages"); + if(!request.getSystem().isEmpty()) + { + messages.addObject().put("role", "system").put("content", request.getSystem().stream().map(SystemBlock::getText).collect(Collectors.joining("\n\n"))); + } + for(ModelMessage message : request.getMessages()) + { + ObjectNode current = null; + for(ModelPart part : message.getParts()) + { + if(part.getType() == ModelPart.Type.TOOL_RESULT) + { + current = null; + messages.addObject().put("role", "tool").put("tool_call_id", part.getToolUseId()) + .put("content", (Boolean.TRUE.equals(part.getIsError()) ? "ERROR: " : "") + part.getText()); + continue; + } + if(current == null) + { + current = messages.addObject().put("role", message.getRole() == ModelMessage.Role.ASSISTANT ? "assistant" : "user"); + current.put("content", ""); + } + if(part.getType() == ModelPart.Type.TEXT) + { + current.put("content", current.path("content").asText() + part.getText()); + } + else if(part.getType() == ModelPart.Type.TOOL_USE) + { + ArrayNode calls = current.has("tool_calls") ? (ArrayNode) current.get("tool_calls") : current.putArray("tool_calls"); + calls.addObject().put("id", part.getToolUseId()).put("type", "function").putObject("function") + .put("name", part.getToolName()).put("arguments", OpenAiHttp.JSON.writeValueAsString(part.getInput())); + } + } + } + if(!request.getTools().isEmpty() && Boolean.TRUE.equals(capabilities.getTools())) + { + ArrayNode tools = body.putArray("tools"); + for(ModelToolDefinition tool : request.getTools()) + { + JsonNode schema = OpenAiHttp.JSON.valueToTree(tool.getInputSchema()); + ObjectNode function = tools.addObject().put("type", "function").putObject("function").put("name", tool.getName()).put("description", tool.getDescription()); + function.set("parameters", schema); + if(nativeOpenAi && Boolean.TRUE.equals(tool.getStrict()) && strictCompatible(schema)) + { + function.put("strict", true); + } + } + if(List.of("OPENAI", "AZURE_OPENAI", "OPENROUTER", "VLLM").contains(kind)) + { + body.put("parallel_tool_calls", Boolean.TRUE.equals(request.getParallelToolCalls()) && Boolean.TRUE.equals(capabilities.getParallelToolCalls())); + } + } + return (body); + } + + + + /******************************************************************************* + ** Every object must forbid extras and require every declared property. + *******************************************************************************/ + static boolean strictCompatible(JsonNode schema) + { + if(schema == null || !schema.isObject()) + { + return (false); + } + JsonNode type = schema.path("type"); + if("object".equals(type.asText()) || schema.has("properties") || (type.isArray() && type.toString().contains("\"object\""))) + { + if(!schema.has("additionalProperties") || !schema.get("additionalProperties").isBoolean() || schema.get("additionalProperties").asBoolean()) + { + return (false); + } + JsonNode required = schema.path("required"); + var names = schema.path("properties").fieldNames(); + while(names.hasNext()) + { + String name = names.next(); + boolean found = false; + for(JsonNode value : required) + { + found |= name.equals(value.asText()); + } + if(!found || !strictCompatible(schema.path("properties").get(name))) + { + return (false); + } + } + } + for(String keyword : List.of("items", "additionalProperties")) + { + if(schema.path(keyword).isObject() && !strictCompatible(schema.get(keyword))) + { + return (false); + } + } + for(String keyword : List.of("anyOf", "oneOf", "allOf", "prefixItems")) + { + for(JsonNode child : schema.path(keyword)) + { + if(!strictCompatible(child)) + { + return (false); + } + } + } + for(String keyword : List.of("$defs", "definitions")) + { + for(JsonNode child : schema.path(keyword)) + { + if(!strictCompatible(child)) + { + return (false); + } + } + } + return (true); + } +} diff --git a/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/LiveOpenAiCompatibleTest.java b/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/LiveOpenAiCompatibleTest.java new file mode 100644 index 0000000..b84349a --- /dev/null +++ b/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/LiveOpenAiCompatibleTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.provider.openai; + + +import java.nio.file.Path; +import java.time.Duration; +import java.util.List; +import com.kingsrook.qbits.assistant.AssistantQBitConfig; +import com.kingsrook.qbits.assistant.provider.ModelMessage; +import com.kingsrook.qbits.assistant.provider.ModelProviderException; +import com.kingsrook.qbits.assistant.provider.ModelRequest; +import com.kingsrook.qbits.assistant.provider.ProviderSettings; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Opt-in smoke test; never contacts an endpoint in the default build. + *******************************************************************************/ +@Tag("live") +class LiveOpenAiCompatibleTest +{ + /******************************************************************************* + ** All connection details come from environment references. + *******************************************************************************/ + @Test + void live() throws Exception + { + String url = System.getenv("ASSISTANT_LIVE_OPENAI_BASE_URL"); + String model = System.getenv("ASSISTANT_LIVE_OPENAI_MODEL"); + Assumptions.assumeTrue(url != null && !url.isBlank() && model != null && !model.isBlank(), "Live URL and model must be set"); + String kind = System.getenv("ASSISTANT_LIVE_OPENAI_KIND"); + String keyEnv = System.getenv("ASSISTANT_LIVE_OPENAI_KEY_ENV"); + ProviderSettings settings = new ProviderSettings().withKind(kind == null || kind.isBlank() ? "OLLAMA" : kind) + .withBaseUrl(url).withApiKey(keyEnv == null || keyEnv.isBlank() ? null : System.getenv(keyEnv)); + String caFile = System.getenv("ASSISTANT_LIVE_OPENAI_CA_FILE"); + if(caFile != null && !caFile.isBlank()) + { + settings.withTlsCaBundle("live").withSslContext(new AssistantQBitConfig().withTrustedCaBundleFile("live", Path.of(caFile)).getTrustedCaBundles().get("live")); + } + try + { + OpenAiCompatibleProviderType type = new OpenAiCompatibleProviderType(); + Assumptions.assumeTrue(Boolean.TRUE.equals(type.checkHealth(settings).healthy()), "Live health probe must pass"); + assertThat(type.discoverModels(settings)).isNotEmpty(); + var provider = type.create(settings); + StringBuilder text = new StringBuilder(); + provider.stream(new ModelRequest().withModel(model).withMaxTokens(128).withMessages(List.of(ModelMessage.user("Reply with OK."))), text::append).await(Duration.ofSeconds(120)); + assertThat(text).isNotEmpty(); + var response = provider.stream(new ModelRequest().withModel(model).withMaxTokens(256).withTools(List.of(OpenAiContractTest.tool())) + .withMessages(List.of(ModelMessage.user("Call lookup with q set to hello. Do not answer without calling the tool."))), t -> {}).await(Duration.ofSeconds(120)); + assertThat(response.getToolUses()).anyMatch(t -> "lookup".equals(t.getToolName())); + } + catch(ModelProviderException e) + { + Assumptions.assumeFalse(e.getKind() == ModelProviderException.Kind.NETWORK || e.getKind() == ModelProviderException.Kind.TIMEOUT, "Live endpoint unreachable (including TLS)"); + throw (e); + } + } +} diff --git a/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/MockOpenAiServer.java b/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/MockOpenAiServer.java new file mode 100644 index 0000000..f41b461 --- /dev/null +++ b/qbit-assistant-provider-openai/src/test/java/com/kingsrook/qbits/assistant/provider/openai/MockOpenAiServer.java @@ -0,0 +1,183 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.assistant.provider.openai; + + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import javax.net.ssl.SSLContext; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsServer; + + +/******************************************************************************* + ** Loopback-only scripted Chat Completions server with captured requests. + *******************************************************************************/ +class MockOpenAiServer implements AutoCloseable +{ + final List requests = new CopyOnWriteArrayList<>(); + private final LinkedBlockingQueue