From 2f6a83db2730f5efa95f0b23c88d4497155bc6d0 Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 6 Nov 2026 10:38:25 -0600 Subject: [PATCH 1/3] test(authentik): print the worker task queue on contract-test timeouts; restore seed state in BlueprintReapplyIT Most contract-test waits depend on Authentik worker tasks (blueprint applies, recovery/invitation emails, notification rules). When a wait times out, print the global task status and the running, queued, warning and error tasks with their logs, so CI shows whether the worker was stuck, backlogged or failing. BlueprintReapplyIT now restores the seed groups and the service-account role's view_user permission in a finally block. A failed re-apply used to leave those edits behind and cascade into unrelated failures (QbitReadContractIT.groups saw "authentik.*, cms.publish"; TokenContractIT got 403 on /core/users/). Co-Authored-By: Claude Opus 5.5 --- .../contract/AuthentikTestStack.java | 44 +++++++++++++++++++ .../contract/BlueprintReapplyIT.java | 38 +++++++++++----- 2 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java b/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java index 4ea1659..0150843 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java @@ -63,6 +63,7 @@ public final class AuthentikTestStack public static final String SA_TOKEN_ID = "qqq-contract-test"; private static AuthentikTestStack instance; + private static volatile AuthentikTestStack starting; private static RuntimeException startupFailure; private final Network network = Network.newNetwork(); @@ -202,6 +203,7 @@ public final class AuthentikTestStack *******************************************************************************/ private void start() { + starting = this; postgres.start(); mailpit.start(); server.start(); @@ -579,7 +581,49 @@ public final class AuthentikTestStack break; } } + AuthentikTestStack stack = starting; + if(stack != null) + { + stack.printTaskDiagnostics(); + } throw (new IllegalStateException("Timed out waiting for " + what)); } + + + /******************************************************************************* + ** Print the Authentik worker queue (most waits depend on worker tasks: + ** blueprint applies, emails, notifications), so a timeout says whether + ** the worker was busy, stuck or failing, and on what. + *******************************************************************************/ + private void printTaskDiagnostics() + { + try + { + TestApi admin = admin(); + System.err.println("authentik tasks status: " + admin.get("/tasks/tasks/status/").body()); + for(String state : List.of("running", "consumed", "queued", "warning", "error")) + { + TestApi.Response tasks = admin.get("/tasks/tasks/?page_size=20&aggregated_status=" + state); + for(Object o : tasks.results()) + { + JSONObject task = (JSONObject) o; + StringBuilder logs = new StringBuilder(); + for(Object l : task.optJSONArray("logs") == null ? new org.json.JSONArray() : task.getJSONArray("logs")) + { + JSONObject log = (JSONObject) l; + logs.append(" | ").append(log.optString("log_level")).append(": ").append(log.optString("event")) + .append(" ").append(String.valueOf(log.optJSONObject("attributes")).substring(0, Math.min(400, String.valueOf(log.optJSONObject("attributes")).length()))); + } + System.err.println("authentik task [" + state + "] " + task.optString("actor_name") + " uid=" + task.optString("uid") + " mtime=" + task.optString("mtime") + + " eta=" + task.optString("eta") + " retries=" + task.optInt("retries") + logs); + } + } + } + catch(RuntimeException e) + { + System.err.println("authentik task diagnostics unavailable: " + e); + } + } + } diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java b/src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java index 1938bf2..90e1efa 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java @@ -82,6 +82,34 @@ class BlueprintReapplyIT /////////////////////////////////////////////////////////////////////// String identityAdmins = admin.groupPk("identity-admins"); String kof22Admins = admin.groupPk("kof22-admins"); + String roleUuid = admin.first("/rbac/roles/?name=" + SERVICE_ACCOUNT).getString("pk"); + try + { + assertEditsSurviveReapply(member, identityAdmins, kof22Admins, roleUuid); + } + finally + { + ////////////////////////////////////////////////////////////////// + // put the seed state back as the bootstrap created it, even if // + // an assertion or re-apply failed, so later contract tests on // + // the shared stack do not inherit this test's edits // + ////////////////////////////////////////////////////////////////// + admin.post("/core/groups/" + identityAdmins + "/remove_user/", new JSONObject().put("pk", member)); + admin.post("/core/groups/" + kof22Admins + "/remove_user/", new JSONObject().put("pk", member)); + admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes", new JSONObject())); + admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes", + new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod")))); + admin.post("/rbac/permissions/assigned_by_roles/" + roleUuid + "/assign/", new JSONObject().put("permissions", new JSONArray().put("authentik_core.view_user"))); + } + } + + + + /******************************************************************************* + ** Body of (c); the caller restores the seed state. + *******************************************************************************/ + private void assertEditsSurviveReapply(int member, String identityAdmins, String kof22Admins, String roleUuid) + { assertThat(admin.post("/core/groups/" + identityAdmins + "/add_user/", new JSONObject().put("pk", member)).status()).isEqualTo(204); assertThat(admin.post("/core/groups/" + kof22Admins + "/add_user/", new JSONObject().put("pk", member)).status()).isEqualTo(204); assertThat(admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes", @@ -89,7 +117,6 @@ class BlueprintReapplyIT assertThat(admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes", new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*").put("cms.publish")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod")))).status()).isEqualTo(200); - String roleUuid = admin.first("/rbac/roles/?name=" + SERVICE_ACCOUNT).getString("pk"); try { assertThat(admin.patch("/rbac/permissions/assigned_by_roles/" + roleUuid + "/unassign/", new JSONObject().put("permissions", new JSONArray().put("authentik_core.view_user"))).status()).isEqualTo(204); @@ -113,15 +140,6 @@ class BlueprintReapplyIT JSONObject serviceAccount = admin.first("/core/users/?username=" + SERVICE_ACCOUNT + "&include_roles=true"); assertThat(BootstrapBlueprintIT.names(serviceAccount.getJSONArray("roles_obj"))).containsExactly(SERVICE_ACCOUNT); assertThat(admin.first("/core/users/?username=" + BREAK_GLASS).getBoolean("is_superuser")).isTrue(); - - /////////////////////////////////////////////////////////// - // put the seed groups back as the bootstrap created them // - /////////////////////////////////////////////////////////// - admin.post("/core/groups/" + identityAdmins + "/remove_user/", new JSONObject().put("pk", member)); - admin.post("/core/groups/" + kof22Admins + "/remove_user/", new JSONObject().put("pk", member)); - admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes", new JSONObject())); - admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes", - new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod")))); } -- 2.54.0 From 7adc25bf7e453238be44b0cdc943ccf480adabea Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 6 Nov 2026 12:00:01 -0600 Subject: [PATCH 2/3] fix(test): drain Authentik's first-boot blueprint work before re-applying ours Root cause of the flaky contract stack (diagnostics from run 6277): on first boot the worker (2 threads) applies ~40 built-in blueprints plus the three kof22 fixtures concurrently. On a slow shared runner ours can validate before the built-in objects they reference exist (managed OAuth2 scope mappings, default flows) and end in "error", which Authentik never retries for an unchanged file; concurrent applies also deadlocked in Postgres. The earlier throttled re-apply made it worse: it enqueued three applies every 30s regardless of pending work (69 queued apply_blueprint tasks, applies running for 6+ minutes), and the backlog then starved the tests' own worker tasks (blueprint applies, recovery/invitation emails, notification rules) and held locks that timed out plain API writes. Now startup waits until no blueprint discovery/apply task is queued or in progress, then re-applies the first non-successful kof22 blueprint in dependency order, one at a time, never enqueueing while an apply is pending. The startup budget is 15 minutes (with a comment) because a cold shared runner can need well over 8 minutes for the built-in storm. Co-Authored-By: Claude Opus 5.5 --- .../contract/AuthentikTestStack.java | 70 ++++++++++++++----- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java b/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java index 0150843..acf4674 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java @@ -257,35 +257,47 @@ public final class AuthentikTestStack TestApi admin = admin(); try { - long[] nextRetry = { System.currentTimeMillis() + 60_000 }; - waitFor(Duration.ofMinutes(8), "all three kof22 blueprints applied", () -> + ////////////////////////////////////////////////////////////////////////// + // On first boot the worker discovers ~40 built-in blueprints and ours // + // and applies them concurrently. Ours depend on objects the built-in // + // ones create (managed OAuth2 scope mappings, default flows), so on a // + // slow runner ours can validate first and end in "error" -- and // + // Authentik never re-applies a failed blueprint whose file is // + // unchanged. Concurrent applies can also deadlock in Postgres. So: let // + // the worker drain ALL blueprint work first, then re-apply ours one at // + // a time in dependency order, never enqueueing while an apply is still // + // pending (piling up applies only deepens the backlog and the lock // + // contention). A cold, shared CI runner can need well over 8 minutes // + // for the built-in storm, hence the 15-minute budget. // + ////////////////////////////////////////////////////////////////////////// + waitFor(Duration.ofMinutes(15), "all three kof22 blueprints applied", () -> { - //////////////////////////////////////////////////////////////////// - // built-in blueprints may race our custom dependencies on first // - // boot, leaving one of ours in "error" or never applied // - // ("unknown"); re-apply it (throttled). // - //////////////////////////////////////////////////////////////////// - if(System.currentTimeMillis() >= nextRetry[0]) + try { - nextRetry[0] = System.currentTimeMillis() + 30_000; + if(blueprintWorkPending(admin)) + { + return (false); + } for(String name : List.of(INFRA_BLUEPRINT, BOOTSTRAP_BLUEPRINT, TOKEN_BLUEPRINT)) { - try + JSONObject current = blueprint(name); + if(current == null) { - JSONObject current = blueprint(name); - if(current != null && List.of("error", "unknown").contains(current.optString("status"))) - { - admin.post("/managed/blueprints/" + current.getString("pk") + "/apply/", new JSONObject()); - } + return (false); } - catch(Exception e) + if(!"successful".equals(current.optString("status"))) { - // Bootstrap authentication and built-in dependencies settle after HTTP readiness. + admin.post("/managed/blueprints/" + current.getString("pk") + "/apply/", new JSONObject()); + return (false); } } + return (admin.first("/core/users/?username=" + SERVICE_ACCOUNT) != null); + } + catch(Exception e) + { + // Bootstrap authentication and built-in dependencies settle after HTTP readiness. + return (false); } - return (isSuccessful(admin, INFRA_BLUEPRINT) && isSuccessful(admin, BOOTSTRAP_BLUEPRINT) && isSuccessful(admin, TOKEN_BLUEPRINT) - && admin.first("/core/users/?username=" + SERVICE_ACCOUNT) != null); }); } catch(IllegalStateException e) @@ -525,6 +537,26 @@ public final class AuthentikTestStack + /******************************************************************************* + ** Whether the worker still has blueprint discovery or apply tasks that are + ** queued or in progress (unknown counts as pending). + *******************************************************************************/ + private static boolean blueprintWorkPending(TestApi admin) + { + for(String actor : List.of("authentik.blueprints.v1.tasks.blueprints_discovery", "authentik.blueprints.v1.tasks.apply_blueprint")) + { + TestApi.Response response = admin.get("/tasks/tasks/?page_size=1&actor_name=" + actor + + "&aggregated_status=queued&aggregated_status=consumed&aggregated_status=preprocess&aggregated_status=running&aggregated_status=postprocess"); + if(response.status() != 200 || response.json().getJSONObject("pagination").getInt("count") > 0) + { + return (true); + } + } + return (false); + } + + + /******************************************************************************* ** Create a content-based blueprint instance and wait for it to apply. *******************************************************************************/ -- 2.54.0 From d3f334cd420b344b916e81b2bd59232ff2fd77be Mon Sep 17 00:00:00 2001 From: James Maes Date: Fri, 6 Nov 2026 17:36:47 -0600 Subject: [PATCH 3/3] fix(test): turn off Postgres JIT in the contract stack Root cause of the slow/failed Authentik contract stack on busy CI runners: Authentik validates every policy binding in a blueprint by loading PolicyBindingModel.select_subclasses(), a ~20-way LEFT JOIN whose planner estimate on a fresh database (cost ~1e8) is far above Postgres' JIT thresholds. Postgres JIT-compiled it on every call: 7.6 s instead of 77 ms at 0.5 CPU, with the Postgres container pegged and the worker idle. The fixtures' nested metaapplyblueprint chain (tokens -> bootstrap -> oauth2-apps) runs oauth2-apps 8 times per tokens apply, so one apply outran the worker's 10-minute task limit, was killed and retried, and the stack never became ready (runs 5754, 6277, 6363). With jit=off the full contract suite passes locally with every container capped at 0.5 CPU; blueprint startup drops from >14 minutes to ~1.5 minutes and BlueprintReapplyIT from 228 s to 34 s. Also give waits on worker-delivered results (emails, notifications, events) a shared 3-minute budget and blueprint re-applies 10 minutes; the waits still return as soon as the condition holds. Co-Authored-By: Claude Opus 5.5 --- .../contract/AuthentikTestStack.java | 30 ++++++++++++++++--- .../InvitationEnrollmentContractIT.java | 2 +- .../contract/QbitWriteContractIT.java | 7 ++--- .../authentik/contract/TokenContractIT.java | 8 ++--- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java b/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java index acf4674..66107c4 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java @@ -62,6 +62,15 @@ public final class AuthentikTestStack private static final String SECRET_KEY = "contract-test-secret-key-0123456789abcdefghijklmnop"; public static final String SA_TOKEN_ID = "qqq-contract-test"; + //////////////////////////////////////////////////////////////////////////// + // Budget for anything Authentik's worker delivers asynchronously (emails, // + // notifications, events, blueprint applies). The waits return as soon as // + // the condition holds; the budget only matters on a busy shared CI // + // runner, where the worker queue can lag by minutes. // + //////////////////////////////////////////////////////////////////////////// + public static final Duration ASYNC_TIMEOUT = Duration.ofMinutes(3); + public static final Duration APPLY_TIMEOUT = Duration.ofMinutes(10); + private static AuthentikTestStack instance; private static volatile AuthentikTestStack starting; private static RuntimeException startupFailure; @@ -120,7 +129,20 @@ public final class AuthentikTestStack .withNetworkAliases("postgresql") .withDatabaseName("authentik") .withUsername("authentik") - .withPassword("authentik-test"); + .withPassword("authentik-test") + ////////////////////////////////////////////////////////////////////////// + // jit=off: Authentik validates every policy binding in a blueprint by // + // loading PolicyBindingModel.select_subclasses(), a ~20-way LEFT JOIN // + // whose planner estimate (cost ~1e8 on a fresh database) is far above // + // the JIT thresholds. Postgres then spends seconds JIT-compiling a // + // query that runs in milliseconds: 7.6 s instead of 77 ms at 0.5 CPU. // + // Nested metaapplyblueprint entries multiply it (kof22-qqq-tokens -> // + // bootstrap -> oauth2-apps validates and applies oauth2-apps 8 times), // + // so on a busy CI runner one apply outran the worker's 10-minute task // + // limit, was killed and retried, and the stack never became ready. // + // fsync=off is the Testcontainers default this command replaces. // + ////////////////////////////////////////////////////////////////////////// + .withCommand("postgres", "-c", "fsync=off", "-c", "jit=off"); mailpit = new GenericContainer<>(DockerImageName.parse(mailpitImage)) .withNetwork(network) @@ -267,8 +289,8 @@ public final class AuthentikTestStack // the worker drain ALL blueprint work first, then re-apply ours one at // // a time in dependency order, never enqueueing while an apply is still // // pending (piling up applies only deepens the backlog and the lock // - // contention). A cold, shared CI runner can need well over 8 minutes // - // for the built-in storm, hence the 15-minute budget. // + // contention). With Postgres JIT off this takes ~1.5 minutes even at // + // 0.5 CPU; the 15-minute budget is headroom for a saturated runner. // ////////////////////////////////////////////////////////////////////////// waitFor(Duration.ofMinutes(15), "all three kof22 blueprints applied", () -> { @@ -528,7 +550,7 @@ public final class AuthentikTestStack JSONObject before = blueprint(name); String lastApplied = before.optString("last_applied", ""); admin().post("/managed/blueprints/" + before.getString("pk") + "/apply/", new JSONObject()); - waitFor(Duration.ofMinutes(3), "blueprint " + name + " re-applied", () -> + waitFor(APPLY_TIMEOUT, "blueprint " + name + " re-applied", () -> { JSONObject now = blueprint(name); return (!now.optString("last_applied", "").equals(lastApplied) && "successful".equals(now.optString("status"))); diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java b/src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java index 51be1d8..744d086 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java @@ -297,7 +297,7 @@ class InvitationEnrollmentContractIT var result = AuthentikAdmin.forBackend(config.getApiBackendName()).execute(AdminRequest.of(AdminAction.INVITE_USER, "", "Native onboarding contract") .with("email", owner + "@example.test").with("name", "Invitation Contract").with("groupIds", group)); assertThat(result.ok()).as(result.message()).isTrue(); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the invitation email", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the invitation email", () -> { try(var client = HttpClient.newHttpClient()) { diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java b/src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java index 08bd6be..1fa1239 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java @@ -21,7 +21,6 @@ import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.time.Duration; import java.util.EnumSet; import java.util.HashSet; import java.util.List; @@ -228,9 +227,9 @@ class QbitWriteContractIT assertThat(user.getJSONObject("attributes").getString("qqq.io/created-by")).isEqualTo(OPERATOR_EMAIL); assertThat(user.getJSONArray("groups_obj").getJSONObject(0).getString("name")).isEqualTo("wr-readers"); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the recovery email in Mailpit", () -> mailpitHas("wr.created@example.test")); + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the recovery email in Mailpit", () -> mailpitHas("wr.created@example.test")); - AuthentikTestStack.waitFor(Duration.ofSeconds(30), "the audit event naming the operator", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the audit event naming the operator", () -> { for(Object event : admin.get("/events/events/?search=" + TestApi.encode(OPERATOR_EMAIL) + "&ordering=-created").results()) { @@ -437,7 +436,7 @@ class QbitWriteContractIT AuthentikAdmin.AdminResult reset = run(AdminRequest.of(AdminAction.SEND_RECOVERY_EMAIL, String.valueOf(pk), "contract test")); assertThat(reset.ok()).as(reset.message()).isTrue(); assertThat(reset.message()).doesNotContain("http"); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the reset email in Mailpit", () -> mailpitHas("wr.target@example.test")); + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the reset email in Mailpit", () -> mailpitHas("wr.target@example.test")); } diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java b/src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java index 89377ee..480c2f9 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java @@ -424,7 +424,7 @@ class TokenContractIT tokens.add(ordinaryToken); String ordinaryEvent = setKeyEvent(ordinaryToken); String protectedEvent = setKeyEvent(protectedToken); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "protected-token local notification", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "protected-token local notification", () -> notifications(protectedOwner, protectedEvent).length() > 0); Instant deadline = Instant.now().plusSeconds(60); do @@ -489,7 +489,7 @@ class TokenContractIT users.add(servicePk); tokens.add("service-account-" + created.getString("username") + "-password"); String[] eventPk = { null }; - AuthentikTestStack.waitFor(Duration.ofSeconds(30), "service-account model_created event", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "service-account model_created event", () -> { JSONArray events = admin.getOk("/events/events/?action=model_created&ordering=-created&page_size=50&username=" + TestApi.encode(AuthentikTestStack.SERVICE_ACCOUNT)).getJSONArray("results"); @@ -504,7 +504,7 @@ class TokenContractIT } return (false); }); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "service-account creation notification", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "service-account creation notification", () -> notifications(member, eventPk[0]).length() > 0); } @@ -778,7 +778,7 @@ class TokenContractIT String[] found = { null }; try { - AuthentikTestStack.waitFor(Duration.ofSeconds(30), "token model_updated event", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "token model_updated event", () -> { ////////////////////////////////////////////////////////////////////// // set_key logs model_to_dict(token): identifier, intent, user (key // -- 2.54.0