diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java b/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java index 4ea1659..66107c4 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java @@ -62,7 +62,17 @@ public final class AuthentikTestStack private static final String SECRET_KEY = "contract-test-secret-key-0123456789abcdefghijklmnop"; public static final String SA_TOKEN_ID = "qqq-contract-test"; + //////////////////////////////////////////////////////////////////////////// + // Budget for anything Authentik's worker delivers asynchronously (emails, // + // notifications, events, blueprint applies). The waits return as soon as // + // the condition holds; the budget only matters on a busy shared CI // + // runner, where the worker queue can lag by minutes. // + //////////////////////////////////////////////////////////////////////////// + public static final Duration ASYNC_TIMEOUT = Duration.ofMinutes(3); + public static final Duration APPLY_TIMEOUT = Duration.ofMinutes(10); + private static AuthentikTestStack instance; + private static volatile AuthentikTestStack starting; private static RuntimeException startupFailure; private final Network network = Network.newNetwork(); @@ -119,7 +129,20 @@ public final class AuthentikTestStack .withNetworkAliases("postgresql") .withDatabaseName("authentik") .withUsername("authentik") - .withPassword("authentik-test"); + .withPassword("authentik-test") + ////////////////////////////////////////////////////////////////////////// + // jit=off: Authentik validates every policy binding in a blueprint by // + // loading PolicyBindingModel.select_subclasses(), a ~20-way LEFT JOIN // + // whose planner estimate (cost ~1e8 on a fresh database) is far above // + // the JIT thresholds. Postgres then spends seconds JIT-compiling a // + // query that runs in milliseconds: 7.6 s instead of 77 ms at 0.5 CPU. // + // Nested metaapplyblueprint entries multiply it (kof22-qqq-tokens -> // + // bootstrap -> oauth2-apps validates and applies oauth2-apps 8 times), // + // so on a busy CI runner one apply outran the worker's 10-minute task // + // limit, was killed and retried, and the stack never became ready. // + // fsync=off is the Testcontainers default this command replaces. // + ////////////////////////////////////////////////////////////////////////// + .withCommand("postgres", "-c", "fsync=off", "-c", "jit=off"); mailpit = new GenericContainer<>(DockerImageName.parse(mailpitImage)) .withNetwork(network) @@ -202,6 +225,7 @@ public final class AuthentikTestStack *******************************************************************************/ private void start() { + starting = this; postgres.start(); mailpit.start(); server.start(); @@ -255,35 +279,47 @@ public final class AuthentikTestStack TestApi admin = admin(); try { - long[] nextRetry = { System.currentTimeMillis() + 60_000 }; - waitFor(Duration.ofMinutes(8), "all three kof22 blueprints applied", () -> + ////////////////////////////////////////////////////////////////////////// + // On first boot the worker discovers ~40 built-in blueprints and ours // + // and applies them concurrently. Ours depend on objects the built-in // + // ones create (managed OAuth2 scope mappings, default flows), so on a // + // slow runner ours can validate first and end in "error" -- and // + // Authentik never re-applies a failed blueprint whose file is // + // unchanged. Concurrent applies can also deadlock in Postgres. So: let // + // the worker drain ALL blueprint work first, then re-apply ours one at // + // a time in dependency order, never enqueueing while an apply is still // + // pending (piling up applies only deepens the backlog and the lock // + // contention). With Postgres JIT off this takes ~1.5 minutes even at // + // 0.5 CPU; the 15-minute budget is headroom for a saturated runner. // + ////////////////////////////////////////////////////////////////////////// + waitFor(Duration.ofMinutes(15), "all three kof22 blueprints applied", () -> { - //////////////////////////////////////////////////////////////////// - // built-in blueprints may race our custom dependencies on first // - // boot, leaving one of ours in "error" or never applied // - // ("unknown"); re-apply it (throttled). // - //////////////////////////////////////////////////////////////////// - if(System.currentTimeMillis() >= nextRetry[0]) + try { - nextRetry[0] = System.currentTimeMillis() + 30_000; + if(blueprintWorkPending(admin)) + { + return (false); + } for(String name : List.of(INFRA_BLUEPRINT, BOOTSTRAP_BLUEPRINT, TOKEN_BLUEPRINT)) { - try + JSONObject current = blueprint(name); + if(current == null) { - JSONObject current = blueprint(name); - if(current != null && List.of("error", "unknown").contains(current.optString("status"))) - { - admin.post("/managed/blueprints/" + current.getString("pk") + "/apply/", new JSONObject()); - } + return (false); } - catch(Exception e) + if(!"successful".equals(current.optString("status"))) { - // Bootstrap authentication and built-in dependencies settle after HTTP readiness. + admin.post("/managed/blueprints/" + current.getString("pk") + "/apply/", new JSONObject()); + return (false); } } + return (admin.first("/core/users/?username=" + SERVICE_ACCOUNT) != null); + } + catch(Exception e) + { + // Bootstrap authentication and built-in dependencies settle after HTTP readiness. + return (false); } - return (isSuccessful(admin, INFRA_BLUEPRINT) && isSuccessful(admin, BOOTSTRAP_BLUEPRINT) && isSuccessful(admin, TOKEN_BLUEPRINT) - && admin.first("/core/users/?username=" + SERVICE_ACCOUNT) != null); }); } catch(IllegalStateException e) @@ -514,7 +550,7 @@ public final class AuthentikTestStack JSONObject before = blueprint(name); String lastApplied = before.optString("last_applied", ""); admin().post("/managed/blueprints/" + before.getString("pk") + "/apply/", new JSONObject()); - waitFor(Duration.ofMinutes(3), "blueprint " + name + " re-applied", () -> + waitFor(APPLY_TIMEOUT, "blueprint " + name + " re-applied", () -> { JSONObject now = blueprint(name); return (!now.optString("last_applied", "").equals(lastApplied) && "successful".equals(now.optString("status"))); @@ -523,6 +559,26 @@ public final class AuthentikTestStack + /******************************************************************************* + ** Whether the worker still has blueprint discovery or apply tasks that are + ** queued or in progress (unknown counts as pending). + *******************************************************************************/ + private static boolean blueprintWorkPending(TestApi admin) + { + for(String actor : List.of("authentik.blueprints.v1.tasks.blueprints_discovery", "authentik.blueprints.v1.tasks.apply_blueprint")) + { + TestApi.Response response = admin.get("/tasks/tasks/?page_size=1&actor_name=" + actor + + "&aggregated_status=queued&aggregated_status=consumed&aggregated_status=preprocess&aggregated_status=running&aggregated_status=postprocess"); + if(response.status() != 200 || response.json().getJSONObject("pagination").getInt("count") > 0) + { + return (true); + } + } + return (false); + } + + + /******************************************************************************* ** Create a content-based blueprint instance and wait for it to apply. *******************************************************************************/ @@ -579,7 +635,49 @@ public final class AuthentikTestStack break; } } + AuthentikTestStack stack = starting; + if(stack != null) + { + stack.printTaskDiagnostics(); + } throw (new IllegalStateException("Timed out waiting for " + what)); } + + + /******************************************************************************* + ** Print the Authentik worker queue (most waits depend on worker tasks: + ** blueprint applies, emails, notifications), so a timeout says whether + ** the worker was busy, stuck or failing, and on what. + *******************************************************************************/ + private void printTaskDiagnostics() + { + try + { + TestApi admin = admin(); + System.err.println("authentik tasks status: " + admin.get("/tasks/tasks/status/").body()); + for(String state : List.of("running", "consumed", "queued", "warning", "error")) + { + TestApi.Response tasks = admin.get("/tasks/tasks/?page_size=20&aggregated_status=" + state); + for(Object o : tasks.results()) + { + JSONObject task = (JSONObject) o; + StringBuilder logs = new StringBuilder(); + for(Object l : task.optJSONArray("logs") == null ? new org.json.JSONArray() : task.getJSONArray("logs")) + { + JSONObject log = (JSONObject) l; + logs.append(" | ").append(log.optString("log_level")).append(": ").append(log.optString("event")) + .append(" ").append(String.valueOf(log.optJSONObject("attributes")).substring(0, Math.min(400, String.valueOf(log.optJSONObject("attributes")).length()))); + } + System.err.println("authentik task [" + state + "] " + task.optString("actor_name") + " uid=" + task.optString("uid") + " mtime=" + task.optString("mtime") + + " eta=" + task.optString("eta") + " retries=" + task.optInt("retries") + logs); + } + } + } + catch(RuntimeException e) + { + System.err.println("authentik task diagnostics unavailable: " + e); + } + } + } diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java b/src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java index 1938bf2..90e1efa 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java @@ -82,6 +82,34 @@ class BlueprintReapplyIT /////////////////////////////////////////////////////////////////////// String identityAdmins = admin.groupPk("identity-admins"); String kof22Admins = admin.groupPk("kof22-admins"); + String roleUuid = admin.first("/rbac/roles/?name=" + SERVICE_ACCOUNT).getString("pk"); + try + { + assertEditsSurviveReapply(member, identityAdmins, kof22Admins, roleUuid); + } + finally + { + ////////////////////////////////////////////////////////////////// + // put the seed state back as the bootstrap created it, even if // + // an assertion or re-apply failed, so later contract tests on // + // the shared stack do not inherit this test's edits // + ////////////////////////////////////////////////////////////////// + admin.post("/core/groups/" + identityAdmins + "/remove_user/", new JSONObject().put("pk", member)); + admin.post("/core/groups/" + kof22Admins + "/remove_user/", new JSONObject().put("pk", member)); + admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes", new JSONObject())); + admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes", + new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod")))); + admin.post("/rbac/permissions/assigned_by_roles/" + roleUuid + "/assign/", new JSONObject().put("permissions", new JSONArray().put("authentik_core.view_user"))); + } + } + + + + /******************************************************************************* + ** Body of (c); the caller restores the seed state. + *******************************************************************************/ + private void assertEditsSurviveReapply(int member, String identityAdmins, String kof22Admins, String roleUuid) + { assertThat(admin.post("/core/groups/" + identityAdmins + "/add_user/", new JSONObject().put("pk", member)).status()).isEqualTo(204); assertThat(admin.post("/core/groups/" + kof22Admins + "/add_user/", new JSONObject().put("pk", member)).status()).isEqualTo(204); assertThat(admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes", @@ -89,7 +117,6 @@ class BlueprintReapplyIT assertThat(admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes", new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*").put("cms.publish")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod")))).status()).isEqualTo(200); - String roleUuid = admin.first("/rbac/roles/?name=" + SERVICE_ACCOUNT).getString("pk"); try { assertThat(admin.patch("/rbac/permissions/assigned_by_roles/" + roleUuid + "/unassign/", new JSONObject().put("permissions", new JSONArray().put("authentik_core.view_user"))).status()).isEqualTo(204); @@ -113,15 +140,6 @@ class BlueprintReapplyIT JSONObject serviceAccount = admin.first("/core/users/?username=" + SERVICE_ACCOUNT + "&include_roles=true"); assertThat(BootstrapBlueprintIT.names(serviceAccount.getJSONArray("roles_obj"))).containsExactly(SERVICE_ACCOUNT); assertThat(admin.first("/core/users/?username=" + BREAK_GLASS).getBoolean("is_superuser")).isTrue(); - - /////////////////////////////////////////////////////////// - // put the seed groups back as the bootstrap created them // - /////////////////////////////////////////////////////////// - admin.post("/core/groups/" + identityAdmins + "/remove_user/", new JSONObject().put("pk", member)); - admin.post("/core/groups/" + kof22Admins + "/remove_user/", new JSONObject().put("pk", member)); - admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes", new JSONObject())); - admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes", - new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod")))); } diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java b/src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java index 51be1d8..744d086 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java @@ -297,7 +297,7 @@ class InvitationEnrollmentContractIT var result = AuthentikAdmin.forBackend(config.getApiBackendName()).execute(AdminRequest.of(AdminAction.INVITE_USER, "", "Native onboarding contract") .with("email", owner + "@example.test").with("name", "Invitation Contract").with("groupIds", group)); assertThat(result.ok()).as(result.message()).isTrue(); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the invitation email", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the invitation email", () -> { try(var client = HttpClient.newHttpClient()) { diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java b/src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java index 08bd6be..1fa1239 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java @@ -21,7 +21,6 @@ import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.time.Duration; import java.util.EnumSet; import java.util.HashSet; import java.util.List; @@ -228,9 +227,9 @@ class QbitWriteContractIT assertThat(user.getJSONObject("attributes").getString("qqq.io/created-by")).isEqualTo(OPERATOR_EMAIL); assertThat(user.getJSONArray("groups_obj").getJSONObject(0).getString("name")).isEqualTo("wr-readers"); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the recovery email in Mailpit", () -> mailpitHas("wr.created@example.test")); + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the recovery email in Mailpit", () -> mailpitHas("wr.created@example.test")); - AuthentikTestStack.waitFor(Duration.ofSeconds(30), "the audit event naming the operator", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the audit event naming the operator", () -> { for(Object event : admin.get("/events/events/?search=" + TestApi.encode(OPERATOR_EMAIL) + "&ordering=-created").results()) { @@ -437,7 +436,7 @@ class QbitWriteContractIT AuthentikAdmin.AdminResult reset = run(AdminRequest.of(AdminAction.SEND_RECOVERY_EMAIL, String.valueOf(pk), "contract test")); assertThat(reset.ok()).as(reset.message()).isTrue(); assertThat(reset.message()).doesNotContain("http"); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the reset email in Mailpit", () -> mailpitHas("wr.target@example.test")); + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the reset email in Mailpit", () -> mailpitHas("wr.target@example.test")); } diff --git a/src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java b/src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java index 89377ee..480c2f9 100644 --- a/src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java +++ b/src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java @@ -424,7 +424,7 @@ class TokenContractIT tokens.add(ordinaryToken); String ordinaryEvent = setKeyEvent(ordinaryToken); String protectedEvent = setKeyEvent(protectedToken); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "protected-token local notification", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "protected-token local notification", () -> notifications(protectedOwner, protectedEvent).length() > 0); Instant deadline = Instant.now().plusSeconds(60); do @@ -489,7 +489,7 @@ class TokenContractIT users.add(servicePk); tokens.add("service-account-" + created.getString("username") + "-password"); String[] eventPk = { null }; - AuthentikTestStack.waitFor(Duration.ofSeconds(30), "service-account model_created event", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "service-account model_created event", () -> { JSONArray events = admin.getOk("/events/events/?action=model_created&ordering=-created&page_size=50&username=" + TestApi.encode(AuthentikTestStack.SERVICE_ACCOUNT)).getJSONArray("results"); @@ -504,7 +504,7 @@ class TokenContractIT } return (false); }); - AuthentikTestStack.waitFor(Duration.ofSeconds(60), "service-account creation notification", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "service-account creation notification", () -> notifications(member, eventPk[0]).length() > 0); } @@ -778,7 +778,7 @@ class TokenContractIT String[] found = { null }; try { - AuthentikTestStack.waitFor(Duration.ofSeconds(30), "token model_updated event", () -> + AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "token model_updated event", () -> { ////////////////////////////////////////////////////////////////////// // set_key logs model_to_dict(token): identifier, intent, user (key //