src/test/java/com/kingsrook/qbits/authentik/contract/AuthentikTestStack.java

@@ -62,7 +62,17 @@ public final class AuthentikTestStack
    private static final String SECRET_KEY        = "contract-test-secret-key-0123456789abcdefghijklmnop";
    public static final String SA_TOKEN_ID       = "qqq-contract-test";
 
+   ////////////////////////////////////////////////////////////////////////////
+   // Budget for anything Authentik's worker delivers asynchronously (emails, //
+   // notifications, events, blueprint applies). The waits return as soon as //
+   // the condition holds; the budget only matters on a busy shared CI       //
+   // runner, where the worker queue can lag by minutes.                     //
+   ////////////////////////////////////////////////////////////////////////////
+   public static final Duration ASYNC_TIMEOUT       = Duration.ofMinutes(3);
+   public static final Duration APPLY_TIMEOUT       = Duration.ofMinutes(10);
+
    private static AuthentikTestStack instance;
+   private static volatile AuthentikTestStack starting;
    private static RuntimeException startupFailure;
 
    private final Network                         network = Network.newNetwork();
@@ -119,7 +129,20 @@ public final class AuthentikTestStack
          .withNetworkAliases("postgresql")
          .withDatabaseName("authentik")
          .withUsername("authentik")
-         .withPassword("authentik-test");
+         .withPassword("authentik-test")
+         //////////////////////////////////////////////////////////////////////////
+         // jit=off: Authentik validates every policy binding in a blueprint by   //
+         // loading PolicyBindingModel.select_subclasses(), a ~20-way LEFT JOIN   //
+         // whose planner estimate (cost ~1e8 on a fresh database) is far above   //
+         // the JIT thresholds. Postgres then spends seconds JIT-compiling a      //
+         // query that runs in milliseconds: 7.6 s instead of 77 ms at 0.5 CPU.   //
+         // Nested metaapplyblueprint entries multiply it (kof22-qqq-tokens ->    //
+         // bootstrap -> oauth2-apps validates and applies oauth2-apps 8 times),  //
+         // so on a busy CI runner one apply outran the worker's 10-minute task   //
+         // limit, was killed and retried, and the stack never became ready.      //
+         // fsync=off is the Testcontainers default this command replaces.        //
+         //////////////////////////////////////////////////////////////////////////
+         .withCommand("postgres", "-c", "fsync=off", "-c", "jit=off");
 
       mailpit = new GenericContainer<>(DockerImageName.parse(mailpitImage))
          .withNetwork(network)
@@ -202,6 +225,7 @@ public final class AuthentikTestStack
     *******************************************************************************/
    private void start()
    {
+      starting = this;
       postgres.start();
       mailpit.start();
       server.start();
@@ -255,35 +279,47 @@ public final class AuthentikTestStack
       TestApi admin = admin();
       try
       {
-         long[] nextRetry = { System.currentTimeMillis() + 60_000 };
-         waitFor(Duration.ofMinutes(8), "all three kof22 blueprints applied", () ->
+         //////////////////////////////////////////////////////////////////////////
+         // On first boot the worker discovers ~40 built-in blueprints and ours   //
+         // and applies them concurrently. Ours depend on objects the built-in    //
+         // ones create (managed OAuth2 scope mappings, default flows), so on a   //
+         // slow runner ours can validate first and end in "error" -- and         //
+         // Authentik never re-applies a failed blueprint whose file is           //
+         // unchanged. Concurrent applies can also deadlock in Postgres. So: let  //
+         // the worker drain ALL blueprint work first, then re-apply ours one at  //
+         // a time in dependency order, never enqueueing while an apply is still  //
+         // pending (piling up applies only deepens the backlog and the lock      //
+         // contention). With Postgres JIT off this takes ~1.5 minutes even at   //
+         // 0.5 CPU; the 15-minute budget is headroom for a saturated runner.    //
+         //////////////////////////////////////////////////////////////////////////
+         waitFor(Duration.ofMinutes(15), "all three kof22 blueprints applied", () ->
          {
-            ////////////////////////////////////////////////////////////////////
-            // built-in blueprints may race our custom dependencies on first  //
-            // boot, leaving one of ours in "error" or never applied          //
-            // ("unknown"); re-apply it (throttled).                          //
-            ////////////////////////////////////////////////////////////////////
-            if(System.currentTimeMillis() >= nextRetry[0])
+            try
             {
-               nextRetry[0] = System.currentTimeMillis() + 30_000;
+               if(blueprintWorkPending(admin))
+               {
+                  return (false);
+               }
                for(String name : List.of(INFRA_BLUEPRINT, BOOTSTRAP_BLUEPRINT, TOKEN_BLUEPRINT))
                {
-                  try
+                  JSONObject current = blueprint(name);
+                  if(current == null)
                   {
-                     JSONObject current = blueprint(name);
-                     if(current != null && List.of("error", "unknown").contains(current.optString("status")))
-                     {
-                        admin.post("/managed/blueprints/" + current.getString("pk") + "/apply/", new JSONObject());
-                     }
+                     return (false);
                   }
-                  catch(Exception e)
+                  if(!"successful".equals(current.optString("status")))
                   {
-                     // Bootstrap authentication and built-in dependencies settle after HTTP readiness.
+                     admin.post("/managed/blueprints/" + current.getString("pk") + "/apply/", new JSONObject());
+                     return (false);
                   }
                }
+               return (admin.first("/core/users/?username=" + SERVICE_ACCOUNT) != null);
+            }
+            catch(Exception e)
+            {
+               // Bootstrap authentication and built-in dependencies settle after HTTP readiness.
+               return (false);
             }
-            return (isSuccessful(admin, INFRA_BLUEPRINT) && isSuccessful(admin, BOOTSTRAP_BLUEPRINT) && isSuccessful(admin, TOKEN_BLUEPRINT)
-               && admin.first("/core/users/?username=" + SERVICE_ACCOUNT) != null);
          });
       }
       catch(IllegalStateException e)
@@ -514,7 +550,7 @@ public final class AuthentikTestStack
       JSONObject before      = blueprint(name);
       String     lastApplied = before.optString("last_applied", "");
       admin().post("/managed/blueprints/" + before.getString("pk") + "/apply/", new JSONObject());
-      waitFor(Duration.ofMinutes(3), "blueprint " + name + " re-applied", () ->
+      waitFor(APPLY_TIMEOUT, "blueprint " + name + " re-applied", () ->
       {
          JSONObject now = blueprint(name);
          return (!now.optString("last_applied", "").equals(lastApplied) && "successful".equals(now.optString("status")));
@@ -523,6 +559,26 @@ public final class AuthentikTestStack
 
 
 
+   /*******************************************************************************
+    ** Whether the worker still has blueprint discovery or apply tasks that are
+    ** queued or in progress (unknown counts as pending).
+    *******************************************************************************/
+   private static boolean blueprintWorkPending(TestApi admin)
+   {
+      for(String actor : List.of("authentik.blueprints.v1.tasks.blueprints_discovery", "authentik.blueprints.v1.tasks.apply_blueprint"))
+      {
+         TestApi.Response response = admin.get("/tasks/tasks/?page_size=1&actor_name=" + actor
+            + "&aggregated_status=queued&aggregated_status=consumed&aggregated_status=preprocess&aggregated_status=running&aggregated_status=postprocess");
+         if(response.status() != 200 || response.json().getJSONObject("pagination").getInt("count") > 0)
+         {
+            return (true);
+         }
+      }
+      return (false);
+   }
+
+
+
    /*******************************************************************************
     ** Create a content-based blueprint instance and wait for it to apply.
     *******************************************************************************/
@@ -579,7 +635,49 @@ public final class AuthentikTestStack
             break;
          }
       }
+      AuthentikTestStack stack = starting;
+      if(stack != null)
+      {
+         stack.printTaskDiagnostics();
+      }
       throw (new IllegalStateException("Timed out waiting for " + what));
    }
 
+
+
+   /*******************************************************************************
+    ** Print the Authentik worker queue (most waits depend on worker tasks:
+    ** blueprint applies, emails, notifications), so a timeout says whether
+    ** the worker was busy, stuck or failing, and on what.
+    *******************************************************************************/
+   private void printTaskDiagnostics()
+   {
+      try
+      {
+         TestApi admin = admin();
+         System.err.println("authentik tasks status: " + admin.get("/tasks/tasks/status/").body());
+         for(String state : List.of("running", "consumed", "queued", "warning", "error"))
+         {
+            TestApi.Response tasks = admin.get("/tasks/tasks/?page_size=20&aggregated_status=" + state);
+            for(Object o : tasks.results())
+            {
+               JSONObject task = (JSONObject) o;
+               StringBuilder logs = new StringBuilder();
+               for(Object l : task.optJSONArray("logs") == null ? new org.json.JSONArray() : task.getJSONArray("logs"))
+               {
+                  JSONObject log = (JSONObject) l;
+                  logs.append(" | ").append(log.optString("log_level")).append(": ").append(log.optString("event"))
+                     .append(" ").append(String.valueOf(log.optJSONObject("attributes")).substring(0, Math.min(400, String.valueOf(log.optJSONObject("attributes")).length())));
+               }
+               System.err.println("authentik task [" + state + "] " + task.optString("actor_name") + " uid=" + task.optString("uid") + " mtime=" + task.optString("mtime")
+                  + " eta=" + task.optString("eta") + " retries=" + task.optInt("retries") + logs);
+            }
+         }
+      }
+      catch(RuntimeException e)
+      {
+         System.err.println("authentik task diagnostics unavailable: " + e);
+      }
+   }
+
 }

src/test/java/com/kingsrook/qbits/authentik/contract/BlueprintReapplyIT.java

@@ -82,6 +82,34 @@ class BlueprintReapplyIT
       ///////////////////////////////////////////////////////////////////////
       String identityAdmins = admin.groupPk("identity-admins");
       String kof22Admins    = admin.groupPk("kof22-admins");
+      String roleUuid       = admin.first("/rbac/roles/?name=" + SERVICE_ACCOUNT).getString("pk");
+      try
+      {
+         assertEditsSurviveReapply(member, identityAdmins, kof22Admins, roleUuid);
+      }
+      finally
+      {
+         //////////////////////////////////////////////////////////////////
+         // put the seed state back as the bootstrap created it, even if  //
+         // an assertion or re-apply failed, so later contract tests on   //
+         // the shared stack do not inherit this test's edits             //
+         //////////////////////////////////////////////////////////////////
+         admin.post("/core/groups/" + identityAdmins + "/remove_user/", new JSONObject().put("pk", member));
+         admin.post("/core/groups/" + kof22Admins + "/remove_user/", new JSONObject().put("pk", member));
+         admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes", new JSONObject()));
+         admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes",
+            new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod"))));
+         admin.post("/rbac/permissions/assigned_by_roles/" + roleUuid + "/assign/", new JSONObject().put("permissions", new JSONArray().put("authentik_core.view_user")));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Body of (c); the caller restores the seed state.
+    *******************************************************************************/
+   private void assertEditsSurviveReapply(int member, String identityAdmins, String kof22Admins, String roleUuid)
+   {
       assertThat(admin.post("/core/groups/" + identityAdmins + "/add_user/", new JSONObject().put("pk", member)).status()).isEqualTo(204);
       assertThat(admin.post("/core/groups/" + kof22Admins + "/add_user/", new JSONObject().put("pk", member)).status()).isEqualTo(204);
       assertThat(admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes",
@@ -89,7 +117,6 @@ class BlueprintReapplyIT
       assertThat(admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes",
          new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*").put("cms.publish")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod")))).status()).isEqualTo(200);
 
-      String roleUuid = admin.first("/rbac/roles/?name=" + SERVICE_ACCOUNT).getString("pk");
       try
       {
          assertThat(admin.patch("/rbac/permissions/assigned_by_roles/" + roleUuid + "/unassign/", new JSONObject().put("permissions", new JSONArray().put("authentik_core.view_user"))).status()).isEqualTo(204);
@@ -113,15 +140,6 @@ class BlueprintReapplyIT
       JSONObject serviceAccount = admin.first("/core/users/?username=" + SERVICE_ACCOUNT + "&include_roles=true");
       assertThat(BootstrapBlueprintIT.names(serviceAccount.getJSONArray("roles_obj"))).containsExactly(SERVICE_ACCOUNT);
       assertThat(admin.first("/core/users/?username=" + BREAK_GLASS).getBoolean("is_superuser")).isTrue();
-
-      ///////////////////////////////////////////////////////////
-      // put the seed groups back as the bootstrap created them //
-      ///////////////////////////////////////////////////////////
-      admin.post("/core/groups/" + identityAdmins + "/remove_user/", new JSONObject().put("pk", member));
-      admin.post("/core/groups/" + kof22Admins + "/remove_user/", new JSONObject().put("pk", member));
-      admin.patch("/core/groups/" + kof22Admins + "/", new JSONObject().put("attributes", new JSONObject()));
-      admin.patch("/core/groups/" + identityAdmins + "/", new JSONObject().put("attributes",
-         new JSONObject().put("qqq.io/permissions", new JSONArray().put("authentik.*")).put("qqq.io/app-access", new JSONArray().put("kof22-website-prod"))));
    }
 
 

src/test/java/com/kingsrook/qbits/authentik/contract/InvitationEnrollmentContractIT.java

@@ -297,7 +297,7 @@ class InvitationEnrollmentContractIT
          var result = AuthentikAdmin.forBackend(config.getApiBackendName()).execute(AdminRequest.of(AdminAction.INVITE_USER, "", "Native onboarding contract")
             .with("email", owner + "@example.test").with("name", "Invitation Contract").with("groupIds", group));
          assertThat(result.ok()).as(result.message()).isTrue();
-         AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the invitation email", () ->
+         AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the invitation email", () ->
          {
             try(var client = HttpClient.newHttpClient())
             {

src/test/java/com/kingsrook/qbits/authentik/contract/QbitWriteContractIT.java

@@ -21,7 +21,6 @@ import java.net.URI;
 import java.net.http.HttpClient;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
-import java.time.Duration;
 import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
@@ -228,9 +227,9 @@ class QbitWriteContractIT
       assertThat(user.getJSONObject("attributes").getString("qqq.io/created-by")).isEqualTo(OPERATOR_EMAIL);
       assertThat(user.getJSONArray("groups_obj").getJSONObject(0).getString("name")).isEqualTo("wr-readers");
 
-      AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the recovery email in Mailpit", () -> mailpitHas("wr.created@example.test"));
+      AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the recovery email in Mailpit", () -> mailpitHas("wr.created@example.test"));
 
-      AuthentikTestStack.waitFor(Duration.ofSeconds(30), "the audit event naming the operator", () ->
+      AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the audit event naming the operator", () ->
       {
          for(Object event : admin.get("/events/events/?search=" + TestApi.encode(OPERATOR_EMAIL) + "&ordering=-created").results())
          {
@@ -437,7 +436,7 @@ class QbitWriteContractIT
       AuthentikAdmin.AdminResult reset = run(AdminRequest.of(AdminAction.SEND_RECOVERY_EMAIL, String.valueOf(pk), "contract test"));
       assertThat(reset.ok()).as(reset.message()).isTrue();
       assertThat(reset.message()).doesNotContain("http");
-      AuthentikTestStack.waitFor(Duration.ofSeconds(60), "the reset email in Mailpit", () -> mailpitHas("wr.target@example.test"));
+      AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "the reset email in Mailpit", () -> mailpitHas("wr.target@example.test"));
    }
 
 

src/test/java/com/kingsrook/qbits/authentik/contract/TokenContractIT.java

@@ -424,7 +424,7 @@ class TokenContractIT
       tokens.add(ordinaryToken);
       String ordinaryEvent = setKeyEvent(ordinaryToken);
       String protectedEvent = setKeyEvent(protectedToken);
-      AuthentikTestStack.waitFor(Duration.ofSeconds(60), "protected-token local notification", () ->
+      AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "protected-token local notification", () ->
          notifications(protectedOwner, protectedEvent).length() > 0);
       Instant deadline = Instant.now().plusSeconds(60);
       do
@@ -489,7 +489,7 @@ class TokenContractIT
       users.add(servicePk);
       tokens.add("service-account-" + created.getString("username") + "-password");
       String[] eventPk = { null };
-      AuthentikTestStack.waitFor(Duration.ofSeconds(30), "service-account model_created event", () ->
+      AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "service-account model_created event", () ->
       {
          JSONArray events = admin.getOk("/events/events/?action=model_created&ordering=-created&page_size=50&username="
             + TestApi.encode(AuthentikTestStack.SERVICE_ACCOUNT)).getJSONArray("results");
@@ -504,7 +504,7 @@ class TokenContractIT
          }
          return (false);
       });
-      AuthentikTestStack.waitFor(Duration.ofSeconds(60), "service-account creation notification", () ->
+      AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "service-account creation notification", () ->
          notifications(member, eventPk[0]).length() > 0);
    }
 
@@ -778,7 +778,7 @@ class TokenContractIT
       String[] found = { null };
       try
       {
-      AuthentikTestStack.waitFor(Duration.ofSeconds(30), "token model_updated event", () ->
+      AuthentikTestStack.waitFor(AuthentikTestStack.ASYNC_TIMEOUT, "token model_updated event", () ->
       {
          //////////////////////////////////////////////////////////////////////
          // set_key logs model_to_dict(token): identifier, intent, user (key  //