docs/DURABLE-QUEUE.md

@@ -26,7 +26,7 @@ Submitted records provide only IDs. The source body is always fetched afresh thr
 
 ## Database/deployment constraints
 
-The durable queue supports file-backed H2 with configured and actual WRITE_DELAY=0, or native MySQL 8.4 under the live checks in [MYSQL.md](MYSQL.md), or native PostgreSQL 17 under the equivalent live checks in `GenerationPostgres`. Both require fresh independent writable connections and the owned tables on one backend. Ambient QQQ transactions are refused. Native mapping drift and unqualified physical targets are denied. Owned statements have five-second JDBC deadlines; MySQL row/metadata lock waits must each be 1–5 seconds. H2 tests pin 2.5.252 and close all connection providers before a cold reopen. Actual pinned MySQL 8.4.13 tests cover concurrent claims/edits, physical pooled isolation restoration, weakened durability/settings, index/engine/mapping drift, bounded lock contention and server restart. PostgreSQL/Aurora are not implicitly supported. Schema changes are operator migrations; metadata registration does not execute DDL. Host power-loss and volume/backup restoration remain deployment acceptance work.
+The durable queue supports file-backed H2 with configured and actual WRITE_DELAY=0, or native MySQL 8.4 under the live checks in [MYSQL.md](MYSQL.md), or native PostgreSQL 17 under the equivalent live checks in `GenerationPostgres`. Both require fresh independent writable connections and the owned tables on one backend. Ambient QQQ transactions are refused. Native mapping drift and unqualified physical targets are denied. Owned statements have five-second JDBC deadlines; MySQL row/metadata lock waits must each be 1–5 seconds. H2 tests pin 2.5.252 and close all connection providers before a cold reopen. Actual pinned MySQL 8.4.13 tests cover concurrent claims/edits, physical pooled isolation restoration, weakened durability/settings, index/engine/mapping drift, bounded lock contention and server restart. Amazon Aurora PostgreSQL 17 passes through the same PostgreSQL checks with one difference: Aurora always runs with `full_page_writes` off and has no parameter to change it, because its storage layer does not write torn pages, so `GenerationPostgres` accepts that value when the server resolves `aurora_version()` (`to_regprocedure('aurora_version()') IS NOT NULL`, evaluated in the settings query). Stock PostgreSQL must still report `full_page_writes` on, and `fsync`, `synchronous_commit` and every other check are unchanged on Aurora. The test for this rule runs on a real `postgres:17` engine with a simulated `aurora_version()` function; no test in this repository runs on Aurora itself. Aurora MySQL is not qualified (see [MYSQL.md](MYSQL.md)). Schema changes are operator migrations; metadata registration does not execute DDL. Host power-loss and volume/backup restoration remain deployment acceptance work.
 
 No queue/render/file/script code is executed through a trusted host JavaScript VM. The renderer and scripting integrations continue to require qualified Linux/runsc; this workstation has no runsc runtime, so hostile execution remains unavailable. The queue does not weaken those gates.
 

foundation/src/main/java/com/kingsrook/qbits/documentgeneration/GenerationPostgres.java

@@ -38,7 +38,8 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaDat
 /*******************************************************************************
  ** Qualification of the native QQQ PostgreSQL17 backend (vendor and backend type
  ** "postgres") and this QBit's owned tables. Observes the live connection only;
- ** it never changes session or server settings, schema or data.
+ ** it never changes session or server settings, schema or data. Amazon Aurora
+ ** PostgreSQL is the one accepted variant: see durable().
  *******************************************************************************/
 final class GenerationPostgres
 {
@@ -46,7 +47,11 @@ final class GenerationPostgres
 
    private static final String SETTINGS = "SELECT current_setting('server_version_num')::int, current_setting('transaction_read_only'), pg_is_in_recovery(),"
       + " current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes'), current_setting('TimeZone'),"
-      + " (SELECT setting::bigint FROM pg_settings WHERE name='lock_timeout'), current_setting('transaction_isolation'), current_schema(), current_database()";
+      + " (SELECT setting::bigint FROM pg_settings WHERE name='lock_timeout'), current_setting('transaction_isolation'), current_schema(), current_database(),"
+      + " to_regprocedure('aurora_version()') IS NOT NULL";
+
+   private static final String REQUIREMENTS = "Generation requires synchronous PostgreSQL commits, full-page writes (except on Amazon Aurora PostgreSQL),"
+      + " a primary writable server, READ COMMITTED and UTC sessions";
 
    private static final String TABLE = "SELECT c.relpersistence FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace"
       + " WHERE n.nspname=current_schema() AND c.relname=? AND c.relkind='r'";
@@ -91,7 +96,25 @@ final class GenerationPostgres
    }
 
    /***************************************************************************
-    ** Exact server, durability, session and owned-table checks for PostgreSQL 17.
+    ** Commit durability of the observed server. Every server needs
+    ** synchronous_commit on or remote_apply and fsync on. Stock PostgreSQL also
+    ** needs full_page_writes on, which protects crash recovery from torn pages.
+    ** Amazon Aurora PostgreSQL runs with full_page_writes off and offers no
+    ** parameter to change it, because its storage layer does not write torn
+    ** pages; so off is accepted there, and only there. Aurora is detected in the
+    ** settings query by to_regprocedure('aurora_version()') IS NOT NULL, which is
+    ** true on Aurora, false on stock PostgreSQL and never raises an error (calling
+    ** a missing function would abort the transaction).
+    ***************************************************************************/
+   static boolean durable(String synchronousCommit, String fsync, String fullPageWrites, boolean aurora)
+   {
+      return (("on".equals(synchronousCommit) || "remote_apply".equals(synchronousCommit)) && "on".equals(fsync)
+         && ("on".equals(fullPageWrites) || aurora && "off".equals(fullPageWrites)));
+   }
+
+   /***************************************************************************
+    ** Exact server, durability, session and owned-table checks for PostgreSQL 17,
+    ** with the single Amazon Aurora PostgreSQL allowance stated on durable().
     ***************************************************************************/
    static void qualify(Connection connection, RDBMSBackendMetaData backend) throws SQLException, QException
    {
@@ -105,7 +128,7 @@ final class GenerationPostgres
       {
          if(!row.next())
          {
-            throw new QException("Generation requires synchronous PostgreSQL commits, full-page writes, a primary writable server, READ COMMITTED and UTC sessions.");
+            throw new QException(REQUIREMENTS + ".");
          }
          List<String> weak = new java.util.ArrayList<>();
          if(row.getInt(1) / 10000 != 17 || row.getString(10) == null || !Objects.equals(backend.getDatabaseName(), row.getString(11)))
@@ -116,7 +139,7 @@ final class GenerationPostgres
          {
             weak.add("writable primary");
          }
-         if(!Set.of("on", "remote_apply").contains(row.getString(4)) || !"on".equals(row.getString(5)) || !"on".equals(row.getString(6)))
+         if(!durable(row.getString(4), row.getString(5), row.getString(6), row.getBoolean(12)))
          {
             weak.add("synchronous commit");
          }
@@ -130,7 +153,7 @@ final class GenerationPostgres
          }
          if(!weak.isEmpty())
          {
-            throw new QException("Generation requires synchronous PostgreSQL commits, full-page writes, a primary writable server, READ COMMITTED and UTC sessions; not satisfied: " + String.join(", ", weak));
+            throw new QException(REQUIREMENTS + "; not satisfied: " + String.join(", ", weak));
          }
          long lockTimeout = row.getLong(8);
          if(lockTimeout < 1 || lockTimeout > 5000)

foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationDatabase.java

@@ -52,13 +52,24 @@ final class PostgresGenerationDatabase implements AutoCloseable
     ** Starts only this suite's loopback container with durable settings.
     ***************************************************************************/
    PostgresGenerationDatabase()
+   {
+      this(true);
+   }
+
+   /***************************************************************************
+    ** Pinned durability settings come from the server command line, which no
+    ** ALTER SYSTEM can override. An unpinned engine starts with the same values
+    ** (the PostgreSQL defaults) and lets a test weaken them through system().
+    ***************************************************************************/
+   PostgresGenerationDatabase(boolean pinned)
    {
       // pgjdbc announces the JVM zone as the session TimeZone; qualification requires UTC.
       previousZone = java.util.TimeZone.getDefault();
       java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC"));
       container = new GenericContainer<>(IMAGE)
          .withEnv("POSTGRES_PASSWORD", rootPassword).withEnv("POSTGRES_DB", "bootstrap")
-         .withCommand("postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "max_connections=60")
+         .withCommand(pinned ? new String[] { "postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "max_connections=60" }
+            : new String[] { "postgres", "-c", "max_connections=60" })
          .withExposedPorts(5432)
          .withCreateContainerCmdModifier(command -> command.getHostConfig().withMemory(768L * 1024 * 1024).withNanoCPUs(2_000_000_000L)
             .withPortBindings(FixtureEndpoint.publishedBinding(5432)))
@@ -143,6 +154,38 @@ final class PostgresGenerationDatabase implements AutoCloseable
       }
    }
 
+   /***************************************************************************
+    ** Change one reloadable server setting of an unpinned engine (ALTER SYSTEM and
+    ** a configuration reload), then wait until a new session actually reports it.
+    ** Fixed fixture names and values only, never application input.
+    ***************************************************************************/
+   void system(String name, String value) throws Exception
+   {
+      ConnectionManager.closeConnectionProviders();
+      try(var connection = DriverManager.getConnection(server + "bootstrap" + OPTIONS, "postgres", rootPassword); var statement = connection.createStatement())
+      {
+         statement.execute("ALTER SYSTEM SET " + name + " = '" + value + "'");
+         statement.execute("SELECT pg_reload_conf()");
+      }
+      long deadline = System.nanoTime() + java.time.Duration.ofSeconds(30).toNanos();
+      while(true)
+      {
+         try(var connection = DriverManager.getConnection(server + "bootstrap" + OPTIONS, "postgres", rootPassword); var statement = connection.createStatement();
+            var row = statement.executeQuery("SELECT current_setting('" + name + "')"))
+         {
+            if(row.next() && value.equals(row.getString(1)))
+            {
+               return;
+            }
+         }
+         if(System.nanoTime() >= deadline)
+         {
+            throw new IllegalStateException("PostgreSQL fixture did not apply " + name);
+         }
+         Thread.sleep(100);
+      }
+   }
+
    /***************************************************************************
     ** Cold server restart retains only this container's data layer (committed
     ** data must survive; Docker may publish a different loopback port).

foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationQueueTest.java

@@ -41,6 +41,9 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest
 {
    private static PostgresGenerationDatabase postgres;
 
+   // The engine the fixture calls of the running test address: the shared pinned one unless that test started its own.
+   private PostgresGenerationDatabase current;
+
    /***********************************************************************
     ** One exact disposable engine per suite; all test cases use new databases.
     ***********************************************************************/
@@ -68,7 +71,7 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest
    @Override
    protected RDBMSBackendMetaData database(String name, String defaultUrl) throws Exception
    {
-      return (postgres.database(name));
+      return (engine().database(name));
    }
 
    /***********************************************************************
@@ -77,7 +80,7 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest
    @Override
    protected Connection connection() throws Exception
    {
-      return (postgres.connection());
+      return (engine().connection());
    }
 
    /***********************************************************************
@@ -86,7 +89,15 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest
    @Override
    protected void sql(String source) throws Exception
    {
-      postgres.sql(source);
+      engine().sql(source);
+   }
+
+   /***********************************************************************
+    ** The shared pinned engine, or the unpinned one a single test started.
+    ***********************************************************************/
+   private PostgresGenerationDatabase engine()
+   {
+      return (current == null ? postgres : current);
    }
 
    /***********************************************************************
@@ -178,6 +189,86 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest
       assertThat(queue.status(enqueue("restored"), -1, 10).items()).hasSize(2);
    }
 
+   /***************************************************************************
+    ** Amazon Aurora PostgreSQL always runs with full_page_writes off (its storage
+    ** layer does not write torn pages), so that one setting is accepted there and
+    ** nowhere else. The engine is a real PostgreSQL 17 whose settings are really
+    ** changed. Only Aurora itself is simulated: a function named aurora_version()
+    ** is created in the test database, so the same to_regprocedure detection that
+    ** is true on Aurora resolves here.
+    ***************************************************************************/
+   @Test
+   void fullPageWritesOffIsAcceptedOnlyOnAuroraAndNothingElseIsRelaxed() throws Exception
+   {
+      current = new PostgresGenerationDatabase(false);
+      try
+      {
+         // Rebuild the instance, schema and definitions on the unpinned engine (durable defaults, so this qualifies).
+         cleanup();
+         setup();
+         assertThat(queue.advance(enqueue("stock-durable")).state()).isEqualTo("GENERATED");
+
+         // Stock PostgreSQL with full_page_writes off is refused, and the refusal states the rule.
+         current.system("full_page_writes", "off");
+         assertThatThrownBy(() -> enqueue("stock-torn-pages")).isInstanceOf(QException.class)
+            .hasMessageContaining("full-page writes (except on Amazon Aurora PostgreSQL)").hasMessageContaining("not satisfied: synchronous commit");
+
+         // The same server, now resolving aurora_version(), is accepted and work really commits.
+         sql("CREATE FUNCTION aurora_version() RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT '17.9'::text $$");
+         ConnectionManager.closeConnectionProviders();
+         assertThat(queue.advance(enqueue("aurora")).state()).isEqualTo("GENERATED");
+
+         // Aurora relaxes nothing else: fsync off is refused.
+         current.system("fsync", "off");
+         assertThatThrownBy(() -> enqueue("aurora-no-fsync")).isInstanceOf(QException.class).hasMessageContaining("not satisfied: synchronous commit");
+         current.system("fsync", "on");
+
+         // Aurora relaxes nothing else: asynchronous commit is refused.
+         sql("ALTER ROLE generation_app SET synchronous_commit='off'");
+         ConnectionManager.closeConnectionProviders();
+         assertThatThrownBy(() -> enqueue("aurora-async-commit")).isInstanceOf(QException.class).hasMessageContaining("not satisfied: synchronous commit");
+         sql("ALTER ROLE generation_app SET synchronous_commit='on'");
+         ConnectionManager.closeConnectionProviders();
+         assertThat(enqueue("aurora-restored")).isNotNull();
+
+         // Without the function the server is stock PostgreSQL again and is refused again.
+         sql("DROP FUNCTION aurora_version()");
+         ConnectionManager.closeConnectionProviders();
+         assertThatThrownBy(() -> enqueue("stock-again")).isInstanceOf(QException.class).hasMessageContaining("not satisfied: synchronous commit");
+      }
+      finally
+      {
+         ConnectionManager.closeConnectionProviders();
+         current.close();
+         current = null;
+      }
+   }
+
+   /***************************************************************************
+    ** The durability decision itself, for the combinations the live case does
+    ** not reach: no server and nothing simulated, only the values of one row.
+    ***************************************************************************/
+   @Test
+   void durabilityRuleAcceptsOnlyTheAuroraFullPageWritesException()
+   {
+      assertThat(GenerationPostgres.durable("on", "on", "on", false)).isTrue();
+      assertThat(GenerationPostgres.durable("remote_apply", "on", "on", false)).isTrue();
+      assertThat(GenerationPostgres.durable("on", "on", "on", true)).isTrue();
+      assertThat(GenerationPostgres.durable("on", "on", "off", true)).isTrue();
+      assertThat(GenerationPostgres.durable("remote_apply", "on", "off", true)).isTrue();
+      assertThat(GenerationPostgres.durable("on", "on", "off", false)).isFalse();
+      assertThat(GenerationPostgres.durable("on", "off", "on", false)).isFalse();
+      assertThat(GenerationPostgres.durable("on", "off", "off", true)).isFalse();
+      for(String weak : List.of("off", "local", "remote_write"))
+      {
+         assertThat(GenerationPostgres.durable(weak, "on", "on", false)).isFalse();
+         assertThat(GenerationPostgres.durable(weak, "on", "off", true)).isFalse();
+      }
+      assertThat(GenerationPostgres.durable(null, "on", "on", true)).isFalse();
+      assertThat(GenerationPostgres.durable("on", null, "on", true)).isFalse();
+      assertThat(GenerationPostgres.durable("on", "on", null, true)).isFalse();
+   }
+
    /***************************************************************************
     ** The native identity must be the postgres module on a real PostgreSQL17 server, not a vendor label.
     ***************************************************************************/