From 77c95b1f627be0613c4fb02195d032d329c1250f Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 9 Nov 2026 20:25:27 -0600 Subject: [PATCH] fix(postgres): accept full_page_writes off on Amazon Aurora PostgreSQL only Generation qualification refused every Amazon Aurora PostgreSQL server: Aurora always reports full_page_writes = off and has no parameter to change it, because its storage layer does not write torn pages. The settings query now also returns to_regprocedure('aurora_version()') IS NOT NULL (true on Aurora, false on stock PostgreSQL, never an error). full_page_writes = off is accepted only when that is true. Stock PostgreSQL must still report it on, and fsync on, synchronous_commit on or remote_apply and every other check are unchanged on both. The refusal text now says full-page writes are not required on Aurora. Tests run on a real postgres:17 engine whose settings are really changed; only Aurora is simulated, by creating a function named aurora_version() in the test database. Co-Authored-By: Claude Fable 5.2 --- docs/DURABLE-QUEUE.md | 2 +- .../GenerationPostgres.java | 35 +++++-- .../PostgresGenerationDatabase.java | 45 ++++++++- .../PostgresGenerationQueueTest.java | 97 ++++++++++++++++++- 4 files changed, 168 insertions(+), 11 deletions(-) diff --git a/docs/DURABLE-QUEUE.md b/docs/DURABLE-QUEUE.md index 843a7fe..a0acd76 100644 --- a/docs/DURABLE-QUEUE.md +++ b/docs/DURABLE-QUEUE.md @@ -26,7 +26,7 @@ Submitted records provide only IDs. The source body is always fetched afresh thr ## Database/deployment constraints -The durable queue supports file-backed H2 with configured and actual WRITE_DELAY=0, or native MySQL 8.4 under the live checks in [MYSQL.md](MYSQL.md), or native PostgreSQL 17 under the equivalent live checks in `GenerationPostgres`. Both require fresh independent writable connections and the owned tables on one backend. Ambient QQQ transactions are refused. Native mapping drift and unqualified physical targets are denied. Owned statements have five-second JDBC deadlines; MySQL row/metadata lock waits must each be 1–5 seconds. H2 tests pin 2.5.252 and close all connection providers before a cold reopen. Actual pinned MySQL 8.4.13 tests cover concurrent claims/edits, physical pooled isolation restoration, weakened durability/settings, index/engine/mapping drift, bounded lock contention and server restart. PostgreSQL/Aurora are not implicitly supported. Schema changes are operator migrations; metadata registration does not execute DDL. Host power-loss and volume/backup restoration remain deployment acceptance work. +The durable queue supports file-backed H2 with configured and actual WRITE_DELAY=0, or native MySQL 8.4 under the live checks in [MYSQL.md](MYSQL.md), or native PostgreSQL 17 under the equivalent live checks in `GenerationPostgres`. Both require fresh independent writable connections and the owned tables on one backend. Ambient QQQ transactions are refused. Native mapping drift and unqualified physical targets are denied. Owned statements have five-second JDBC deadlines; MySQL row/metadata lock waits must each be 1–5 seconds. H2 tests pin 2.5.252 and close all connection providers before a cold reopen. Actual pinned MySQL 8.4.13 tests cover concurrent claims/edits, physical pooled isolation restoration, weakened durability/settings, index/engine/mapping drift, bounded lock contention and server restart. Amazon Aurora PostgreSQL 17 passes through the same PostgreSQL checks with one difference: Aurora always runs with `full_page_writes` off and has no parameter to change it, because its storage layer does not write torn pages, so `GenerationPostgres` accepts that value when the server resolves `aurora_version()` (`to_regprocedure('aurora_version()') IS NOT NULL`, evaluated in the settings query). Stock PostgreSQL must still report `full_page_writes` on, and `fsync`, `synchronous_commit` and every other check are unchanged on Aurora. The test for this rule runs on a real `postgres:17` engine with a simulated `aurora_version()` function; no test in this repository runs on Aurora itself. Aurora MySQL is not qualified (see [MYSQL.md](MYSQL.md)). Schema changes are operator migrations; metadata registration does not execute DDL. Host power-loss and volume/backup restoration remain deployment acceptance work. No queue/render/file/script code is executed through a trusted host JavaScript VM. The renderer and scripting integrations continue to require qualified Linux/runsc; this workstation has no runsc runtime, so hostile execution remains unavailable. The queue does not weaken those gates. diff --git a/foundation/src/main/java/com/kingsrook/qbits/documentgeneration/GenerationPostgres.java b/foundation/src/main/java/com/kingsrook/qbits/documentgeneration/GenerationPostgres.java index 77aceb5..43d1056 100644 --- a/foundation/src/main/java/com/kingsrook/qbits/documentgeneration/GenerationPostgres.java +++ b/foundation/src/main/java/com/kingsrook/qbits/documentgeneration/GenerationPostgres.java @@ -38,7 +38,8 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaDat /******************************************************************************* ** Qualification of the native QQQ PostgreSQL17 backend (vendor and backend type ** "postgres") and this QBit's owned tables. Observes the live connection only; - ** it never changes session or server settings, schema or data. + ** it never changes session or server settings, schema or data. Amazon Aurora + ** PostgreSQL is the one accepted variant: see durable(). *******************************************************************************/ final class GenerationPostgres { @@ -46,7 +47,11 @@ final class GenerationPostgres private static final String SETTINGS = "SELECT current_setting('server_version_num')::int, current_setting('transaction_read_only'), pg_is_in_recovery()," + " current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes'), current_setting('TimeZone')," - + " (SELECT setting::bigint FROM pg_settings WHERE name='lock_timeout'), current_setting('transaction_isolation'), current_schema(), current_database()"; + + " (SELECT setting::bigint FROM pg_settings WHERE name='lock_timeout'), current_setting('transaction_isolation'), current_schema(), current_database()," + + " to_regprocedure('aurora_version()') IS NOT NULL"; + + private static final String REQUIREMENTS = "Generation requires synchronous PostgreSQL commits, full-page writes (except on Amazon Aurora PostgreSQL)," + + " a primary writable server, READ COMMITTED and UTC sessions"; private static final String TABLE = "SELECT c.relpersistence FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace" + " WHERE n.nspname=current_schema() AND c.relname=? AND c.relkind='r'"; @@ -91,7 +96,25 @@ final class GenerationPostgres } /*************************************************************************** - ** Exact server, durability, session and owned-table checks for PostgreSQL 17. + ** Commit durability of the observed server. Every server needs + ** synchronous_commit on or remote_apply and fsync on. Stock PostgreSQL also + ** needs full_page_writes on, which protects crash recovery from torn pages. + ** Amazon Aurora PostgreSQL runs with full_page_writes off and offers no + ** parameter to change it, because its storage layer does not write torn + ** pages; so off is accepted there, and only there. Aurora is detected in the + ** settings query by to_regprocedure('aurora_version()') IS NOT NULL, which is + ** true on Aurora, false on stock PostgreSQL and never raises an error (calling + ** a missing function would abort the transaction). + ***************************************************************************/ + static boolean durable(String synchronousCommit, String fsync, String fullPageWrites, boolean aurora) + { + return (("on".equals(synchronousCommit) || "remote_apply".equals(synchronousCommit)) && "on".equals(fsync) + && ("on".equals(fullPageWrites) || aurora && "off".equals(fullPageWrites))); + } + + /*************************************************************************** + ** Exact server, durability, session and owned-table checks for PostgreSQL 17, + ** with the single Amazon Aurora PostgreSQL allowance stated on durable(). ***************************************************************************/ static void qualify(Connection connection, RDBMSBackendMetaData backend) throws SQLException, QException { @@ -105,7 +128,7 @@ final class GenerationPostgres { if(!row.next()) { - throw new QException("Generation requires synchronous PostgreSQL commits, full-page writes, a primary writable server, READ COMMITTED and UTC sessions."); + throw new QException(REQUIREMENTS + "."); } List weak = new java.util.ArrayList<>(); if(row.getInt(1) / 10000 != 17 || row.getString(10) == null || !Objects.equals(backend.getDatabaseName(), row.getString(11))) @@ -116,7 +139,7 @@ final class GenerationPostgres { weak.add("writable primary"); } - if(!Set.of("on", "remote_apply").contains(row.getString(4)) || !"on".equals(row.getString(5)) || !"on".equals(row.getString(6))) + if(!durable(row.getString(4), row.getString(5), row.getString(6), row.getBoolean(12))) { weak.add("synchronous commit"); } @@ -130,7 +153,7 @@ final class GenerationPostgres } if(!weak.isEmpty()) { - throw new QException("Generation requires synchronous PostgreSQL commits, full-page writes, a primary writable server, READ COMMITTED and UTC sessions; not satisfied: " + String.join(", ", weak)); + throw new QException(REQUIREMENTS + "; not satisfied: " + String.join(", ", weak)); } long lockTimeout = row.getLong(8); if(lockTimeout < 1 || lockTimeout > 5000) diff --git a/foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationDatabase.java b/foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationDatabase.java index 3733cb8..cd46f62 100644 --- a/foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationDatabase.java +++ b/foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationDatabase.java @@ -52,13 +52,24 @@ final class PostgresGenerationDatabase implements AutoCloseable ** Starts only this suite's loopback container with durable settings. ***************************************************************************/ PostgresGenerationDatabase() + { + this(true); + } + + /*************************************************************************** + ** Pinned durability settings come from the server command line, which no + ** ALTER SYSTEM can override. An unpinned engine starts with the same values + ** (the PostgreSQL defaults) and lets a test weaken them through system(). + ***************************************************************************/ + PostgresGenerationDatabase(boolean pinned) { // pgjdbc announces the JVM zone as the session TimeZone; qualification requires UTC. previousZone = java.util.TimeZone.getDefault(); java.util.TimeZone.setDefault(java.util.TimeZone.getTimeZone("UTC")); container = new GenericContainer<>(IMAGE) .withEnv("POSTGRES_PASSWORD", rootPassword).withEnv("POSTGRES_DB", "bootstrap") - .withCommand("postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "max_connections=60") + .withCommand(pinned ? new String[] { "postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on", "-c", "max_connections=60" } + : new String[] { "postgres", "-c", "max_connections=60" }) .withExposedPorts(5432) .withCreateContainerCmdModifier(command -> command.getHostConfig().withMemory(768L * 1024 * 1024).withNanoCPUs(2_000_000_000L) .withPortBindings(FixtureEndpoint.publishedBinding(5432))) @@ -143,6 +154,38 @@ final class PostgresGenerationDatabase implements AutoCloseable } } + /*************************************************************************** + ** Change one reloadable server setting of an unpinned engine (ALTER SYSTEM and + ** a configuration reload), then wait until a new session actually reports it. + ** Fixed fixture names and values only, never application input. + ***************************************************************************/ + void system(String name, String value) throws Exception + { + ConnectionManager.closeConnectionProviders(); + try(var connection = DriverManager.getConnection(server + "bootstrap" + OPTIONS, "postgres", rootPassword); var statement = connection.createStatement()) + { + statement.execute("ALTER SYSTEM SET " + name + " = '" + value + "'"); + statement.execute("SELECT pg_reload_conf()"); + } + long deadline = System.nanoTime() + java.time.Duration.ofSeconds(30).toNanos(); + while(true) + { + try(var connection = DriverManager.getConnection(server + "bootstrap" + OPTIONS, "postgres", rootPassword); var statement = connection.createStatement(); + var row = statement.executeQuery("SELECT current_setting('" + name + "')")) + { + if(row.next() && value.equals(row.getString(1))) + { + return; + } + } + if(System.nanoTime() >= deadline) + { + throw new IllegalStateException("PostgreSQL fixture did not apply " + name); + } + Thread.sleep(100); + } + } + /*************************************************************************** ** Cold server restart retains only this container's data layer (committed ** data must survive; Docker may publish a different loopback port). diff --git a/foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationQueueTest.java b/foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationQueueTest.java index 3cd394e..67ae8a9 100644 --- a/foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationQueueTest.java +++ b/foundation/src/test/java/com/kingsrook/qbits/documentgeneration/PostgresGenerationQueueTest.java @@ -41,6 +41,9 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest { private static PostgresGenerationDatabase postgres; + // The engine the fixture calls of the running test address: the shared pinned one unless that test started its own. + private PostgresGenerationDatabase current; + /*********************************************************************** ** One exact disposable engine per suite; all test cases use new databases. ***********************************************************************/ @@ -68,7 +71,7 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest @Override protected RDBMSBackendMetaData database(String name, String defaultUrl) throws Exception { - return (postgres.database(name)); + return (engine().database(name)); } /*********************************************************************** @@ -77,7 +80,7 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest @Override protected Connection connection() throws Exception { - return (postgres.connection()); + return (engine().connection()); } /*********************************************************************** @@ -86,7 +89,15 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest @Override protected void sql(String source) throws Exception { - postgres.sql(source); + engine().sql(source); + } + + /*********************************************************************** + ** The shared pinned engine, or the unpinned one a single test started. + ***********************************************************************/ + private PostgresGenerationDatabase engine() + { + return (current == null ? postgres : current); } /*********************************************************************** @@ -178,6 +189,86 @@ class PostgresGenerationQueueTest extends GenerationQueueJdbcTest assertThat(queue.status(enqueue("restored"), -1, 10).items()).hasSize(2); } + /*************************************************************************** + ** Amazon Aurora PostgreSQL always runs with full_page_writes off (its storage + ** layer does not write torn pages), so that one setting is accepted there and + ** nowhere else. The engine is a real PostgreSQL 17 whose settings are really + ** changed. Only Aurora itself is simulated: a function named aurora_version() + ** is created in the test database, so the same to_regprocedure detection that + ** is true on Aurora resolves here. + ***************************************************************************/ + @Test + void fullPageWritesOffIsAcceptedOnlyOnAuroraAndNothingElseIsRelaxed() throws Exception + { + current = new PostgresGenerationDatabase(false); + try + { + // Rebuild the instance, schema and definitions on the unpinned engine (durable defaults, so this qualifies). + cleanup(); + setup(); + assertThat(queue.advance(enqueue("stock-durable")).state()).isEqualTo("GENERATED"); + + // Stock PostgreSQL with full_page_writes off is refused, and the refusal states the rule. + current.system("full_page_writes", "off"); + assertThatThrownBy(() -> enqueue("stock-torn-pages")).isInstanceOf(QException.class) + .hasMessageContaining("full-page writes (except on Amazon Aurora PostgreSQL)").hasMessageContaining("not satisfied: synchronous commit"); + + // The same server, now resolving aurora_version(), is accepted and work really commits. + sql("CREATE FUNCTION aurora_version() RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT '17.9'::text $$"); + ConnectionManager.closeConnectionProviders(); + assertThat(queue.advance(enqueue("aurora")).state()).isEqualTo("GENERATED"); + + // Aurora relaxes nothing else: fsync off is refused. + current.system("fsync", "off"); + assertThatThrownBy(() -> enqueue("aurora-no-fsync")).isInstanceOf(QException.class).hasMessageContaining("not satisfied: synchronous commit"); + current.system("fsync", "on"); + + // Aurora relaxes nothing else: asynchronous commit is refused. + sql("ALTER ROLE generation_app SET synchronous_commit='off'"); + ConnectionManager.closeConnectionProviders(); + assertThatThrownBy(() -> enqueue("aurora-async-commit")).isInstanceOf(QException.class).hasMessageContaining("not satisfied: synchronous commit"); + sql("ALTER ROLE generation_app SET synchronous_commit='on'"); + ConnectionManager.closeConnectionProviders(); + assertThat(enqueue("aurora-restored")).isNotNull(); + + // Without the function the server is stock PostgreSQL again and is refused again. + sql("DROP FUNCTION aurora_version()"); + ConnectionManager.closeConnectionProviders(); + assertThatThrownBy(() -> enqueue("stock-again")).isInstanceOf(QException.class).hasMessageContaining("not satisfied: synchronous commit"); + } + finally + { + ConnectionManager.closeConnectionProviders(); + current.close(); + current = null; + } + } + + /*************************************************************************** + ** The durability decision itself, for the combinations the live case does + ** not reach: no server and nothing simulated, only the values of one row. + ***************************************************************************/ + @Test + void durabilityRuleAcceptsOnlyTheAuroraFullPageWritesException() + { + assertThat(GenerationPostgres.durable("on", "on", "on", false)).isTrue(); + assertThat(GenerationPostgres.durable("remote_apply", "on", "on", false)).isTrue(); + assertThat(GenerationPostgres.durable("on", "on", "on", true)).isTrue(); + assertThat(GenerationPostgres.durable("on", "on", "off", true)).isTrue(); + assertThat(GenerationPostgres.durable("remote_apply", "on", "off", true)).isTrue(); + assertThat(GenerationPostgres.durable("on", "on", "off", false)).isFalse(); + assertThat(GenerationPostgres.durable("on", "off", "on", false)).isFalse(); + assertThat(GenerationPostgres.durable("on", "off", "off", true)).isFalse(); + for(String weak : List.of("off", "local", "remote_write")) + { + assertThat(GenerationPostgres.durable(weak, "on", "on", false)).isFalse(); + assertThat(GenerationPostgres.durable(weak, "on", "off", true)).isFalse(); + } + assertThat(GenerationPostgres.durable(null, "on", "on", true)).isFalse(); + assertThat(GenerationPostgres.durable("on", null, "on", true)).isFalse(); + assertThat(GenerationPostgres.durable("on", "on", null, true)).isFalse(); + } + /*************************************************************************** ** The native identity must be the postgres module on a real PostgreSQL17 server, not a vendor label. ***************************************************************************/ -- 2.54.0