diff --git a/CHANGELOG.md b/CHANGELOG.md index 2051017..7541022 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All notable changes are documented here, following [Keep a Changelog](https://ke ### Fixed +- The retained upload journal's PostgreSQL 17 qualification accepts Amazon Aurora PostgreSQL, which reports `full_page_writes=off` and does not let a deployment change it. `off` is accepted only when the settings query detects Aurora (`to_regprocedure('aurora_version()') IS NOT NULL`); stock PostgreSQL with `full_page_writes=off` is still refused, and every other requirement is unchanged on Aurora. See `docs/RETAINED-UPLOAD-JOURNAL.md`. - Bounded STRING custom fields carry QQQ's native `ValueTooLongBehavior.ERROR`, so a write longer than `maxLength` is rejected with a field error and the stored value is preserved, instead of passing through unchecked (the QQQ default `PASS_THROUGH`). - Persisted help content is reloaded after each committed runtime metadata refresh, so sections and fields created by publishing a custom-field definition receive their stored `helpContent` rows without a restart. - QBit metadata reports the Maven artifact version from a filtered resource, including release versions. diff --git a/docs/RETAINED-UPLOAD-JOURNAL.md b/docs/RETAINED-UPLOAD-JOURNAL.md index 541eb87..972ea98 100644 --- a/docs/RETAINED-UPLOAD-JOURNAL.md +++ b/docs/RETAINED-UPLOAD-JOURNAL.md @@ -25,7 +25,11 @@ Both owner tables must use the same QQQ backend. Hosts apply DDL through their m | --- | --- | --- | | File H2 | Native `h2`, `jdbc:h2:file:`, actual configured connection, writable real transaction, live `WRITE_DELAY=0` | `UploadJournalTable.h2Script()` | | MySQL 8.4 | Native `mysql`, actual MySQL 8.4 product/version, exact configured URL target and catalog, writable real transaction, both owner tables InnoDB, fixed owner table mappings, journal `utf8mb4_0900_bin` columns/table, single-column unique locator/intake indexes, synchronous commit settings | `UploadJournalTable.mysqlScript()` | -| Aurora MySQL, MariaDB, PostgreSQL, other versions/providers | Refused; no inferred compatibility or durability claim | None | +| PostgreSQL 17 | Native `postgres` (vendor and backend type), actual PostgreSQL 17 product/version, exact configured `jdbc:postgresql://host/catalog` target and catalog, writable real transaction, not in recovery, `synchronous_commit` `on` or `remote_apply`, `fsync=on`, `full_page_writes=on` (see the Aurora rule below), session `TimeZone` UTC, `lock_timeout` from 1 to 5000 ms, READ COMMITTED, both owner tables permanent ordinary tables with fixed mappings, no non-deterministic collation, complete unique indexes on `locator_hash` and `intake_key` | `UploadJournalTable.postgresScript()` | +| Amazon Aurora PostgreSQL 17 | Every PostgreSQL 17 requirement above, except that `full_page_writes=off` is accepted | `UploadJournalTable.postgresScript()` | +| Aurora MySQL, MariaDB, other versions/providers | Refused; no inferred compatibility or durability claim | None | + +Aurora rule: stock PostgreSQL must report `full_page_writes=on`, because recovery repairs a partly written page from the full page image in the write-ahead log. Amazon Aurora PostgreSQL reports `full_page_writes=off` and the parameter is not in its cluster parameter group, so a deployment cannot turn it on; its storage layer does not write torn pages, so the protection is not needed there. The qualification therefore accepts `off` only when the same settings query reports `to_regprocedure('aurora_version()') IS NOT NULL`, which is true on Aurora, false on stock PostgreSQL 17 and never raises. Nothing else is relaxed on Aurora: `synchronous_commit`, `fsync`, recovery state, isolation, time zone and lock timeout are checked exactly as on stock PostgreSQL. The detection is a name lookup, so a function named `aurora_version()` on the session's search path of a stock server is reported as Aurora; creating one is a schema change inside the host's trust boundary, like the settings themselves. The MySQL adapter resolves explicit or native-generated URLs through QQQ `ConnectionManager`. It accepts a single ordinary `jdbc:mysql://host:port/catalog` target with a simple catalog identifier. Proxy/load-balancing URLs, alternate catalogs or table remapping fail closed. Error messages do not echo URLs or credentials. Driver options remain host-owned; TLS and credentials are not changed by the journal. diff --git a/src/main/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresJournalDatabase.java b/src/main/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresJournalDatabase.java index 11c26bf..3b435a7 100644 --- a/src/main/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresJournalDatabase.java +++ b/src/main/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresJournalDatabase.java @@ -47,14 +47,25 @@ import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSTableBackendDe ** pair, on the native QQQ postgres backend (vendor and backend type "postgres"). ** Observation only: no schema migration and no session or server setting is ** changed at runtime. + ** + ** Amazon Aurora PostgreSQL 17 is the one qualified variant: it is held to every + ** rule here except full_page_writes, which Aurora reports as "off" and does not + ** let a deployment change (see tornPagesPrevented). *******************************************************************************/ final class PostgresJournalDatabase { static final String VENDOR = "postgres"; - private static final String SETTINGS = "SELECT current_setting('server_version_num')::int, current_setting('transaction_read_only'), pg_is_in_recovery()," + ///////////////////////////////////////////////////////////////////////////////////////////// + // Column 11 detects Amazon Aurora PostgreSQL inside the same statement. to_regprocedure // + // returns NULL when no aurora_version() function resolves, so it is true on Aurora, false // + // on stock PostgreSQL 17 and never raises. Calling aurora_version() directly would abort // + // the transaction on stock PostgreSQL, and pg_proc does not list the function on Aurora. // + ///////////////////////////////////////////////////////////////////////////////////////////// + static final String SETTINGS = "SELECT current_setting('server_version_num')::int, current_setting('transaction_read_only'), pg_is_in_recovery()," + " current_setting('synchronous_commit'), current_setting('fsync'), current_setting('full_page_writes'), current_setting('TimeZone')," - + " (SELECT setting::bigint FROM pg_settings WHERE name='lock_timeout'), current_setting('transaction_isolation'), current_database()"; + + " (SELECT setting::bigint FROM pg_settings WHERE name='lock_timeout'), current_setting('transaction_isolation'), current_database()," + + " to_regprocedure('aurora_version()') IS NOT NULL"; private static final String TABLES = "SELECT c.relname, c.relkind, c.relpersistence FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace" + " WHERE n.nspname=current_schema() AND c.relname = ANY(?)"; @@ -124,7 +135,8 @@ final class PostgresJournalDatabase try(ResultSet settings = statement.executeQuery(SETTINGS)) { if(!settings.next() || settings.getInt(1) / 10000 != 17 || !"off".equals(settings.getString(2)) || settings.getBoolean(3) - || !Set.of("on", "remote_apply").contains(settings.getString(4)) || !"on".equals(settings.getString(5)) || !"on".equals(settings.getString(6)) + || !Set.of("on", "remote_apply").contains(settings.getString(4)) || !"on".equals(settings.getString(5)) + || !tornPagesPrevented(settings.getString(6), settings.getBoolean(11)) || !Set.of("UTC", "Etc/UTC").contains(settings.getString(7)) || settings.getLong(8) < 1 || settings.getLong(8) > 5000 || !"read committed".equals(settings.getString(9)) || !catalog.equals(settings.getString(10))) { @@ -147,6 +159,28 @@ final class PostgresJournalDatabase + /******************************************************************************* + ** The torn-page rule, as a pure decision over the observed full_page_writes + ** value and the Aurora detection column. + ** + ** Stock PostgreSQL must report full_page_writes "on": after a crash that leaves + ** a partly written page, recovery repairs it from the full page image in the + ** write-ahead log, so "off" there can lose committed journal rows. + ** + ** Amazon Aurora PostgreSQL reports full_page_writes "off" (measured on engine + ** 17.9: source default, and the parameter is not in the cluster parameter group, + ** so a deployment cannot turn it on). Its storage layer does not write torn + ** pages, so the protection is not needed there. "off" is therefore accepted + ** only when the server is Aurora; every other rule (synchronous_commit, fsync, + ** recovery, isolation, time zone, lock timeout) stays as strict on Aurora. + *******************************************************************************/ + static boolean tornPagesPrevented(String fullPageWrites, boolean aurora) + { + return ("on".equals(fullPageWrites) || (aurora && "off".equals(fullPageWrites))); + } + + + /******************************************************************************* ** Permanent (logged) ordinary tables only, deterministic text comparison, and ** complete valid non-partial unique indexes for the journal's two identities. diff --git a/src/test/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresJournalDatabaseTest.java b/src/test/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresJournalDatabaseTest.java new file mode 100644 index 0000000..2c6585d --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresJournalDatabaseTest.java @@ -0,0 +1,83 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.collaboration.attachments.uploadjournal; + + +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** The torn-page decision of the PostgreSQL journal qualification, without a + ** database: the rule is a pure function of the observed full_page_writes value + ** and the Aurora detection column. The real-server cases, including the + ** detection column itself, are in the opt-in PostgresUploadJournalTest. + *******************************************************************************/ +class PostgresJournalDatabaseTest +{ + + /******************************************************************************* + ** "on" qualifies everywhere; "off" qualifies only on Amazon Aurora PostgreSQL. + *******************************************************************************/ + @Test + void fullPageWritesOffIsAcceptedOnlyOnAurora() + { + assertThat(PostgresJournalDatabase.tornPagesPrevented("on", false)).isTrue(); + assertThat(PostgresJournalDatabase.tornPagesPrevented("on", true)).isTrue(); + assertThat(PostgresJournalDatabase.tornPagesPrevented("off", true)).isTrue(); + assertThat(PostgresJournalDatabase.tornPagesPrevented("off", false)).isFalse(); + } + + + + /******************************************************************************* + ** Anything that is neither "on" nor Aurora's "off" is refused, Aurora or not: + ** a missing value, another spelling or an unexpected value. + *******************************************************************************/ + @Test + void unknownFullPageWritesValuesAreRefused() + { + for(boolean aurora : new boolean[] { false, true }) + { + assertThat(PostgresJournalDatabase.tornPagesPrevented(null, aurora)).isFalse(); + assertThat(PostgresJournalDatabase.tornPagesPrevented("", aurora)).isFalse(); + assertThat(PostgresJournalDatabase.tornPagesPrevented("OFF", aurora)).isFalse(); + assertThat(PostgresJournalDatabase.tornPagesPrevented("ON", aurora)).isFalse(); + assertThat(PostgresJournalDatabase.tornPagesPrevented("false", aurora)).isFalse(); + } + } + + + + /******************************************************************************* + ** The settings query detects Aurora only through to_regprocedure, as its last + ** column. It never calls aurora_version() directly, which would abort the + ** transaction on stock PostgreSQL. + *******************************************************************************/ + @Test + void settingsQueryDetectsAuroraOnlyThroughToRegprocedure() + { + String detection = "to_regprocedure('aurora_version()') IS NOT NULL"; + assertThat(PostgresJournalDatabase.SETTINGS).endsWith(", " + detection); + assertThat(PostgresJournalDatabase.SETTINGS.replace(detection, "")).doesNotContain("aurora"); + } +} diff --git a/src/test/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresUploadJournalTest.java b/src/test/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresUploadJournalTest.java index 2b68742..481bfa1 100644 --- a/src/test/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresUploadJournalTest.java +++ b/src/test/java/com/kingsrook/qbits/collaboration/attachments/uploadjournal/PostgresUploadJournalTest.java @@ -81,6 +81,11 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; ** dynamic port, random synthetic passwords); the application connects as a ** DML-only role, and the superuser is used only for fixture DDL and fault ** injection. The attachment row is a minimal stand-in, not a full intake fixture. + ** + ** No Amazon Aurora server is used. The Aurora cases run on this stock PostgreSQL + ** 17 server and simulate exactly two things: a function named aurora_version() + ** created in the suite's database (so to_regprocedure resolves, as it does on + ** Aurora), and full_page_writes turned off on the server (as Aurora reports it). *******************************************************************************/ @EnabledIfSystemProperty(named = "postgresJournalTests", matches = "true") class PostgresUploadJournalTest @@ -107,6 +112,9 @@ class PostgresUploadJournalTest /******************************************************************************* ** Start only the disposable server with explicit durable settings. pgjdbc ** announces the JVM zone as the session TimeZone, so the suite runs in UTC. + ** fsync and full_page_writes are set through ALTER SYSTEM rather than on the + ** command line, because a command-line value cannot be changed by a reload and + ** the Aurora and durability cases need to turn each one off and back on. *******************************************************************************/ @BeforeAll static void start() throws Exception @@ -115,7 +123,7 @@ class PostgresUploadJournalTest TimeZone.setDefault(TimeZone.getTimeZone("UTC")); postgres = new GenericContainer<>("postgres:17") .withEnv("POSTGRES_PASSWORD", ROOT_PASSWORD).withEnv("POSTGRES_DB", DATABASE) - .withCommand("postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on") + .withCommand("postgres", "-c", "synchronous_commit=on") .withExposedPorts(5432) .withCreateContainerCmdModifier(command -> { @@ -138,6 +146,8 @@ class PostgresUploadJournalTest statement.execute("ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO " + APP_ROLE); statement.execute("ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO " + APP_ROLE); } + serverSetting("fsync", "on"); + serverSetting("full_page_writes", "on"); } @@ -174,6 +184,9 @@ class PostgresUploadJournalTest execute("ALTER ROLE " + APP_ROLE + " RESET synchronous_commit"); execute("ALTER ROLE " + APP_ROLE + " RESET default_transaction_isolation"); execute("ALTER ROLE " + APP_ROLE + " RESET default_transaction_read_only"); + serverSetting("fsync", "on"); + serverSetting("full_page_writes", "on"); + execute("DROP FUNCTION IF EXISTS aurora_version()"); backend = new PostgreSQLBackendMetaData(); backend.setName("postgres_journal"); backend.setDatabaseName(DATABASE); @@ -440,6 +453,67 @@ class PostgresUploadJournalTest + /******************************************************************************* + ** The settings query carries the Aurora detection as its eleventh column. As + ** the DML-only application role on a stock server it reports false and does not + ** raise, so the transaction stays usable; once aurora_version() resolves (the + ** simulated Aurora) it reports true. + *******************************************************************************/ + @Test + void settingsQueryDetectsAuroraWithoutRaising() throws Exception + { + assertThat(settingsQueryReportsAurora()).isFalse(); + simulateAurora(); + assertThat(settingsQueryReportsAurora()).isTrue(); + } + + + + /******************************************************************************* + ** full_page_writes=off is refused on stock PostgreSQL and accepted only once + ** the server is detected as Aurora (simulated); removing the detection refuses + ** it again, with no row written by a refused call. + *******************************************************************************/ + @Test + void fullPageWritesOffIsAcceptedOnlyOnAurora() throws Exception + { + serverSetting("full_page_writes", "off"); + assertUnsupported(); + assertThat(count("collaboration_upload_journal")).isZero(); + simulateAurora(); + ConnectionManager.closeConnectionProviders(); + assertThat(journal.prepare(candidate("aurora")).id()).isPositive(); + assertThat(count("collaboration_upload_journal")).isEqualTo(1); + execute("DROP FUNCTION aurora_version()"); + assertUnsupported(); + assertThat(count("collaboration_upload_journal")).isEqualTo(1); + } + + + + /******************************************************************************* + ** Aurora (simulated, full_page_writes=off) gets no other allowance: with + ** synchronous_commit off, or with fsync off, it is refused like any server. + *******************************************************************************/ + @Test + void auroraIsStillRefusedWithoutSynchronousCommitOrFsync() throws Exception + { + simulateAurora(); + serverSetting("full_page_writes", "off"); + assertThat(journal.prepare(candidate("aurora-baseline")).id()).isPositive(); + execute("ALTER ROLE " + APP_ROLE + " SET synchronous_commit='off'"); + assertUnsupported(); + execute("ALTER ROLE " + APP_ROLE + " RESET synchronous_commit"); + serverSetting("fsync", "off"); + assertUnsupported(); + serverSetting("fsync", "on"); + ConnectionManager.closeConnectionProviders(); + assertThat(journal.prepare(candidate("aurora-restored")).id()).isPositive(); + assertThat(count("collaboration_upload_journal")).isEqualTo(2); + } + + + /******************************************************************************* ** Physical drift is refused: an unlogged owner table (lost in crash recovery) ** and a non-deterministic collation on a journal column. @@ -565,6 +639,75 @@ class PostgresUploadJournalTest + /******************************************************************************* + ** Make this stock server look like Amazon Aurora PostgreSQL to the detection: + ** a function named aurora_version() in the suite's schema. Nothing else about + ** Aurora is simulated by this call. + *******************************************************************************/ + private void simulateAurora() throws SQLException + { + execute("CREATE FUNCTION aurora_version() RETURNS text LANGUAGE sql AS 'SELECT ''17.9''::text'"); + } + + + + /******************************************************************************* + ** Run the production settings query as the application role inside one + ** transaction, then prove that transaction was not aborted by it. + *******************************************************************************/ + private boolean settingsQueryReportsAurora() throws SQLException + { + try(Connection connection = DriverManager.getConnection(appUrl(), APP_ROLE, APP_PASSWORD); Statement statement = connection.createStatement()) + { + connection.setAutoCommit(false); + boolean aurora; + try(ResultSet settings = statement.executeQuery(PostgresJournalDatabase.SETTINGS)) + { + assertThat(settings.getMetaData().getColumnCount()).isEqualTo(11); + assertThat(settings.next()).isTrue(); + aurora = settings.getBoolean(11); + assertThat(settings.wasNull()).isFalse(); + } + try(ResultSet stillUsable = statement.executeQuery("SELECT 1")) + { + assertThat(stillUsable.next()).isTrue(); + } + connection.rollback(); + return (aurora); + } + } + + + + /******************************************************************************* + ** Change a reload-level (sighup) setting of the disposable server and wait + ** until a new session reports it, so the next pooled session observes it too. + *******************************************************************************/ + private static void serverSetting(String name, String value) throws Exception + { + try(Connection connection = root(); Statement statement = connection.createStatement()) + { + statement.execute("ALTER SYSTEM SET " + name + "=" + value); + statement.execute("SELECT pg_reload_conf()"); + } + long deadline = System.nanoTime() + Duration.ofSeconds(30).toNanos(); + while(System.nanoTime() < deadline) + { + try(Connection connection = root(); Statement statement = connection.createStatement(); + ResultSet setting = statement.executeQuery("SELECT current_setting('" + name + "')")) + { + if(setting.next() && value.equals(setting.getString(1))) + { + return; + } + } + Thread.sleep(100); + } + throw (new IllegalStateException("The disposable server did not apply " + name + "=" + value + ".")); + } + + + /******************************************************************************* ** Prepare a leased manifest whose stored bytes have been verified by owner. *******************************************************************************/