diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e238a1..a6f3854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and ## [Unreleased] ### Added +- Foreign keys between all five tables in `docs/ddl/webhooks-postgres.sql`, named `fk__`: an endpoint's secrets and subscriptions are deleted with it (CASCADE), deliveries and attempts keep their endpoint (RESTRICT: cancel and Delete Deliveries before deleting an endpoint with history), a delivery outlives its subscription (SET NULL) and attempts go with their delivery (CASCADE). New index `webhooks_delivery_subscription_ix`. +- `docs/ddl/webhooks-foreign-keys-upgrade.sql` for existing databases: records orphan rows in `qqq_fk_quarantine` (nulls nullable references, deletes rows with a missing required parent and keeps them as JSON), replaces the NO ACTION endpoint keys, and is idempotent. +- `ForeignKeyGuardrailTest` applies the DDL to PostgreSQL 17 (Testcontainers) and fails when a join or table possible-value field between webhooks tables has no foreign key. - Every table explains itself: a help bundle (`help/qbit-webhooks.yaml`, QQQ help-bundle format) says what each `webhooks_*` table is for and how its rows get created, including where event types come from. It shows on the table screen and when the table is empty. Applied by `WebhooksHelp` (STAND-IN for QRun-IO/qqq#1198); a host's own help is kept. - A Setup card on the Webhooks home lists event types, encryption key, endpoints, signing secrets and subscriptions with their state and the next step. - `WebhooksTablesGuardrailTest` (each table is used at runtime, seeded or explained, and has help) and a test that a previous secret stops signing at its Signs Until date even before the maintenance job retires it. diff --git a/docs/ddl/webhooks-foreign-keys-upgrade.sql b/docs/ddl/webhooks-foreign-keys-upgrade.sql new file mode 100644 index 0000000..4f39634 --- /dev/null +++ b/docs/ddl/webhooks-foreign-keys-upgrade.sql @@ -0,0 +1,68 @@ +-- qbit-webhooks: add the foreign keys to an existing PostgreSQL schema +-- (one created from webhooks-postgres.sql before they were declared there). +-- +-- Safe to run more than once: a foreign key that already exists with the +-- right ON DELETE (under any name) is left alone. Before each key is added, +-- rows that point at a missing parent are recorded in qqq_fk_quarantine and +-- then set to NULL (nullable column) or deleted (NOT NULL column; the whole +-- row is kept in row_data). An existing key on the column with a different +-- ON DELETE (endpoint_secret and subscription -> endpoint were NO ACTION) is +-- replaced by the named one. +-- +-- Order matters: deliveries of a missing endpoint are removed before their +-- attempts are checked. + +BEGIN; + +CREATE TABLE IF NOT EXISTS qqq_fk_quarantine (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(100) NOT NULL, column_name VARCHAR(100) NOT NULL, row_id VARCHAR(100), missing_parent_id VARCHAR(100), action VARCHAR(20) NOT NULL, row_data TEXT, changeset VARCHAR(200), quarantined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP); + +CREATE OR REPLACE FUNCTION pg_temp.qqq_add_fk(p_table TEXT, p_column TEXT, p_parent TEXT, p_on_delete TEXT) RETURNS VOID LANGUAGE plpgsql AS +$fn$ +DECLARE + v_name TEXT := 'fk_' || p_table || '_' || p_column; + v_rule "char" := CASE p_on_delete WHEN 'CASCADE' THEN 'c' WHEN 'SET NULL' THEN 'n' WHEN 'RESTRICT' THEN 'r' ELSE 'a' END; + v_nullable BOOLEAN; + v_changeset TEXT := 'qbit-webhooks:webhooks-foreign-keys-upgrade.sql'; + v_old RECORD; +BEGIN + IF EXISTS (SELECT 1 FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1] + WHERE c.contype = 'f' AND c.conrelid = p_table::regclass AND cardinality(c.conkey) = 1 AND a.attname = p_column + AND c.confrelid = p_parent::regclass AND c.confdeltype = v_rule) THEN + RETURN; + END IF; + + FOR v_old IN SELECT c.conname FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1] + WHERE c.contype = 'f' AND c.conrelid = p_table::regclass AND cardinality(c.conkey) = 1 AND a.attname = p_column LOOP + EXECUTE format('ALTER TABLE %I DROP CONSTRAINT %I', p_table, v_old.conname); + END LOOP; + + SELECT is_nullable = 'YES' INTO v_nullable FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = p_table AND column_name = p_column; + + IF v_nullable THEN + EXECUTE format('INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_id, action, changeset) ' + || 'SELECT %L, %L, t.id::TEXT, t.%I::TEXT, ''NULLED'', %L FROM %I t WHERE t.%I IS NOT NULL AND NOT EXISTS (SELECT 1 FROM %I p WHERE p.id = t.%I)', + p_table, p_column, p_column, v_changeset, p_table, p_column, p_parent, p_column); + EXECUTE format('UPDATE %I t SET %I = NULL WHERE t.%I IS NOT NULL AND NOT EXISTS (SELECT 1 FROM %I p WHERE p.id = t.%I)', + p_table, p_column, p_column, p_parent, p_column); + ELSE + EXECUTE format('INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_id, action, row_data, changeset) ' + || 'SELECT %L, %L, t.id::TEXT, t.%I::TEXT, ''DELETED'', row_to_json(t)::TEXT, %L FROM %I t WHERE NOT EXISTS (SELECT 1 FROM %I p WHERE p.id = t.%I)', + p_table, p_column, p_column, v_changeset, p_table, p_parent, p_column); + EXECUTE format('DELETE FROM %I t WHERE NOT EXISTS (SELECT 1 FROM %I p WHERE p.id = t.%I)', p_table, p_parent, p_column); + END IF; + + EXECUTE format('ALTER TABLE %I ADD CONSTRAINT %I FOREIGN KEY (%I) REFERENCES %I (id) ON DELETE %s', p_table, v_name, p_column, p_parent, p_on_delete); +END +$fn$; + +SELECT pg_temp.qqq_add_fk('webhooks_endpoint_secret', 'endpoint_id', 'webhooks_endpoint', 'CASCADE'); +SELECT pg_temp.qqq_add_fk('webhooks_subscription', 'endpoint_id', 'webhooks_endpoint', 'CASCADE'); +SELECT pg_temp.qqq_add_fk('webhooks_delivery', 'endpoint_id', 'webhooks_endpoint', 'RESTRICT'); +SELECT pg_temp.qqq_add_fk('webhooks_delivery', 'subscription_id', 'webhooks_subscription', 'SET NULL'); +SELECT pg_temp.qqq_add_fk('webhooks_delivery_attempt', 'delivery_id', 'webhooks_delivery', 'CASCADE'); +SELECT pg_temp.qqq_add_fk('webhooks_delivery_attempt', 'endpoint_id', 'webhooks_endpoint', 'RESTRICT'); + +CREATE INDEX IF NOT EXISTS webhooks_delivery_subscription_ix ON webhooks_delivery (subscription_id); + +COMMIT; diff --git a/docs/ddl/webhooks-postgres.sql b/docs/ddl/webhooks-postgres.sql index b9512e0..ae3af61 100644 --- a/docs/ddl/webhooks-postgres.sql +++ b/docs/ddl/webhooks-postgres.sql @@ -7,6 +7,18 @@ -- an event being queued twice, and (delivery_id, attempt_number) on -- webhooks_delivery_attempt is the claim that stops two replicas sending the -- same attempt. +-- +-- Foreign keys (named fk_
_): +-- endpoint_secret.endpoint_id, subscription.endpoint_id -> endpoint CASCADE: +-- an endpoint's secrets and subscriptions are its own configuration, so +-- deleting the endpoint (webhooks.admin) removes them. +-- delivery.endpoint_id, delivery_attempt.endpoint_id -> endpoint RESTRICT: +-- delivery history keeps its endpoint; cancel and Delete Deliveries first. +-- delivery.subscription_id -> subscription SET NULL: +-- a delivery outlives the subscription that queued it. +-- delivery_attempt.delivery_id -> delivery CASCADE: +-- attempts belong to their delivery (Retention deletes both). +-- Existing databases: run webhooks-foreign-keys-upgrade.sql. CREATE TABLE webhooks_endpoint ( @@ -40,7 +52,7 @@ CREATE TABLE webhooks_endpoint_secret id SERIAL PRIMARY KEY, create_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), modify_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), - endpoint_id INTEGER NOT NULL REFERENCES webhooks_endpoint (id), + endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_endpoint_secret_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE CASCADE, status VARCHAR(20) DEFAULT 'ACTIVE', source VARCHAR(20) NOT NULL, secret_ref VARCHAR(100), @@ -59,7 +71,7 @@ CREATE TABLE webhooks_subscription id SERIAL PRIMARY KEY, create_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), modify_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), - endpoint_id INTEGER NOT NULL REFERENCES webhooks_endpoint (id), + endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_subscription_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE CASCADE, event_type_name VARCHAR(200) NOT NULL, active_status VARCHAR(20) DEFAULT 'ACTIVE', description VARCHAR(250), @@ -83,8 +95,8 @@ CREATE TABLE webhooks_delivery modify_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), event_id VARCHAR(40) NOT NULL, event_type_name VARCHAR(200) NOT NULL, - endpoint_id INTEGER NOT NULL, - subscription_id INTEGER, + endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE RESTRICT, + subscription_id INTEGER CONSTRAINT fk_webhooks_delivery_subscription_id REFERENCES webhooks_subscription (id) ON DELETE SET NULL, source VARCHAR(20), source_table VARCHAR(100), source_record_id VARCHAR(100), @@ -113,14 +125,16 @@ CREATE INDEX webhooks_delivery_status_ix ON webhooks_delivery (status, modify_da CREATE INDEX webhooks_delivery_delivered_ix ON webhooks_delivery (delivered_date); +CREATE INDEX webhooks_delivery_subscription_ix ON webhooks_delivery (subscription_id); + CREATE TABLE webhooks_delivery_attempt ( id SERIAL PRIMARY KEY, create_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), modify_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), - delivery_id INTEGER NOT NULL, - endpoint_id INTEGER NOT NULL, + delivery_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_attempt_delivery_id REFERENCES webhooks_delivery (id) ON DELETE CASCADE, + endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_attempt_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE RESTRICT, attempt_number INTEGER NOT NULL, outcome VARCHAR(20) NOT NULL, claimed_by VARCHAR(100), diff --git a/pom.xml b/pom.xml index dc0bdb1..76a7eae 100644 --- a/pom.xml +++ b/pom.xml @@ -115,6 +115,19 @@ ${h2.version} test + + + org.postgresql + postgresql + 42.7.9 + test + + + org.testcontainers + testcontainers-postgresql + 2.0.3 + test + org.junit.jupiter junit-jupiter diff --git a/src/test/java/com/kingsrook/qbits/webhooks/ForeignKeyGuardrailTest.java b/src/test/java/com/kingsrook/qbits/webhooks/ForeignKeyGuardrailTest.java new file mode 100644 index 0000000..71f2e52 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/webhooks/ForeignKeyGuardrailTest.java @@ -0,0 +1,431 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.webhooks; + + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.postgresql.PostgreSQLContainer; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + + +/******************************************************************************* + ** Referential integrity guardrail: docs/ddl/webhooks-postgres.sql, applied to + ** a real PostgreSQL, has a foreign key for every join and every table + ** possible-value field between the qbit's tables (or the relation is in + ** {@link #EXEMPT} with a reason), each with the intended ON DELETE. A new + ** table or join without its key fails here. The upgrade script is checked + ** too: it quarantines orphans, adds the same keys, and is idempotent. + *******************************************************************************/ +class ForeignKeyGuardrailTest +{ + private static final Path DDL = Path.of("docs/ddl/webhooks-postgres.sql"); + private static final Path UPGRADE = Path.of("docs/ddl/webhooks-foreign-keys-upgrade.sql"); + + ///////////////////////////////////////////////////////////////////////////// + // child_table.child_column -> parent_table : ON DELETE (pg confdeltype) // + // c = CASCADE, n = SET NULL, r = RESTRICT // + ///////////////////////////////////////////////////////////////////////////// + private static final Map EXPECTED = new TreeMap<>(Map.of( + "webhooks_endpoint_secret.endpoint_id -> webhooks_endpoint", "c", + "webhooks_subscription.endpoint_id -> webhooks_endpoint", "c", + "webhooks_delivery.endpoint_id -> webhooks_endpoint", "r", + "webhooks_delivery.subscription_id -> webhooks_subscription", "n", + "webhooks_delivery_attempt.delivery_id -> webhooks_delivery", "c", + "webhooks_delivery_attempt.endpoint_id -> webhooks_endpoint", "r")); + + ///////////////////////////////////////////////////////////////////////////// + // relations that are deliberately not foreign keys, with the reason // + ///////////////////////////////////////////////////////////////////////////// + private static final Map EXEMPT = Map.of(); + + private static PostgreSQLContainer postgres; + + + + /******************************************************************************* + ** Start PostgreSQL once (a missing Docker fails in CI, skips locally). + *******************************************************************************/ + @BeforeAll + static void startPostgres() + { + boolean docker; + try + { + docker = DockerClientFactory.instance().isDockerAvailable(); + } + catch(RuntimeException e) + { + docker = false; + } + if(!docker && System.getenv("CI") != null) + { + Assertions.fail("Docker is required for the foreign key guardrail in CI"); + } + Assumptions.assumeTrue(docker, "Docker is not available"); + postgres = new PostgreSQLContainer("postgres:17-alpine").withDatabaseName("webhooks").withUsername("qqq").withPassword("qqq"); + postgres.start(); + } + + + + /******************************************************************************* + ** Stop PostgreSQL. + *******************************************************************************/ + @AfterAll + static void stopPostgres() + { + if(postgres != null) + { + postgres.stop(); + } + } + + + + /******************************************************************************* + ** A clean schema from the reference DDL. + *******************************************************************************/ + @BeforeEach + void setUp() throws Exception + { + try(Connection c = connect(); Statement st = c.createStatement()) + { + st.execute("DROP SCHEMA public CASCADE"); + st.execute("CREATE SCHEMA public"); + st.execute(Files.readString(DDL, StandardCharsets.UTF_8)); + } + } + + + + /******************************************************************************* + ** Clear the QContext the metadata setup opened. + *******************************************************************************/ + @AfterEach + void tearDown() + { + QContext.clear(); + } + + + + /******************************************************************************* + ** Every declared relation is a foreign key (or exempt), with the intended + ** ON DELETE, and every foreign key column leads an index. + *******************************************************************************/ + @Test + void everyRelationHasAForeignKey() throws Exception + { + TreeSet declared = declaredRelations(WebhooksTestSupport.setUp()); + Map keys = foreignKeys(); + + List missing = new ArrayList<>(); + for(String relation : declared) + { + if(!keys.containsKey(relation) && !EXEMPT.containsKey(relation)) + { + missing.add(relation); + } + } + assertThat(missing).as("relations declared in metadata without a foreign key in " + DDL + " (add one, or exempt it with a reason)").isEmpty(); + assertThat(declared).as("every expected relation is still declared").containsAll(EXPECTED.keySet()); + assertThat(keys).as("foreign keys and their ON DELETE").isEqualTo(EXPECTED); + assertThat(unindexedForeignKeys()).as("foreign key columns that lead no index").isEmpty(); + } + + + + /******************************************************************************* + ** The upgrade script adds the same keys to a schema without them, recording + ** and removing orphans first, and is a no-op the second time. + *******************************************************************************/ + @Test + void upgradeQuarantinesOrphansAndIsIdempotent() throws Exception + { + try(Connection c = connect(); Statement st = c.createStatement()) + { + dropForeignKeys(st); + st.execute(""" + INSERT INTO webhooks_endpoint (id, name, url) VALUES (1, 'one', 'https://one.example'); + INSERT INTO webhooks_subscription (id, endpoint_id, event_type_name) VALUES (1, 1, 'person.inserted'); + INSERT INTO webhooks_delivery (id, event_id, event_type_name, endpoint_id, subscription_id, dedup_key, status) + VALUES (1, 'e1', 'person.inserted', 1, 1, 'd1', 'PENDING'), + (2, 'e2', 'person.inserted', 1, 99, 'd2', 'PENDING'), + (3, 'e3', 'person.inserted', 98, NULL, 'd3', 'PENDING'); + INSERT INTO webhooks_delivery_attempt (id, delivery_id, endpoint_id, attempt_number, outcome) + VALUES (1, 1, 1, 1, 'SUCCESS'), (2, 3, 1, 1, 'FAILURE'); + """); + + st.execute(Files.readString(UPGRADE, StandardCharsets.UTF_8)); + assertThat(foreignKeys()).isEqualTo(EXPECTED); + assertThat(quarantine(st)).containsExactlyInAnyOrder( + "webhooks_delivery.subscription_id row 2 parent 99 NULLED", + "webhooks_delivery.endpoint_id row 3 parent 98 DELETED", + "webhooks_delivery_attempt.delivery_id row 2 parent 3 DELETED"); + assertThat(count(st, "SELECT COUNT(*) FROM webhooks_delivery")).isEqualTo(2); + assertThat(count(st, "SELECT COUNT(*) FROM webhooks_delivery WHERE id = 2 AND subscription_id IS NULL")).isEqualTo(1); + + st.execute(Files.readString(UPGRADE, StandardCharsets.UTF_8)); + assertThat(foreignKeys()).isEqualTo(EXPECTED); + assertThat(quarantine(st)).hasSize(3); + } + } + + + + /******************************************************************************* + ** Deleting an endpoint takes its secrets and subscriptions with it, and is + ** refused while it has delivery history. + *******************************************************************************/ + @Test + void endpointDeleteCascadesConfigurationButNotHistory() throws Exception + { + try(Connection c = connect(); Statement st = c.createStatement()) + { + st.execute(""" + INSERT INTO webhooks_endpoint (id, name, url) VALUES (1, 'one', 'https://one.example'), (2, 'two', 'https://two.example'); + INSERT INTO webhooks_endpoint_secret (endpoint_id, source) VALUES (1, 'GENERATED'), (2, 'GENERATED'); + INSERT INTO webhooks_subscription (id, endpoint_id, event_type_name) VALUES (1, 1, 'person.inserted'), (2, 2, 'person.inserted'); + INSERT INTO webhooks_delivery (id, event_id, event_type_name, endpoint_id, subscription_id, dedup_key, status) + VALUES (1, 'e1', 'person.inserted', 2, 2, 'd1', 'DELIVERED'); + INSERT INTO webhooks_delivery_attempt (delivery_id, endpoint_id, attempt_number, outcome) VALUES (1, 2, 1, 'SUCCESS'); + """); + + st.execute("DELETE FROM webhooks_endpoint WHERE id = 1"); + assertThat(count(st, "SELECT COUNT(*) FROM webhooks_endpoint_secret WHERE endpoint_id = 1")).isZero(); + assertThat(count(st, "SELECT COUNT(*) FROM webhooks_subscription WHERE endpoint_id = 1")).isZero(); + + assertThatThrownBy(() -> st.execute("DELETE FROM webhooks_endpoint WHERE id = 2")).isInstanceOf(SQLException.class).hasMessageContaining("fk_webhooks_delivery"); + + st.execute("DELETE FROM webhooks_subscription WHERE id = 2"); + assertThat(count(st, "SELECT COUNT(*) FROM webhooks_delivery WHERE id = 1 AND subscription_id IS NULL")).isEqualTo(1); + + st.execute("DELETE FROM webhooks_delivery WHERE id = 1"); + assertThat(count(st, "SELECT COUNT(*) FROM webhooks_delivery_attempt")).isZero(); + st.execute("DELETE FROM webhooks_endpoint WHERE id = 2"); + } + } + + + + /******************************************************************************* + ** child_table.child_column -> parent_table for every join and table PVS + ** field between two of the qbit's tables. + *******************************************************************************/ + static TreeSet declaredRelations(QInstance instance) + { + TreeSet relations = new TreeSet<>(); + for(QTableMetaData table : instance.getTables().values()) + { + if(!isQbitTable(table)) + { + continue; + } + for(QFieldMetaData field : table.getFields().values()) + { + QPossibleValueSource pvs = field.getPossibleValueSourceName() == null ? null : instance.getPossibleValueSource(field.getPossibleValueSourceName()); + if(pvs != null && pvs.getType() == QPossibleValueSourceType.TABLE && isQbitTable(instance.getTable(pvs.getTableName()))) + { + relations.add(relation(table, field, instance.getTable(pvs.getTableName()))); + } + } + } + for(QJoinMetaData join : instance.getJoins().values()) + { + QTableMetaData left = instance.getTable(join.getLeftTable()); + QTableMetaData right = instance.getTable(join.getRightTable()); + if(!isQbitTable(left) || !isQbitTable(right)) + { + continue; + } + for(JoinOn on : join.getJoinOns()) + { + if(on.getLeftField().equals(left.getPrimaryKeyField())) + { + relations.add(relation(right, right.getField(on.getRightField()), left)); + } + else if(on.getRightField().equals(right.getPrimaryKeyField())) + { + relations.add(relation(left, left.getField(on.getLeftField()), right)); + } + } + } + return (relations); + } + + + + /******************************************************************************* + ** Whether a table is one of the qbit's. + *******************************************************************************/ + private static boolean isQbitTable(QTableMetaData table) + { + return (table != null && table.getName().startsWith("webhooks_")); + } + + + + /******************************************************************************* + ** A relation key. + *******************************************************************************/ + private static String relation(QTableMetaData child, QFieldMetaData field, QTableMetaData parent) + { + String column = field.getBackendName() != null ? field.getBackendName() : field.getName(); + return (child.getName() + "." + column + " -> " + parent.getName()); + } + + + + /******************************************************************************* + ** The single-column foreign keys in the database, with their ON DELETE. + *******************************************************************************/ + private static Map foreignKeys() throws SQLException + { + Map keys = new TreeMap<>(); + try(Connection c = connect(); Statement st = c.createStatement(); ResultSet rs = st.executeQuery(""" + SELECT c.conrelid::regclass::text AS child, a.attname AS col, c.confrelid::regclass::text AS parent, c.confdeltype::text AS rule + FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1] + WHERE c.contype = 'f' AND cardinality(c.conkey) = 1 + """)) + { + while(rs.next()) + { + keys.put(rs.getString("child") + "." + rs.getString("col") + " -> " + rs.getString("parent"), rs.getString("rule")); + } + } + return (keys); + } + + + + /******************************************************************************* + ** Foreign key columns that are not the first column of any index. + *******************************************************************************/ + private static List unindexedForeignKeys() throws SQLException + { + List list = new ArrayList<>(); + try(Connection c = connect(); Statement st = c.createStatement(); ResultSet rs = st.executeQuery(""" + SELECT c.conrelid::regclass::text || '.' || a.attname AS col + FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1] + WHERE c.contype = 'f' + AND NOT EXISTS (SELECT 1 FROM pg_index i WHERE i.indrelid = c.conrelid AND i.indkey[0] = c.conkey[1]) + """)) + { + while(rs.next()) + { + list.add(rs.getString("col")); + } + } + return (list); + } + + + + /******************************************************************************* + ** Drop every foreign key (a schema as it was before they were declared). + *******************************************************************************/ + private static void dropForeignKeys(Statement st) throws SQLException + { + List drops = new ArrayList<>(); + try(ResultSet rs = st.executeQuery("SELECT conrelid::regclass::text AS t, conname FROM pg_constraint WHERE contype = 'f'")) + { + while(rs.next()) + { + drops.add("ALTER TABLE " + rs.getString("t") + " DROP CONSTRAINT " + rs.getString("conname")); + } + } + for(String drop : drops) + { + st.execute(drop); + } + } + + + + /******************************************************************************* + ** The quarantine rows, as text. + *******************************************************************************/ + private static List quarantine(Statement st) throws SQLException + { + List rows = new ArrayList<>(); + try(ResultSet rs = st.executeQuery("SELECT table_name, column_name, row_id, missing_parent_id, action FROM qqq_fk_quarantine ORDER BY id")) + { + while(rs.next()) + { + rows.add(rs.getString(1) + "." + rs.getString(2) + " row " + rs.getString(3) + " parent " + rs.getString(4) + " " + rs.getString(5)); + } + } + return (rows); + } + + + + /******************************************************************************* + ** A count. + *******************************************************************************/ + private static int count(Statement st, String sql) throws SQLException + { + try(ResultSet rs = st.executeQuery(sql)) + { + rs.next(); + return (rs.getInt(1)); + } + } + + + + /******************************************************************************* + ** A connection. + *******************************************************************************/ + private static Connection connect() throws SQLException + { + return (DriverManager.getConnection(postgres.getJdbcUrl(), "qqq", "qqq")); + } +} diff --git a/src/test/resources/webhooks-h2.sql b/src/test/resources/webhooks-h2.sql index d989b10..f714403 100644 --- a/src/test/resources/webhooks-h2.sql +++ b/src/test/resources/webhooks-h2.sql @@ -49,7 +49,7 @@ CREATE TABLE webhooks_endpoint_secret id INT AUTO_INCREMENT PRIMARY KEY, create_date TIMESTAMP NOT NULL DEFAULT now(), modify_date TIMESTAMP NOT NULL DEFAULT now(), - endpoint_id INTEGER NOT NULL REFERENCES webhooks_endpoint (id), + endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_endpoint_secret_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE CASCADE, status VARCHAR(20) DEFAULT 'ACTIVE', source VARCHAR(20) NOT NULL, secret_ref VARCHAR(100), @@ -68,7 +68,7 @@ CREATE TABLE webhooks_subscription id INT AUTO_INCREMENT PRIMARY KEY, create_date TIMESTAMP NOT NULL DEFAULT now(), modify_date TIMESTAMP NOT NULL DEFAULT now(), - endpoint_id INTEGER NOT NULL REFERENCES webhooks_endpoint (id), + endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_subscription_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE CASCADE, event_type_name VARCHAR(200) NOT NULL, active_status VARCHAR(20) DEFAULT 'ACTIVE', description VARCHAR(250), @@ -92,8 +92,8 @@ CREATE TABLE webhooks_delivery modify_date TIMESTAMP NOT NULL DEFAULT now(), event_id VARCHAR(40) NOT NULL, event_type_name VARCHAR(200) NOT NULL, - endpoint_id INTEGER NOT NULL, - subscription_id INTEGER, + endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE RESTRICT, + subscription_id INTEGER CONSTRAINT fk_webhooks_delivery_subscription_id REFERENCES webhooks_subscription (id) ON DELETE SET NULL, source VARCHAR(20), source_table VARCHAR(100), source_record_id VARCHAR(100), @@ -122,14 +122,16 @@ CREATE INDEX webhooks_delivery_status_ix ON webhooks_delivery (status, modify_da CREATE INDEX webhooks_delivery_delivered_ix ON webhooks_delivery (delivered_date); +CREATE INDEX webhooks_delivery_subscription_ix ON webhooks_delivery (subscription_id); + CREATE TABLE webhooks_delivery_attempt ( id INT AUTO_INCREMENT PRIMARY KEY, create_date TIMESTAMP NOT NULL DEFAULT now(), modify_date TIMESTAMP NOT NULL DEFAULT now(), - delivery_id INTEGER NOT NULL, - endpoint_id INTEGER NOT NULL, + delivery_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_attempt_delivery_id REFERENCES webhooks_delivery (id) ON DELETE CASCADE, + endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_attempt_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE RESTRICT, attempt_number INTEGER NOT NULL, outcome VARCHAR(20) NOT NULL, claimed_by VARCHAR(100),