CHANGELOG.md

@@ -6,6 +6,9 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and
 ## [Unreleased]
 
 ### Added
+- Foreign keys between all five tables in `docs/ddl/webhooks-postgres.sql`, named `fk_<table>_<column>`: an endpoint's secrets and subscriptions are deleted with it (CASCADE), deliveries and attempts keep their endpoint (RESTRICT: cancel and Delete Deliveries before deleting an endpoint with history), a delivery outlives its subscription (SET NULL) and attempts go with their delivery (CASCADE). New index `webhooks_delivery_subscription_ix`.
+- `docs/ddl/webhooks-foreign-keys-upgrade.sql` for existing databases: records orphan rows in `qqq_fk_quarantine` (nulls nullable references, deletes rows with a missing required parent and keeps them as JSON), replaces the NO ACTION endpoint keys, and is idempotent.
+- `ForeignKeyGuardrailTest` applies the DDL to PostgreSQL 17 (Testcontainers) and fails when a join or table possible-value field between webhooks tables has no foreign key.
 - Every table explains itself: a help bundle (`help/qbit-webhooks.yaml`, QQQ help-bundle format) says what each `webhooks_*` table is for and how its rows get created, including where event types come from. It shows on the table screen and when the table is empty. Applied by `WebhooksHelp` (STAND-IN for QRun-IO/qqq#1198); a host's own help is kept.
 - A Setup card on the Webhooks home lists event types, encryption key, endpoints, signing secrets and subscriptions with their state and the next step.
 - `WebhooksTablesGuardrailTest` (each table is used at runtime, seeded or explained, and has help) and a test that a previous secret stops signing at its Signs Until date even before the maintenance job retires it.

docs/ddl/webhooks-foreign-keys-upgrade.sql

@@ -0,0 +1,68 @@
+-- qbit-webhooks: add the foreign keys to an existing PostgreSQL schema
+-- (one created from webhooks-postgres.sql before they were declared there).
+--
+-- Safe to run more than once: a foreign key that already exists with the
+-- right ON DELETE (under any name) is left alone. Before each key is added,
+-- rows that point at a missing parent are recorded in qqq_fk_quarantine and
+-- then set to NULL (nullable column) or deleted (NOT NULL column; the whole
+-- row is kept in row_data). An existing key on the column with a different
+-- ON DELETE (endpoint_secret and subscription -> endpoint were NO ACTION) is
+-- replaced by the named one.
+--
+-- Order matters: deliveries of a missing endpoint are removed before their
+-- attempts are checked.
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS qqq_fk_quarantine (id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, table_name VARCHAR(100) NOT NULL, column_name VARCHAR(100) NOT NULL, row_id VARCHAR(100), missing_parent_id VARCHAR(100), action VARCHAR(20) NOT NULL, row_data TEXT, changeset VARCHAR(200), quarantined_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP);
+
+CREATE OR REPLACE FUNCTION pg_temp.qqq_add_fk(p_table TEXT, p_column TEXT, p_parent TEXT, p_on_delete TEXT) RETURNS VOID LANGUAGE plpgsql AS
+$fn$
+DECLARE
+   v_name      TEXT    := 'fk_' || p_table || '_' || p_column;
+   v_rule      "char"  := CASE p_on_delete WHEN 'CASCADE' THEN 'c' WHEN 'SET NULL' THEN 'n' WHEN 'RESTRICT' THEN 'r' ELSE 'a' END;
+   v_nullable  BOOLEAN;
+   v_changeset TEXT    := 'qbit-webhooks:webhooks-foreign-keys-upgrade.sql';
+   v_old       RECORD;
+BEGIN
+   IF EXISTS (SELECT 1 FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1]
+              WHERE c.contype = 'f' AND c.conrelid = p_table::regclass AND cardinality(c.conkey) = 1 AND a.attname = p_column
+                AND c.confrelid = p_parent::regclass AND c.confdeltype = v_rule) THEN
+      RETURN;
+   END IF;
+
+   FOR v_old IN SELECT c.conname FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1]
+                WHERE c.contype = 'f' AND c.conrelid = p_table::regclass AND cardinality(c.conkey) = 1 AND a.attname = p_column LOOP
+      EXECUTE format('ALTER TABLE %I DROP CONSTRAINT %I', p_table, v_old.conname);
+   END LOOP;
+
+   SELECT is_nullable = 'YES' INTO v_nullable FROM information_schema.columns
+   WHERE table_schema = current_schema() AND table_name = p_table AND column_name = p_column;
+
+   IF v_nullable THEN
+      EXECUTE format('INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_id, action, changeset) '
+         || 'SELECT %L, %L, t.id::TEXT, t.%I::TEXT, ''NULLED'', %L FROM %I t WHERE t.%I IS NOT NULL AND NOT EXISTS (SELECT 1 FROM %I p WHERE p.id = t.%I)',
+         p_table, p_column, p_column, v_changeset, p_table, p_column, p_parent, p_column);
+      EXECUTE format('UPDATE %I t SET %I = NULL WHERE t.%I IS NOT NULL AND NOT EXISTS (SELECT 1 FROM %I p WHERE p.id = t.%I)',
+         p_table, p_column, p_column, p_parent, p_column);
+   ELSE
+      EXECUTE format('INSERT INTO qqq_fk_quarantine (table_name, column_name, row_id, missing_parent_id, action, row_data, changeset) '
+         || 'SELECT %L, %L, t.id::TEXT, t.%I::TEXT, ''DELETED'', row_to_json(t)::TEXT, %L FROM %I t WHERE NOT EXISTS (SELECT 1 FROM %I p WHERE p.id = t.%I)',
+         p_table, p_column, p_column, v_changeset, p_table, p_parent, p_column);
+      EXECUTE format('DELETE FROM %I t WHERE NOT EXISTS (SELECT 1 FROM %I p WHERE p.id = t.%I)', p_table, p_parent, p_column);
+   END IF;
+
+   EXECUTE format('ALTER TABLE %I ADD CONSTRAINT %I FOREIGN KEY (%I) REFERENCES %I (id) ON DELETE %s', p_table, v_name, p_column, p_parent, p_on_delete);
+END
+$fn$;
+
+SELECT pg_temp.qqq_add_fk('webhooks_endpoint_secret', 'endpoint_id', 'webhooks_endpoint', 'CASCADE');
+SELECT pg_temp.qqq_add_fk('webhooks_subscription', 'endpoint_id', 'webhooks_endpoint', 'CASCADE');
+SELECT pg_temp.qqq_add_fk('webhooks_delivery', 'endpoint_id', 'webhooks_endpoint', 'RESTRICT');
+SELECT pg_temp.qqq_add_fk('webhooks_delivery', 'subscription_id', 'webhooks_subscription', 'SET NULL');
+SELECT pg_temp.qqq_add_fk('webhooks_delivery_attempt', 'delivery_id', 'webhooks_delivery', 'CASCADE');
+SELECT pg_temp.qqq_add_fk('webhooks_delivery_attempt', 'endpoint_id', 'webhooks_endpoint', 'RESTRICT');
+
+CREATE INDEX IF NOT EXISTS webhooks_delivery_subscription_ix ON webhooks_delivery (subscription_id);
+
+COMMIT;

docs/ddl/webhooks-postgres.sql

@@ -7,6 +7,18 @@
 -- an event being queued twice, and (delivery_id, attempt_number) on
 -- webhooks_delivery_attempt is the claim that stops two replicas sending the
 -- same attempt.
+--
+-- Foreign keys (named fk_<table>_<column>):
+--   endpoint_secret.endpoint_id, subscription.endpoint_id -> endpoint  CASCADE:
+--     an endpoint's secrets and subscriptions are its own configuration, so
+--     deleting the endpoint (webhooks.admin) removes them.
+--   delivery.endpoint_id, delivery_attempt.endpoint_id -> endpoint      RESTRICT:
+--     delivery history keeps its endpoint; cancel and Delete Deliveries first.
+--   delivery.subscription_id -> subscription                           SET NULL:
+--     a delivery outlives the subscription that queued it.
+--   delivery_attempt.delivery_id -> delivery                           CASCADE:
+--     attempts belong to their delivery (Retention deletes both).
+-- Existing databases: run webhooks-foreign-keys-upgrade.sql.
 
 CREATE TABLE webhooks_endpoint
 (
@@ -40,7 +52,7 @@ CREATE TABLE webhooks_endpoint_secret
    id SERIAL PRIMARY KEY,
    create_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
    modify_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
-   endpoint_id INTEGER NOT NULL REFERENCES webhooks_endpoint (id),
+   endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_endpoint_secret_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE CASCADE,
    status VARCHAR(20) DEFAULT 'ACTIVE',
    source VARCHAR(20) NOT NULL,
    secret_ref VARCHAR(100),
@@ -59,7 +71,7 @@ CREATE TABLE webhooks_subscription
    id SERIAL PRIMARY KEY,
    create_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
    modify_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
-   endpoint_id INTEGER NOT NULL REFERENCES webhooks_endpoint (id),
+   endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_subscription_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE CASCADE,
    event_type_name VARCHAR(200) NOT NULL,
    active_status VARCHAR(20) DEFAULT 'ACTIVE',
    description VARCHAR(250),
@@ -83,8 +95,8 @@ CREATE TABLE webhooks_delivery
    modify_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
    event_id VARCHAR(40) NOT NULL,
    event_type_name VARCHAR(200) NOT NULL,
-   endpoint_id INTEGER NOT NULL,
-   subscription_id INTEGER,
+   endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE RESTRICT,
+   subscription_id INTEGER CONSTRAINT fk_webhooks_delivery_subscription_id REFERENCES webhooks_subscription (id) ON DELETE SET NULL,
    source VARCHAR(20),
    source_table VARCHAR(100),
    source_record_id VARCHAR(100),
@@ -113,14 +125,16 @@ CREATE INDEX webhooks_delivery_status_ix ON webhooks_delivery (status, modify_da
 
 CREATE INDEX webhooks_delivery_delivered_ix ON webhooks_delivery (delivered_date);
 
+CREATE INDEX webhooks_delivery_subscription_ix ON webhooks_delivery (subscription_id);
+
 
 CREATE TABLE webhooks_delivery_attempt
 (
    id SERIAL PRIMARY KEY,
    create_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
    modify_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
-   delivery_id INTEGER NOT NULL,
-   endpoint_id INTEGER NOT NULL,
+   delivery_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_attempt_delivery_id REFERENCES webhooks_delivery (id) ON DELETE CASCADE,
+   endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_attempt_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE RESTRICT,
    attempt_number INTEGER NOT NULL,
    outcome VARCHAR(20) NOT NULL,
    claimed_by VARCHAR(100),

pom.xml

@@ -115,6 +115,19 @@
          <version>${h2.version}</version>
          <scope>test</scope>
       </dependency>
+      <!-- the foreign key guardrail test: docs/ddl on a real PostgreSQL (src/test only) -->
+      <dependency>
+         <groupId>org.postgresql</groupId>
+         <artifactId>postgresql</artifactId>
+         <version>42.7.9</version>
+         <scope>test</scope>
+      </dependency>
+      <dependency>
+         <groupId>org.testcontainers</groupId>
+         <artifactId>testcontainers-postgresql</artifactId>
+         <version>2.0.3</version>
+         <scope>test</scope>
+      </dependency>
       <dependency>
          <groupId>org.junit.jupiter</groupId>
          <artifactId>junit-jupiter</artifactId>

src/test/java/com/kingsrook/qbits/webhooks/ForeignKeyGuardrailTest.java

@@ -0,0 +1,431 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.webhooks;
+
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn;
+import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource;
+import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType;
+import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.DockerClientFactory;
+import org.testcontainers.postgresql.PostgreSQLContainer;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+
+/*******************************************************************************
+ ** Referential integrity guardrail: docs/ddl/webhooks-postgres.sql, applied to
+ ** a real PostgreSQL, has a foreign key for every join and every table
+ ** possible-value field between the qbit's tables (or the relation is in
+ ** {@link #EXEMPT} with a reason), each with the intended ON DELETE. A new
+ ** table or join without its key fails here. The upgrade script is checked
+ ** too: it quarantines orphans, adds the same keys, and is idempotent.
+ *******************************************************************************/
+class ForeignKeyGuardrailTest
+{
+   private static final Path DDL     = Path.of("docs/ddl/webhooks-postgres.sql");
+   private static final Path UPGRADE = Path.of("docs/ddl/webhooks-foreign-keys-upgrade.sql");
+
+   /////////////////////////////////////////////////////////////////////////////
+   // child_table.child_column -> parent_table : ON DELETE (pg confdeltype)    //
+   // c = CASCADE, n = SET NULL, r = RESTRICT                                  //
+   /////////////////////////////////////////////////////////////////////////////
+   private static final Map<String, String> EXPECTED = new TreeMap<>(Map.of(
+      "webhooks_endpoint_secret.endpoint_id -> webhooks_endpoint", "c",
+      "webhooks_subscription.endpoint_id -> webhooks_endpoint", "c",
+      "webhooks_delivery.endpoint_id -> webhooks_endpoint", "r",
+      "webhooks_delivery.subscription_id -> webhooks_subscription", "n",
+      "webhooks_delivery_attempt.delivery_id -> webhooks_delivery", "c",
+      "webhooks_delivery_attempt.endpoint_id -> webhooks_endpoint", "r"));
+
+   /////////////////////////////////////////////////////////////////////////////
+   // relations that are deliberately not foreign keys, with the reason       //
+   /////////////////////////////////////////////////////////////////////////////
+   private static final Map<String, String> EXEMPT = Map.of();
+
+   private static PostgreSQLContainer postgres;
+
+
+
+   /*******************************************************************************
+    ** Start PostgreSQL once (a missing Docker fails in CI, skips locally).
+    *******************************************************************************/
+   @BeforeAll
+   static void startPostgres()
+   {
+      boolean docker;
+      try
+      {
+         docker = DockerClientFactory.instance().isDockerAvailable();
+      }
+      catch(RuntimeException e)
+      {
+         docker = false;
+      }
+      if(!docker && System.getenv("CI") != null)
+      {
+         Assertions.fail("Docker is required for the foreign key guardrail in CI");
+      }
+      Assumptions.assumeTrue(docker, "Docker is not available");
+      postgres = new PostgreSQLContainer("postgres:17-alpine").withDatabaseName("webhooks").withUsername("qqq").withPassword("qqq");
+      postgres.start();
+   }
+
+
+
+   /*******************************************************************************
+    ** Stop PostgreSQL.
+    *******************************************************************************/
+   @AfterAll
+   static void stopPostgres()
+   {
+      if(postgres != null)
+      {
+         postgres.stop();
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A clean schema from the reference DDL.
+    *******************************************************************************/
+   @BeforeEach
+   void setUp() throws Exception
+   {
+      try(Connection c = connect(); Statement st = c.createStatement())
+      {
+         st.execute("DROP SCHEMA public CASCADE");
+         st.execute("CREATE SCHEMA public");
+         st.execute(Files.readString(DDL, StandardCharsets.UTF_8));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Clear the QContext the metadata setup opened.
+    *******************************************************************************/
+   @AfterEach
+   void tearDown()
+   {
+      QContext.clear();
+   }
+
+
+
+   /*******************************************************************************
+    ** Every declared relation is a foreign key (or exempt), with the intended
+    ** ON DELETE, and every foreign key column leads an index.
+    *******************************************************************************/
+   @Test
+   void everyRelationHasAForeignKey() throws Exception
+   {
+      TreeSet<String> declared = declaredRelations(WebhooksTestSupport.setUp());
+      Map<String, String> keys = foreignKeys();
+
+      List<String> missing = new ArrayList<>();
+      for(String relation : declared)
+      {
+         if(!keys.containsKey(relation) && !EXEMPT.containsKey(relation))
+         {
+            missing.add(relation);
+         }
+      }
+      assertThat(missing).as("relations declared in metadata without a foreign key in " + DDL + " (add one, or exempt it with a reason)").isEmpty();
+      assertThat(declared).as("every expected relation is still declared").containsAll(EXPECTED.keySet());
+      assertThat(keys).as("foreign keys and their ON DELETE").isEqualTo(EXPECTED);
+      assertThat(unindexedForeignKeys()).as("foreign key columns that lead no index").isEmpty();
+   }
+
+
+
+   /*******************************************************************************
+    ** The upgrade script adds the same keys to a schema without them, recording
+    ** and removing orphans first, and is a no-op the second time.
+    *******************************************************************************/
+   @Test
+   void upgradeQuarantinesOrphansAndIsIdempotent() throws Exception
+   {
+      try(Connection c = connect(); Statement st = c.createStatement())
+      {
+         dropForeignKeys(st);
+         st.execute("""
+            INSERT INTO webhooks_endpoint (id, name, url) VALUES (1, 'one', 'https://one.example');
+            INSERT INTO webhooks_subscription (id, endpoint_id, event_type_name) VALUES (1, 1, 'person.inserted');
+            INSERT INTO webhooks_delivery (id, event_id, event_type_name, endpoint_id, subscription_id, dedup_key, status)
+               VALUES (1, 'e1', 'person.inserted', 1, 1, 'd1', 'PENDING'),
+                      (2, 'e2', 'person.inserted', 1, 99, 'd2', 'PENDING'),
+                      (3, 'e3', 'person.inserted', 98, NULL, 'd3', 'PENDING');
+            INSERT INTO webhooks_delivery_attempt (id, delivery_id, endpoint_id, attempt_number, outcome)
+               VALUES (1, 1, 1, 1, 'SUCCESS'), (2, 3, 1, 1, 'FAILURE');
+            """);
+
+         st.execute(Files.readString(UPGRADE, StandardCharsets.UTF_8));
+         assertThat(foreignKeys()).isEqualTo(EXPECTED);
+         assertThat(quarantine(st)).containsExactlyInAnyOrder(
+            "webhooks_delivery.subscription_id row 2 parent 99 NULLED",
+            "webhooks_delivery.endpoint_id row 3 parent 98 DELETED",
+            "webhooks_delivery_attempt.delivery_id row 2 parent 3 DELETED");
+         assertThat(count(st, "SELECT COUNT(*) FROM webhooks_delivery")).isEqualTo(2);
+         assertThat(count(st, "SELECT COUNT(*) FROM webhooks_delivery WHERE id = 2 AND subscription_id IS NULL")).isEqualTo(1);
+
+         st.execute(Files.readString(UPGRADE, StandardCharsets.UTF_8));
+         assertThat(foreignKeys()).isEqualTo(EXPECTED);
+         assertThat(quarantine(st)).hasSize(3);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** Deleting an endpoint takes its secrets and subscriptions with it, and is
+    ** refused while it has delivery history.
+    *******************************************************************************/
+   @Test
+   void endpointDeleteCascadesConfigurationButNotHistory() throws Exception
+   {
+      try(Connection c = connect(); Statement st = c.createStatement())
+      {
+         st.execute("""
+            INSERT INTO webhooks_endpoint (id, name, url) VALUES (1, 'one', 'https://one.example'), (2, 'two', 'https://two.example');
+            INSERT INTO webhooks_endpoint_secret (endpoint_id, source) VALUES (1, 'GENERATED'), (2, 'GENERATED');
+            INSERT INTO webhooks_subscription (id, endpoint_id, event_type_name) VALUES (1, 1, 'person.inserted'), (2, 2, 'person.inserted');
+            INSERT INTO webhooks_delivery (id, event_id, event_type_name, endpoint_id, subscription_id, dedup_key, status)
+               VALUES (1, 'e1', 'person.inserted', 2, 2, 'd1', 'DELIVERED');
+            INSERT INTO webhooks_delivery_attempt (delivery_id, endpoint_id, attempt_number, outcome) VALUES (1, 2, 1, 'SUCCESS');
+            """);
+
+         st.execute("DELETE FROM webhooks_endpoint WHERE id = 1");
+         assertThat(count(st, "SELECT COUNT(*) FROM webhooks_endpoint_secret WHERE endpoint_id = 1")).isZero();
+         assertThat(count(st, "SELECT COUNT(*) FROM webhooks_subscription WHERE endpoint_id = 1")).isZero();
+
+         assertThatThrownBy(() -> st.execute("DELETE FROM webhooks_endpoint WHERE id = 2")).isInstanceOf(SQLException.class).hasMessageContaining("fk_webhooks_delivery");
+
+         st.execute("DELETE FROM webhooks_subscription WHERE id = 2");
+         assertThat(count(st, "SELECT COUNT(*) FROM webhooks_delivery WHERE id = 1 AND subscription_id IS NULL")).isEqualTo(1);
+
+         st.execute("DELETE FROM webhooks_delivery WHERE id = 1");
+         assertThat(count(st, "SELECT COUNT(*) FROM webhooks_delivery_attempt")).isZero();
+         st.execute("DELETE FROM webhooks_endpoint WHERE id = 2");
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** child_table.child_column -> parent_table for every join and table PVS
+    ** field between two of the qbit's tables.
+    *******************************************************************************/
+   static TreeSet<String> declaredRelations(QInstance instance)
+   {
+      TreeSet<String> relations = new TreeSet<>();
+      for(QTableMetaData table : instance.getTables().values())
+      {
+         if(!isQbitTable(table))
+         {
+            continue;
+         }
+         for(QFieldMetaData field : table.getFields().values())
+         {
+            QPossibleValueSource pvs = field.getPossibleValueSourceName() == null ? null : instance.getPossibleValueSource(field.getPossibleValueSourceName());
+            if(pvs != null && pvs.getType() == QPossibleValueSourceType.TABLE && isQbitTable(instance.getTable(pvs.getTableName())))
+            {
+               relations.add(relation(table, field, instance.getTable(pvs.getTableName())));
+            }
+         }
+      }
+      for(QJoinMetaData join : instance.getJoins().values())
+      {
+         QTableMetaData left  = instance.getTable(join.getLeftTable());
+         QTableMetaData right = instance.getTable(join.getRightTable());
+         if(!isQbitTable(left) || !isQbitTable(right))
+         {
+            continue;
+         }
+         for(JoinOn on : join.getJoinOns())
+         {
+            if(on.getLeftField().equals(left.getPrimaryKeyField()))
+            {
+               relations.add(relation(right, right.getField(on.getRightField()), left));
+            }
+            else if(on.getRightField().equals(right.getPrimaryKeyField()))
+            {
+               relations.add(relation(left, left.getField(on.getLeftField()), right));
+            }
+         }
+      }
+      return (relations);
+   }
+
+
+
+   /*******************************************************************************
+    ** Whether a table is one of the qbit's.
+    *******************************************************************************/
+   private static boolean isQbitTable(QTableMetaData table)
+   {
+      return (table != null && table.getName().startsWith("webhooks_"));
+   }
+
+
+
+   /*******************************************************************************
+    ** A relation key.
+    *******************************************************************************/
+   private static String relation(QTableMetaData child, QFieldMetaData field, QTableMetaData parent)
+   {
+      String column = field.getBackendName() != null ? field.getBackendName() : field.getName();
+      return (child.getName() + "." + column + " -> " + parent.getName());
+   }
+
+
+
+   /*******************************************************************************
+    ** The single-column foreign keys in the database, with their ON DELETE.
+    *******************************************************************************/
+   private static Map<String, String> foreignKeys() throws SQLException
+   {
+      Map<String, String> keys = new TreeMap<>();
+      try(Connection c = connect(); Statement st = c.createStatement(); ResultSet rs = st.executeQuery("""
+         SELECT c.conrelid::regclass::text AS child, a.attname AS col, c.confrelid::regclass::text AS parent, c.confdeltype::text AS rule
+         FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1]
+         WHERE c.contype = 'f' AND cardinality(c.conkey) = 1
+         """))
+      {
+         while(rs.next())
+         {
+            keys.put(rs.getString("child") + "." + rs.getString("col") + " -> " + rs.getString("parent"), rs.getString("rule"));
+         }
+      }
+      return (keys);
+   }
+
+
+
+   /*******************************************************************************
+    ** Foreign key columns that are not the first column of any index.
+    *******************************************************************************/
+   private static List<String> unindexedForeignKeys() throws SQLException
+   {
+      List<String> list = new ArrayList<>();
+      try(Connection c = connect(); Statement st = c.createStatement(); ResultSet rs = st.executeQuery("""
+         SELECT c.conrelid::regclass::text || '.' || a.attname AS col
+         FROM pg_constraint c JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1]
+         WHERE c.contype = 'f'
+           AND NOT EXISTS (SELECT 1 FROM pg_index i WHERE i.indrelid = c.conrelid AND i.indkey[0] = c.conkey[1])
+         """))
+      {
+         while(rs.next())
+         {
+            list.add(rs.getString("col"));
+         }
+      }
+      return (list);
+   }
+
+
+
+   /*******************************************************************************
+    ** Drop every foreign key (a schema as it was before they were declared).
+    *******************************************************************************/
+   private static void dropForeignKeys(Statement st) throws SQLException
+   {
+      List<String> drops = new ArrayList<>();
+      try(ResultSet rs = st.executeQuery("SELECT conrelid::regclass::text AS t, conname FROM pg_constraint WHERE contype = 'f'"))
+      {
+         while(rs.next())
+         {
+            drops.add("ALTER TABLE " + rs.getString("t") + " DROP CONSTRAINT " + rs.getString("conname"));
+         }
+      }
+      for(String drop : drops)
+      {
+         st.execute(drop);
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** The quarantine rows, as text.
+    *******************************************************************************/
+   private static List<String> quarantine(Statement st) throws SQLException
+   {
+      List<String> rows = new ArrayList<>();
+      try(ResultSet rs = st.executeQuery("SELECT table_name, column_name, row_id, missing_parent_id, action FROM qqq_fk_quarantine ORDER BY id"))
+      {
+         while(rs.next())
+         {
+            rows.add(rs.getString(1) + "." + rs.getString(2) + " row " + rs.getString(3) + " parent " + rs.getString(4) + " " + rs.getString(5));
+         }
+      }
+      return (rows);
+   }
+
+
+
+   /*******************************************************************************
+    ** A count.
+    *******************************************************************************/
+   private static int count(Statement st, String sql) throws SQLException
+   {
+      try(ResultSet rs = st.executeQuery(sql))
+      {
+         rs.next();
+         return (rs.getInt(1));
+      }
+   }
+
+
+
+   /*******************************************************************************
+    ** A connection.
+    *******************************************************************************/
+   private static Connection connect() throws SQLException
+   {
+      return (DriverManager.getConnection(postgres.getJdbcUrl(), "qqq", "qqq"));
+   }
+}

src/test/resources/webhooks-h2.sql

@@ -49,7 +49,7 @@ CREATE TABLE webhooks_endpoint_secret
    id INT AUTO_INCREMENT PRIMARY KEY,
    create_date TIMESTAMP NOT NULL DEFAULT now(),
    modify_date TIMESTAMP NOT NULL DEFAULT now(),
-   endpoint_id INTEGER NOT NULL REFERENCES webhooks_endpoint (id),
+   endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_endpoint_secret_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE CASCADE,
    status VARCHAR(20) DEFAULT 'ACTIVE',
    source VARCHAR(20) NOT NULL,
    secret_ref VARCHAR(100),
@@ -68,7 +68,7 @@ CREATE TABLE webhooks_subscription
    id INT AUTO_INCREMENT PRIMARY KEY,
    create_date TIMESTAMP NOT NULL DEFAULT now(),
    modify_date TIMESTAMP NOT NULL DEFAULT now(),
-   endpoint_id INTEGER NOT NULL REFERENCES webhooks_endpoint (id),
+   endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_subscription_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE CASCADE,
    event_type_name VARCHAR(200) NOT NULL,
    active_status VARCHAR(20) DEFAULT 'ACTIVE',
    description VARCHAR(250),
@@ -92,8 +92,8 @@ CREATE TABLE webhooks_delivery
    modify_date TIMESTAMP NOT NULL DEFAULT now(),
    event_id VARCHAR(40) NOT NULL,
    event_type_name VARCHAR(200) NOT NULL,
-   endpoint_id INTEGER NOT NULL,
-   subscription_id INTEGER,
+   endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE RESTRICT,
+   subscription_id INTEGER CONSTRAINT fk_webhooks_delivery_subscription_id REFERENCES webhooks_subscription (id) ON DELETE SET NULL,
    source VARCHAR(20),
    source_table VARCHAR(100),
    source_record_id VARCHAR(100),
@@ -122,14 +122,16 @@ CREATE INDEX webhooks_delivery_status_ix ON webhooks_delivery (status, modify_da
 
 CREATE INDEX webhooks_delivery_delivered_ix ON webhooks_delivery (delivered_date);
 
+CREATE INDEX webhooks_delivery_subscription_ix ON webhooks_delivery (subscription_id);
+
 
 CREATE TABLE webhooks_delivery_attempt
 (
    id INT AUTO_INCREMENT PRIMARY KEY,
    create_date TIMESTAMP NOT NULL DEFAULT now(),
    modify_date TIMESTAMP NOT NULL DEFAULT now(),
-   delivery_id INTEGER NOT NULL,
-   endpoint_id INTEGER NOT NULL,
+   delivery_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_attempt_delivery_id REFERENCES webhooks_delivery (id) ON DELETE CASCADE,
+   endpoint_id INTEGER NOT NULL CONSTRAINT fk_webhooks_delivery_attempt_endpoint_id REFERENCES webhooks_endpoint (id) ON DELETE RESTRICT,
    attempt_number INTEGER NOT NULL,
    outcome VARCHAR(20) NOT NULL,
    claimed_by VARCHAR(100),