fix(store): index message bodies from plaintext at write time + backfill FTS

The FTS index was built over the encrypted body column, so body search never
matched real terms; it was also created lazily with no backfill. Now migrate()
creates the FTS tables eagerly and save_email writes plaintext subject/sender/
body into a standalone messages_fts table in the same transaction (stored
columns stay AES-256-GCM ciphertext). migrate() backfills rows missing from the
index by decrypting with the engine key. Tests exercise the real save_email
path (no raw-SQL bypass) and backfill. Documented tradeoff: the FTS index holds
plaintext-derived trigrams, so body text is recoverable from the index while
search is enabled -- encrypted index / whole-DB encryption is future work.

Refs #152 #153

crates/nuncio-store/src/db.rs

@@ -358,12 +358,115 @@ impl DatabaseEngine {
             CREATE INDEX IF NOT EXISTS idx_filter_logs_rule ON filter_execution_logs(rule_id, matched_at DESC);
             CREATE INDEX IF NOT EXISTS idx_pending_mutations_status ON pending_remote_mutations(status, created_at ASC);
             CREATE INDEX IF NOT EXISTS idx_worm_audit_seq ON worm_audit_records(sequence ASC);
+
+            -- Full-text search indexes are created eagerly at migration time (not lazily on
+            -- first search) so no message or event saved before the first search call is ever
+            -- permanently unindexed.
+            --
+            -- CONFIDENTIALITY TRADEOFF: `messages.body_plain` / `messages.body_html` are
+            -- encrypted at rest (AES-256-GCM, see `PayloadCipher`). `messages_fts` is a
+            -- standalone FTS5 table -- deliberately NOT an external-content table or trigger
+            -- mirror of the `messages` columns -- because the stored body columns hold
+            -- ciphertext and a trigger-based mirror would index that ciphertext verbatim
+            -- (defeating search entirely). Instead `messages_fts` is populated explicitly from
+            -- the plaintext body in application code, at the moment of encryption in
+            -- `DatabaseEngine::save_email` (see there) and via `backfill_message_fts` below for
+            -- any pre-existing rows. This means the trigram index now contains
+            -- plaintext-derived body text: the FTS index itself is NOT encrypted, so message
+            -- body content is recoverable from `messages_fts` by anyone with filesystem access
+            -- to the SQLite database, even though the `messages.body_plain` column remains
+            -- ciphertext. Column-level body encryption therefore provides only limited
+            -- confidentiality while search is enabled -- full body confidentiality (an
+            -- encrypted search index, or whole-database encryption) is future work and is NOT
+            -- provided today.
+            CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
+                id UNINDEXED,
+                subject,
+                sender,
+                body_plain,
+                tokenize = 'trigram'
+            );
+
+            -- Subject/sender are never encrypted in `messages`, so a delete-only trigger is
+            -- sufficient here: it just keeps the FTS index free of orphaned rows. Insertion and
+            -- update of `messages_fts` content happens explicitly in `save_email`, never via an
+            -- AFTER INSERT/UPDATE trigger, because such a trigger would only ever see the
+            -- ciphertext body column.
+            CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
+                DELETE FROM messages_fts WHERE id = old.id;
+            END;
+
+            -- Calendar event summary/location are never encrypted at rest, so trigger-based
+            -- mirroring (unlike the message body) introduces no confidentiality regression.
+            CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
+                id UNINDEXED,
+                summary,
+                location,
+                tokenize = 'trigram'
+            );
+
+            CREATE TRIGGER IF NOT EXISTS events_ai AFTER INSERT ON calendar_events BEGIN
+                INSERT INTO events_fts(id, summary, location)
+                VALUES (new.id, new.summary, COALESCE(new.location, ''));
+            END;
+
+            CREATE TRIGGER IF NOT EXISTS events_ad AFTER DELETE ON calendar_events BEGIN
+                DELETE FROM events_fts WHERE id = old.id;
+            END;
+
+            CREATE TRIGGER IF NOT EXISTS events_au AFTER UPDATE ON calendar_events BEGIN
+                DELETE FROM events_fts WHERE id = old.id;
+                INSERT INTO events_fts(id, summary, location)
+                VALUES (new.id, new.summary, COALESCE(new.location, ''));
+            END;
             "#,
         )
         .execute(&self.pool)
         .await
         .map_err(DatabaseError::Query)?;
 
+        self.backfill_message_fts().await?;
+
+        Ok(())
+    }
+
+    /// Backfill `messages_fts` for any `messages` row that does not yet have a matching FTS
+    /// entry -- e.g. rows written before the FTS5 index existed, or written directly by a
+    /// process that bypassed `save_email`'s explicit index population. Run automatically as
+    /// part of [`DatabaseEngine::migrate`] (idempotent: a fully-indexed database performs no
+    /// work). The stored `body_plain` column holds AES-256-GCM ciphertext, so each candidate
+    /// row is decrypted with this engine's storage key before being written into the
+    /// plaintext-derived trigram index -- see the confidentiality tradeoff documented above
+    /// `messages_fts`'s `CREATE VIRTUAL TABLE` statement in [`DatabaseEngine::migrate`].
+    async fn backfill_message_fts(&self) -> Result<(), DatabaseError> {
+        let rows: Vec<(String, String, String, Option<String>)> = sqlx::query_as(
+            r#"
+            SELECT m.id, m.subject, m.sender, m.body_plain
+            FROM messages m
+            WHERE NOT EXISTS (SELECT 1 FROM messages_fts f WHERE f.id = m.id)
+            "#,
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(DatabaseError::Query)?;
+
+        for (id, subject, sender, body_plain) in rows {
+            let dec_plain = body_plain
+                .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p))
+                .unwrap_or_default();
+
+            sqlx::query(
+                "INSERT INTO messages_fts (id, subject, sender, body_plain) VALUES (?, ?, ?, ?)",
+            )
+            .bind(&id)
+            .bind(&subject)
+            .bind(&sender)
+            .bind(&dec_plain)
+            .execute(&self.pool)
+            .await
+            .map_err(DatabaseError::Query)?;
+        }
+
         Ok(())
     }
 
@@ -454,6 +557,15 @@ impl DatabaseEngine {
     }
 
     /// Save an [`nuncio_core::model::Email`] to SQLite (INSERT OR REPLACE).
+    ///
+    /// The message body is encrypted (AES-256-GCM) before being written to the `messages`
+    /// table, but the *plaintext* body is also indexed into the standalone `messages_fts`
+    /// FTS5 table at this same write, before the plaintext is discarded. This is what makes
+    /// body search functional at all: a trigger mirroring the encrypted column would only ever
+    /// index ciphertext. See the confidentiality tradeoff documented above the `messages_fts`
+    /// `CREATE VIRTUAL TABLE` statement in [`DatabaseEngine::migrate`] -- the FTS index itself
+    /// is not encrypted, so this intentionally trades some body confidentiality for working
+    /// search.
     pub async fn save_email(&self, email: &nuncio_core::model::Email) -> Result<(), DatabaseError> {
         let enc_plain = email
             .body_plain
@@ -464,6 +576,8 @@ impl DatabaseEngine {
             .as_ref()
             .map(|h| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, h));
 
+        let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?;
+
         sqlx::query(
             r#"
             INSERT OR REPLACE INTO messages
@@ -481,10 +595,33 @@ impl DatabaseEngine {
         .bind(if email.read { 1i64 } else { 0i64 })
         .bind(&enc_plain)
         .bind(&enc_html)
-        .execute(&self.pool)
+        .execute(&mut *tx)
         .await
         .map_err(DatabaseError::Query)?;
 
+        // Re-indexing: delete any prior FTS row for this message id, then insert the fresh
+        // plaintext-derived row. FTS5 has no natural "INSERT OR REPLACE" semantics for a
+        // standalone (non-external-content) table, so this is done explicitly rather than via
+        // trigger.
+        sqlx::query("DELETE FROM messages_fts WHERE id = ?")
+            .bind(&email.id)
+            .execute(&mut *tx)
+            .await
+            .map_err(DatabaseError::Query)?;
+
+        sqlx::query(
+            "INSERT INTO messages_fts (id, subject, sender, body_plain) VALUES (?, ?, ?, ?)",
+        )
+        .bind(&email.id)
+        .bind(&email.subject)
+        .bind(&email.sender)
+        .bind(email.body_plain.as_deref().unwrap_or(""))
+        .execute(&mut *tx)
+        .await
+        .map_err(DatabaseError::Query)?;
+
+        tx.commit().await.map_err(DatabaseError::Query)?;
+
         Ok(())
     }
 

crates/nuncio-store/src/search.rs

@@ -21,67 +21,16 @@ pub struct SearchEngine<'a> {
 
 impl<'a> SearchEngine<'a> {
     /// Create a new `SearchEngine` bound to a `DatabaseEngine`.
+    ///
+    /// Unlike earlier versions of this type, `SearchEngine` no longer lazily creates the FTS5
+    /// virtual tables on first use: [`DatabaseEngine::migrate`] creates `messages_fts` and
+    /// `events_fts` (and backfills `messages_fts` for any pre-existing rows) up front, so every
+    /// message or event ever written is searchable as soon as the engine is open, not just
+    /// those saved after the first search call.
     pub fn new(db: &'a DatabaseEngine) -> Self {
         Self { db }
     }
 
-    /// Initialize FTS5 virtual tables and sync triggers.
-    pub async fn setup_fts_tables(&self) -> Result<(), DatabaseError> {
-        sqlx::query(
-            r#"
-            CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5(
-                id UNINDEXED,
-                subject,
-                sender,
-                body_plain,
-                tokenize = 'trigram'
-            );
-
-            CREATE TRIGGER IF NOT EXISTS messages_ai AFTER INSERT ON messages BEGIN
-                INSERT INTO messages_fts(id, subject, sender, body_plain)
-                VALUES (new.id, new.subject, new.sender, COALESCE(new.body_plain, ''));
-            END;
-
-            CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN
-                DELETE FROM messages_fts WHERE id = old.id;
-            END;
-
-            CREATE TRIGGER IF NOT EXISTS messages_au AFTER UPDATE ON messages BEGIN
-                DELETE FROM messages_fts WHERE id = old.id;
-                INSERT INTO messages_fts(id, subject, sender, body_plain)
-                VALUES (new.id, new.subject, new.sender, COALESCE(new.body_plain, ''));
-            END;
-
-            CREATE VIRTUAL TABLE IF NOT EXISTS events_fts USING fts5(
-                id UNINDEXED,
-                summary,
-                location,
-                tokenize = 'trigram'
-            );
-
-            CREATE TRIGGER IF NOT EXISTS events_ai AFTER INSERT ON calendar_events BEGIN
-                INSERT INTO events_fts(id, summary, location)
-                VALUES (new.id, new.summary, COALESCE(new.location, ''));
-            END;
-
-            CREATE TRIGGER IF NOT EXISTS events_ad AFTER DELETE ON calendar_events BEGIN
-                DELETE FROM events_fts WHERE id = old.id;
-            END;
-
-            CREATE TRIGGER IF NOT EXISTS events_au AFTER UPDATE ON calendar_events BEGIN
-                DELETE FROM events_fts WHERE id = old.id;
-                INSERT INTO events_fts(id, summary, location)
-                VALUES (new.id, new.summary, COALESCE(new.location, ''));
-            END;
-            "#,
-        )
-        .execute(self.db.pool())
-        .await
-        .map_err(DatabaseError::Query)?;
-
-        Ok(())
-    }
-
     /// Sanitize user search inputs to prevent SQLite FTS5 query operator syntax errors.
     pub fn sanitize_fts5_query(query: &str) -> String {
         let clean = query.trim();
@@ -151,55 +100,132 @@ impl<'a> SearchEngine<'a> {
 mod tests {
     use super::*;
 
+    fn sample_email(id: &str, subject: &str, body_plain: &str) -> nuncio_core::model::Email {
+        nuncio_core::model::Email {
+            id: id.to_string(),
+            account_id: "acct-1".to_string(),
+            folder_id: "inbox".to_string(),
+            subject: subject.to_string(),
+            sender: "alice@nuncio.mx".to_string(),
+            recipient: "bob@nuncio.mx".to_string(),
+            received_at: 1_700_000_000,
+            read: false,
+            body_plain: Some(body_plain.to_string()),
+            body_html: None,
+            attachments: Vec::new(),
+        }
+    }
+
+    /// The crux test: proves body search works over the REAL `save_email` write path (no
+    /// raw-SQL plaintext injection bypassing encryption), while the stored body column stays
+    /// ciphertext. This is what the old `fts5_message_search_and_triggers` test failed to
+    /// prove -- it inserted plaintext directly via raw SQL, which trivially "worked" but never
+    /// exercised (or caught the bug in) the encrypt-then-index write path.
     #[tokio::test]
-    async fn fts5_message_search_and_triggers() {
+    async fn fts5_message_body_search_via_real_save_email_write_path() {
         let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
         let search = SearchEngine::new(&db);
-        search.setup_fts_tables().await.unwrap();
 
-        // Empty search returns empty results
+        // Empty search returns empty results, with no setup call needed: migrate() already
+        // created the FTS5 tables eagerly.
         assert!(search.search_messages("").await.unwrap().is_empty());
 
-        // Insert message
-        sqlx::query(
-            "INSERT INTO messages (id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain)
-             VALUES ('msg-1', 'acct-1', 'inbox', 'Quarterly Financial Meeting', 'alice@nuncio.mx', 'bob@nuncio.mx', 1700000000, 0, 'Discuss budget revenue')",
-        )
-        .execute(db.pool())
-        .await
-        .unwrap();
+        let email = sample_email(
+            "msg-1",
+            "Quarterly Financial Meeting",
+            "Let's discuss the annual budget revenue forecast",
+        );
+        db.save_email(&email).await.unwrap();
 
-        // Search trigram match
-        let hits = search.search_messages("Financial").await.unwrap();
+        // Body search matches a plaintext body term -- proves the FTS index holds real,
+        // decrypted-at-write-time plaintext trigrams, not the AES-256-GCM ciphertext that is
+        // actually stored in messages.body_plain.
+        let hits = search.search_messages("revenue").await.unwrap();
         assert_eq!(hits.len(), 1);
         assert_eq!(hits[0].id, "msg-1");
         assert_eq!(hits[0].title, "Quarterly Financial Meeting");
 
-        // Update message trigger
-        sqlx::query("UPDATE messages SET subject = 'Updated Strategy Review' WHERE id = 'msg-1'")
-            .execute(db.pool())
-            .await
-            .unwrap();
+        // Subject search still matches: subject/sender were never encrypted.
+        let subject_hits = search.search_messages("Financial").await.unwrap();
+        assert_eq!(subject_hits.len(), 1);
 
-        let updated_hits = search.search_messages("Strategy").await.unwrap();
-        assert_eq!(updated_hits.len(), 1);
-        assert_eq!(updated_hits[0].title, "Updated Strategy Review");
+        // The body column at rest must remain encrypted ciphertext -- never the plaintext
+        // search term, and never equal to the original plaintext body.
+        let (stored_body,): (String,) =
+            sqlx::query_as("SELECT body_plain FROM messages WHERE id = 'msg-1'")
+                .fetch_one(db.pool())
+                .await
+                .unwrap();
+        assert_ne!(stored_body, email.body_plain.clone().unwrap());
+        assert!(!stored_body.to_lowercase().contains("revenue"));
 
-        // Delete message trigger
+        // Re-saving via save_email (an update, not a fresh insert) replaces the FTS row rather
+        // than duplicating or leaving a stale entry: the old term no longer matches, the new
+        // one does.
+        let mut updated = email.clone();
+        updated.body_plain = Some("Updated strategy review notes".to_string());
+        db.save_email(&updated).await.unwrap();
+
+        assert!(search.search_messages("revenue").await.unwrap().is_empty());
+        let new_hits = search.search_messages("strategy").await.unwrap();
+        assert_eq!(new_hits.len(), 1);
+        assert_eq!(new_hits[0].id, "msg-1");
+
+        // Delete message trigger still cleans up the FTS row (subject/sender/id only -- no
+        // ciphertext involved).
         sqlx::query("DELETE FROM messages WHERE id = 'msg-1'")
             .execute(db.pool())
             .await
             .unwrap();
 
-        let deleted_hits = search.search_messages("Strategy").await.unwrap();
+        let deleted_hits = search.search_messages("strategy").await.unwrap();
         assert!(deleted_hits.is_empty());
     }
 
+    /// Proves the backfill half of the fix: a message saved through the real write path,
+    /// whose FTS entry is then lost (simulating either a row written before the FTS5 index
+    /// existed, or one written by a process that bypassed `save_email`'s explicit indexing),
+    /// is NOT searchable until a migration pass runs -- at which point `backfill_message_fts`
+    /// decrypts the ciphertext body column with this engine's real storage key and repopulates
+    /// the plaintext-derived trigram index, with no search call ever required to trigger it.
+    #[tokio::test]
+    async fn fts5_backfill_indexes_preexisting_rows_missing_from_the_index() {
+        let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
+        let email = sample_email(
+            "msg-legacy-1",
+            "Legacy Roadmap Notes",
+            "Confidential quarterly roadmap details",
+        );
+        db.save_email(&email).await.unwrap();
+
+        // Simulate pre-existing data whose FTS entry is missing (e.g. written before the FTS5
+        // index existed) by dropping its messages_fts row directly.
+        sqlx::query("DELETE FROM messages_fts WHERE id = 'msg-legacy-1'")
+            .execute(db.pool())
+            .await
+            .unwrap();
+
+        let search = SearchEngine::new(&db);
+        assert!(
+            search.search_messages("roadmap").await.unwrap().is_empty(),
+            "message must be unsearchable once its FTS entry is missing"
+        );
+
+        // Re-running migration (idempotent) must backfill the missing row from the encrypted
+        // body column -- this is the exact code path a fresh process re-opening this database
+        // takes via DatabaseEngine::open/connect_file, with no search call involved.
+        db.migrate().await.unwrap();
+
+        let hits = search.search_messages("roadmap").await.unwrap();
+        assert_eq!(hits.len(), 1);
+        assert_eq!(hits[0].id, "msg-legacy-1");
+        assert_eq!(hits[0].title, "Legacy Roadmap Notes");
+    }
+
     #[tokio::test]
     async fn fts5_event_search_and_triggers() {
         let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
         let search = SearchEngine::new(&db);
-        search.setup_fts_tables().await.unwrap();
 
         // Empty search returns empty results
         assert!(search.search_events("").await.unwrap().is_empty());