From 3de75493d5655554f495922a1c90ea6fdefe775f Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 10 Aug 2026 14:48:37 -0500 Subject: [PATCH] feat(mail): capture Message-ID and a content hash at ingest Identity cannot currently be reconstructed for a stored message. The parser never reads Message-ID, the Email model has nowhere to put it, the messages table has no column for it, and the raw octets are dropped once parsed. Any later attempt to give a message a folder-independent identity therefore has nothing to work from except a re-download of the entire mailbox. Capture both now, as nullable columns nothing reads yet, so the data starts accruing while the design that consumes it is still being built. Normalization is fixed and shared rather than left to each call site: strip all ASCII whitespace (a msg-id cannot legally contain any, so this unfolds a wrapped header without knowing how the transport folded it), strip one pair of angle brackets, and preserve case, since only the domain half is case-insensitive and lowercasing wholesale would merge ids the origin considers distinct. Engines that normalize differently would hold the same message under different bytes, and no comparison between their caches would mean anything. Captured at every ingest path, including the two that do not parse a full message. The envelope-only path takes Message-ID from ENVELOPE rather than skipping capture. The headers-only path keeps the id recovered from those headers but discards the hash parse_mime computed over them, because content_hash is defined over a message's full octets and publishing a header-only digest under that name would quietly mean something else. The hash exists to keep Message-ID honest: it is public, echoed in every References header, and reused legitimately -- a mailing-list copy and a direct copy share one, and a draft keeps one across every save -- so the pair distinguishes messages that the id alone would merge. No backfill. Legacy rows stay NULL, which reads as "not captured" rather than "this message has no Message-ID"; inventing a value would be worse than admitting the gap. Refs #386 --- crates/nuncio-core/src/export.rs | 4 + crates/nuncio-core/src/model.rs | 116 ++++++++++++ crates/nuncio-filter/src/engine.rs | 4 + crates/nuncio-mail/src/imap.rs | 18 ++ crates/nuncio-mail/src/jmap.rs | 44 ++++- crates/nuncio-mail/src/mock.rs | 2 + crates/nuncio-mail/src/parser.rs | 38 ++++ crates/nuncio-mail/src/smtp.rs | 2 + crates/nuncio-store/src/db.rs | 167 +++++++++++++++++- crates/nuncio-store/src/search.rs | 2 + crates/nunciod/src/grpc.rs | 16 ++ crates/nunciod/src/outbox.rs | 2 + crates/nunciod/src/sync.rs | 2 + .../nunciod/tests/filters_triage_e2e_test.rs | 2 + .../nunciod/tests/no_secrets_canary_test.rs | 2 + .../nunciod/tests/outbox_executor_e2e_test.rs | 2 + crates/nunciod/tests/spine_e2e_test.rs | 2 + 17 files changed, 416 insertions(+), 9 deletions(-) diff --git a/crates/nuncio-core/src/export.rs b/crates/nuncio-core/src/export.rs index d180291..8c5ad0b 100644 --- a/crates/nuncio-core/src/export.rs +++ b/crates/nuncio-core/src/export.rs @@ -195,6 +195,8 @@ mod tests { body_plain: Some("Hello world body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }, Email { id: "msg-101".to_string(), @@ -210,6 +212,8 @@ mod tests { body_plain: Some("Second body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }, ] } diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index b6e5034..9796db4 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -104,6 +104,26 @@ pub struct Email { pub body_html: Option, /// List of attached files. pub attachments: Vec, + /// The message's RFC 5322 `Message-ID`, normalized by + /// [`Email::normalize_message_id`], or `None` when the message carried + /// none (the header is `SHOULD`, not `MUST`) or the transport did not + /// surface it. + /// + /// Captured but not yet used for identity. It is a *hint* toward a + /// folder-independent identity, never an identity on its own: the value is + /// public (it is echoed in the `References` of every reply) and forgeable, + /// so keying storage on it alone would let a crafted message address, and + /// overwrite, one the user already trusts. + pub message_id: Option, + /// Hex SHA-256 over the full RFC822 octets the message was parsed from, or + /// `None` when only headers or an envelope were available. + /// + /// Pairs with [`Email::message_id`]: the same `Message-ID` carrying + /// different content is two messages, not one. That case is ordinary -- + /// a mailing-list copy and a direct copy of the same mail share a + /// `Message-ID` while differing in headers and footer, and a draft keeps + /// one `Message-ID` across every save. + pub content_hash: Option, } impl Email { @@ -131,6 +151,51 @@ impl Email { } hex::encode(hasher.finalize()) } + + /// Canonicalize a raw `Message-ID` header value. + /// + /// Every engine syncing an account must derive byte-identical values from + /// the same header, or two caches that agree in substance will disagree in + /// bytes and no comparison between them can be trusted. The rules are + /// therefore fixed and deliberately small: + /// + /// 1. Remove **all** ASCII whitespace. A `msg-id` cannot legally contain + /// any, so this unfolds a wrapped header without needing to know how the + /// transport folded it -- and transports differ. + /// 2. Strip one surrounding pair of angle brackets, which servers include + /// or omit inconsistently between `ENVELOPE` and a raw header. + /// 3. Preserve case. Only the domain half is case-insensitive per RFC 5322 + /// ยง3.6.4; lowercasing the whole value would merge ids that the + /// originating server considers distinct, and the same server returns + /// the same bytes to every engine anyway. + /// + /// Returns `None` for an absent or empty value so a missing header stays + /// distinguishable from a present-but-empty one. + pub fn normalize_message_id(raw: &str) -> Option { + let stripped: String = raw.chars().filter(|c| !c.is_ascii_whitespace()).collect(); + let stripped = stripped + .strip_prefix('<') + .and_then(|s| s.strip_suffix('>')) + .unwrap_or(&stripped); + if stripped.is_empty() { + None + } else { + Some(stripped.to_string()) + } + } + + /// Hex SHA-256 over a message's full RFC822 octets, for the + /// `content_hash` field. + /// + /// Deterministic for identical bytes, which is what makes it usable as the + /// tiebreaker against a duplicated `Message-ID`. It is *not* stable across + /// servers that re-serialize a message, so it distinguishes content within + /// an account rather than identifying a message globally. + pub fn content_hash_of(raw_bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(raw_bytes); + hex::encode(hasher.finalize()) + } } /// Mailbox folder entity. @@ -247,4 +312,55 @@ mod tests { Email::surrogate_id("ab", "c", "1", "1"), ); } + + #[test] + fn normalize_message_id_unfolds_and_unwraps_to_one_canonical_form() { + // Every spelling a transport might hand us for the same id must + // collapse to identical bytes, or two engines holding the same message + // cannot be shown to agree. + let canonical = "abc123@mail.example.com"; + for raw in [ + "", + "abc123@mail.example.com", + " ", + "", + "", + ] { + assert_eq!( + Email::normalize_message_id(raw).as_deref(), + Some(canonical), + "input {raw:?} did not canonicalize" + ); + } + } + + #[test] + fn normalize_message_id_preserves_case_and_rejects_empty() { + // Only the domain half is case-insensitive per RFC 5322 3.6.4; + // lowercasing wholesale would merge ids the origin considers distinct. + assert_eq!( + Email::normalize_message_id("").as_deref(), + Some("AbC@Example.COM") + ); + assert_eq!(Email::normalize_message_id(""), None); + assert_eq!(Email::normalize_message_id("<>"), None); + assert_eq!(Email::normalize_message_id(" "), None); + } + + #[test] + fn content_hash_distinguishes_identical_message_ids_with_different_bodies() { + // The case the hash exists for: a list copy and a direct copy share a + // Message-ID but are not the same message. + let direct = b"Message-ID: \r\n\r\nbody"; + let via_list = b"Message-ID: \r\nList-Id: l\r\n\r\nbody\n--\nfooter"; + assert_ne!( + Email::content_hash_of(direct), + Email::content_hash_of(via_list) + ); + assert_eq!( + Email::content_hash_of(direct), + Email::content_hash_of(direct), + "the hash must be deterministic for identical octets" + ); + } } diff --git a/crates/nuncio-filter/src/engine.rs b/crates/nuncio-filter/src/engine.rs index d5d1230..569823c 100644 --- a/crates/nuncio-filter/src/engine.rs +++ b/crates/nuncio-filter/src/engine.rs @@ -495,6 +495,8 @@ mod tests { body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } @@ -518,6 +520,8 @@ mod tests { body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; let results = engine.evaluate(&email); diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index 2de83f7..afae823 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -931,6 +931,14 @@ impl ImapEngine { let received_at = fetch_data.internal_date().map_or(0, |dt| dt.timestamp()); + // No body was returned, so there are no octets to hash -- but the + // ENVELOPE carries `message-id` (RFC 3501 section 7.4.2), so identity + // capture does not have to be skipped on this path. + let message_id = fetch_data + .envelope() + .and_then(|env| env.message_id.as_ref()) + .and_then(|raw| Email::normalize_message_id(&String::from_utf8_lossy(raw))); + Ok(Email { id: email_id, account_id: self.account_id.clone(), @@ -945,6 +953,8 @@ impl ImapEngine { body_plain: None, body_html: None, attachments: Vec::new(), + message_id, + content_hash: None, }) } @@ -982,6 +992,12 @@ impl ImapEngine { email.body_plain = None; email.body_html = None; email.attachments = Vec::new(); + // `parse_mime` hashed the header section it was handed, but + // `content_hash` is defined over a message's FULL octets. + // Keeping that value would publish a hash that silently means + // something else, so drop it; the `Message-ID` it recovered + // from those headers is still correct and is kept. + email.content_hash = None; return email; } } @@ -1004,6 +1020,8 @@ impl ImapEngine { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }) } diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index 80319b7..187f257 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -92,6 +92,10 @@ pub struct JmapEmail { pub received_at: Option, pub is_unread: Option, pub body_snippet: Option, + /// RFC 8621 models `messageId` as a list, because a message may carry more + /// than one `Message-ID` header. The first entry is the one used. + #[serde(default)] + pub message_id: Option>, } /// JMAP email address object. @@ -184,7 +188,7 @@ impl JmapEngine { { "accountId": account_id, "ids": ids, - "properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet"] + "properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet", "messageId"] }, "c1" ] @@ -418,6 +422,15 @@ impl JmapEngine { body_plain: item.body_snippet, body_html: None, attachments: Vec::new(), + message_id: item + .message_id + .as_ref() + .and_then(|ids| ids.first()) + .and_then(|raw| Email::normalize_message_id(raw)), + // `Email/get` returns a `bodySnippet`, not the message's + // octets, so there is nothing to hash here without an + // extra download per message. + content_hash: None, } }) .collect(); @@ -705,6 +718,35 @@ mod tests { assert_eq!(new_state, "s-2"); } + #[test] + fn parse_email_get_response_captures_normalized_message_id() { + // RFC 8621 models `messageId` as a list; the first entry is taken and + // normalized to the same canonical form the IMAP paths produce, so two + // engines syncing one account via different protocols agree. + let raw = r#"{ + "methodResponses": [ + ["Email/get", {"state": "s-1", "list": [ + {"id": "m-1", "subject": "Hi", "messageId": [""]}, + {"id": "m-2", "subject": "No id"} + ]}, "c1"] + ] + }"#; + let engine = JmapEngine::new("acct-1"); + let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get"); + + assert_eq!( + emails[0].message_id.as_deref(), + Some("Abc.123@mail.nuncio.mx") + ); + assert_eq!( + emails[1].message_id, None, + "an absent messageId must stay None, not become empty" + ); + // `Email/get` returns a snippet, never the octets, so there is nothing + // honest to hash on this path. + assert_eq!(emails[0].content_hash, None); + } + #[test] fn parse_invalid_jmap_json_fails() { let engine = JmapEngine::new("acct-1"); diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index 487d09a..706f6ba 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -254,6 +254,8 @@ mod tests { body_plain: Some("Mock body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; mock.add_message(email.clone()); diff --git a/crates/nuncio-mail/src/parser.rs b/crates/nuncio-mail/src/parser.rs index 95e19f7..e2c051e 100644 --- a/crates/nuncio-mail/src/parser.rs +++ b/crates/nuncio-mail/src/parser.rs @@ -95,6 +95,9 @@ impl MimeParserAdapter { let received_at = msg.date().map_or(0, |d| d.to_timestamp()); + let message_id = msg.message_id().and_then(Email::normalize_message_id); + let content_hash = Some(Email::content_hash_of(raw_bytes)); + let body_plain = msg.body_text(0).map(|b| b.to_string()); let body_html = msg.body_html(0).map(|b| b.to_string()); @@ -141,6 +144,8 @@ impl MimeParserAdapter { body_plain, body_html, attachments, + message_id, + content_hash, }) } } @@ -170,6 +175,39 @@ mod tests { ); } + #[test] + fn parse_captures_normalized_message_id_and_content_hash() { + let raw = b"From: Alice \r\n\ + To: Bob \r\n\ + Message-ID: \r\n\ + Subject: Test\r\n\ + \r\n\ + body"; + + let email = + MimeParserAdapter::parse_mime("msg-1", "acct-1", "inbox", raw).expect("parse succeeds"); + assert_eq!(email.message_id.as_deref(), Some("Abc.123@mail.nuncio.mx")); + assert_eq!( + email.content_hash, + Some(nuncio_core::model::Email::content_hash_of(raw)), + "the hash must cover the octets actually parsed" + ); + } + + #[test] + fn parse_reports_no_message_id_when_the_header_is_absent() { + // RFC 5322 3.6.4 makes Message-ID a SHOULD. An absent header must read + // as "none present", never as an invented or empty value. + let raw = b"From: Alice \r\n\ + Subject: No id\r\n\ + \r\n\ + body"; + let email = + MimeParserAdapter::parse_mime("msg-2", "acct-1", "inbox", raw).expect("parse succeeds"); + assert_eq!(email.message_id, None); + assert!(email.content_hash.is_some()); + } + #[test] fn parse_invalid_mime_returns_error() { let raw = b""; diff --git a/crates/nuncio-mail/src/smtp.rs b/crates/nuncio-mail/src/smtp.rs index 894ae6c..25e7a80 100644 --- a/crates/nuncio-mail/src/smtp.rs +++ b/crates/nuncio-mail/src/smtp.rs @@ -368,6 +368,8 @@ mod tests { body_plain: Some("Plaintext status update".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index c22ba55..cbce742 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -512,7 +512,13 @@ impl DatabaseEngine { received_at INTEGER NOT NULL, read_flag INTEGER NOT NULL DEFAULT 0, body_plain TEXT, - body_html TEXT + body_html TEXT, + -- Captured at ingest, not yet read. Both are nullable because + -- neither is always available: `Message-ID` is SHOULD, not + -- MUST (RFC 5322 3.6.4), and a headers-only or envelope-only + -- fetch has no full octets to hash. + message_id TEXT, + content_hash TEXT ); CREATE TABLE IF NOT EXISTS calendar_events ( @@ -751,11 +757,53 @@ impl DatabaseEngine { self.ensure_accounts_tls_mode_columns().await?; self.ensure_accounts_dav_columns().await?; self.ensure_messages_identity_columns().await?; + self.ensure_messages_provenance_columns().await?; self.backfill_message_fts().await?; Ok(()) } + /// Additive migration adding the `message_id` / `content_hash` provenance + /// columns to a pre-existing `messages` table. + /// + /// Both are nullable and there is **no backfill**, deliberately. The values + /// cannot be reconstructed for rows already stored: the raw octets are + /// discarded after parsing and the `Message-ID` header was never persisted, + /// so the only ways to populate history would be to re-download every + /// message or to invent a value. Legacy rows therefore stay `NULL`, which + /// reads as "not captured" rather than as "this message has no + /// `Message-ID`" -- a distinction any future consumer has to respect. + /// + /// A fresh database gets both columns from `CREATE TABLE IF NOT EXISTS` + /// above, so the `PRAGMA table_info` check finds them present and this is a + /// no-op. SQLite has no `ADD COLUMN IF NOT EXISTS`, hence the explicit + /// check, which makes this safe to run on every daemon startup. + async fn ensure_messages_provenance_columns(&self) -> Result<(), DatabaseError> { + let existing_columns: Vec = sqlx::query("PRAGMA table_info(messages)") + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)? + .iter() + .map(|row| row.get::("name")) + .collect(); + + for column in ["message_id", "content_hash"] { + if existing_columns.iter().any(|c| c == column) { + continue; + } + sqlx::query(&format!("ALTER TABLE messages ADD COLUMN {column} TEXT")) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + tracing::info!( + column, + "migrated messages table: added message provenance column" + ); + } + + Ok(()) + } + /// Additive, backfill-safe migration that adds the `remote_id` / /// `uid_validity` message-identity columns to a pre-existing `messages` /// table that predated the opaque surrogate id, and installs the @@ -1295,8 +1343,8 @@ impl DatabaseEngine { sqlx::query( r#" INSERT OR REPLACE INTO messages - (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html, message_id, content_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, ) .bind(&email.id) @@ -1311,6 +1359,8 @@ impl DatabaseEngine { .bind(if email.read { 1i64 } else { 0i64 }) .bind(&enc_plain) .bind(&enc_html) + .bind(&email.message_id) + .bind(&email.content_hash) .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; @@ -1361,9 +1411,11 @@ impl DatabaseEngine { Option, String, String, + Option, + Option, )> = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity + SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages WHERE folder_id = ? ORDER BY received_at DESC @@ -1391,6 +1443,8 @@ impl DatabaseEngine { body_html, remote_id, uid_validity, + message_id, + content_hash, )| { let dec_plain = body_plain .map(|p| { @@ -1424,6 +1478,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id, + content_hash, }) }, ) @@ -1445,7 +1501,7 @@ impl DatabaseEngine { ) -> Result<(Vec, Option<(i64, String)>), DatabaseError> { let fetch = page_size.saturating_add(1); let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages WHERE folder_id = ", + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages WHERE folder_id = ", ); builder.push_bind(folder_id.to_string()); if let Some((ts, id)) = &after { @@ -1474,6 +1530,8 @@ impl DatabaseEngine { Option, String, String, + Option, + Option, )>() .fetch_all(&self.pool) .await @@ -1510,6 +1568,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: r.12, + content_hash: r.13, }) }) .collect::, DatabaseError>>()?; @@ -1545,9 +1605,11 @@ impl DatabaseEngine { Option, String, String, + Option, + Option, ) = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity + SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages WHERE id = ? "#, @@ -1582,6 +1644,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: row.12, + content_hash: row.13, }) } @@ -2435,7 +2499,7 @@ impl DatabaseEngine { limit: usize, ) -> Result, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages " + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages " ); if !last_id.is_empty() { builder.push("WHERE id > "); @@ -2457,6 +2521,8 @@ impl DatabaseEngine { Option, String, String, + Option, + Option, )>(); let rows = query @@ -2492,6 +2558,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: r.12, + content_hash: r.13, }) }) .collect::, DatabaseError>>() @@ -2509,7 +2577,7 @@ impl DatabaseEngine { folder_id: Option<&str>, ) -> Result, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages" + "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages" ); let mut has_filter = false; @@ -2541,6 +2609,8 @@ impl DatabaseEngine { Option, String, String, + Option, + Option, )>(); let rows = query @@ -2576,6 +2646,8 @@ impl DatabaseEngine { body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: r.12, + content_hash: r.13, }) }) .collect::, DatabaseError>>() @@ -3084,6 +3156,8 @@ mod tests { body_plain: Some("Plaintext content".to_string()), body_html: Some("

HTML content

".to_string()), attachments: Vec::new(), + message_id: None, + content_hash: None, }; engine @@ -3133,6 +3207,8 @@ mod tests { body_plain: Some("Sensitive plaintext body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; engine .save_email(&email) @@ -3437,6 +3513,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; engine.save_email(&email).await.expect("save email"); @@ -4129,6 +4207,8 @@ mod tests { body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; engine.save_email(&email).await.unwrap(); } @@ -4256,6 +4336,8 @@ mod tests { body_plain: Some("body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } @@ -4368,6 +4450,73 @@ mod tests { .expect("re-running an already-complete migration is a no-op"); } + #[tokio::test] + async fn message_provenance_round_trips_through_every_read_path() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let mut email = synced_email("acct-1", "INBOX", "42", "7"); + email.message_id = Some("abc123@mail.nuncio.mx".to_string()); + email.content_hash = Some(nuncio_core::model::Email::content_hash_of(b"raw octets")); + engine.save_email(&email).await.expect("save"); + + let by_id = engine.get_message(&email.id).await.expect("get_message"); + assert_eq!(by_id.message_id, email.message_id); + assert_eq!(by_id.content_hash, email.content_hash); + + let listed = engine.list_messages("INBOX", 10).await.expect("list"); + let found = listed + .iter() + .find(|m| m.id == email.id) + .expect("the saved message is listed"); + assert_eq!(found.message_id, email.message_id); + assert_eq!(found.content_hash, email.content_hash); + + // A message with no Message-ID stays None rather than becoming "". + let plain = synced_email("acct-1", "INBOX", "42", "8"); + assert_eq!(plain.message_id, None); + engine.save_email(&plain).await.expect("save"); + let fetched = engine.get_message(&plain.id).await.expect("get_message"); + assert_eq!(fetched.message_id, None); + assert_eq!(fetched.content_hash, None); + } + + #[tokio::test] + async fn provenance_migration_adds_nullable_columns_without_inventing_history() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Reproduce a pre-migration database: the columns cannot be dropped + // from a live table cheaply, so assert the property that matters -- + // a row written before capture existed reads back as "not captured". + sqlx::query( + "INSERT INTO messages \ + (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag) \ + VALUES ('legacy-1', 'acct-1', 'INBOX', '9', '42', 'subj', 'a@nuncio.mx', 'b@nuncio.mx', 0, 0)", + ) + .execute(engine.pool()) + .await + .unwrap(); + + engine + .ensure_messages_provenance_columns() + .await + .expect("migration is idempotent on an already-migrated database"); + + let legacy = engine.get_message("legacy-1").await.expect("get_message"); + assert_eq!( + legacy.message_id, None, + "a legacy row must read as not-captured, never as a fabricated id" + ); + assert_eq!(legacy.content_hash, None); + + // Running it twice more must stay a no-op. + for _ in 0..2 { + engine + .ensure_messages_provenance_columns() + .await + .expect("re-running is a no-op"); + } + } + fn export_test_email(id: &str, account_id: &str, folder_id: &str) -> nuncio_core::model::Email { nuncio_core::model::Email { id: id.to_string(), @@ -4383,6 +4532,8 @@ mod tests { body_plain: Some(format!("Body {id}")), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nuncio-store/src/search.rs b/crates/nuncio-store/src/search.rs index 92d63e1..36c6ac6 100644 --- a/crates/nuncio-store/src/search.rs +++ b/crates/nuncio-store/src/search.rs @@ -266,6 +266,8 @@ mod tests { body_plain: Some(body_plain.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 8d387bf..0cb2ee9 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -1989,6 +1989,8 @@ fn synthetic_preview_email(id: &str) -> nuncio_core::model::Email { body_plain: Some("Sample body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } @@ -3303,6 +3305,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, remote_id: "remote-1".to_string(), uid_validity: "1".to_string(), }; @@ -4498,6 +4502,8 @@ mod tests { body_plain: Some(body.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } @@ -5552,6 +5558,8 @@ mod tests { body_plain: Some("body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; // Before creating anything, the live engine matches nothing. @@ -5663,6 +5671,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }) .is_empty()); } @@ -5813,6 +5823,8 @@ mod tests { body_plain: Some("Please review the attached deck".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }) .await .expect("seed message"); @@ -5987,6 +5999,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }); assert_eq!(matches.len(), 1); assert_eq!(matches[0].0.id, rule_id); @@ -6168,6 +6182,8 @@ mod tests { body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }); assert_eq!(matches.len(), 1); } diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 7de867e..a23dcd1 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -601,6 +601,8 @@ mod tests { body_plain: Some("secret outbox body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index 0c5036a..79e3fac 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -550,6 +550,8 @@ mod tests { body_plain: Some(format!("body for {id}")), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/tests/filters_triage_e2e_test.rs b/crates/nunciod/tests/filters_triage_e2e_test.rs index 60689d3..9fa2b36 100644 --- a/crates/nunciod/tests/filters_triage_e2e_test.rs +++ b/crates/nunciod/tests/filters_triage_e2e_test.rs @@ -53,6 +53,8 @@ fn sample_email(id: &str, subject: &str) -> Email { body_plain: Some("body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/tests/no_secrets_canary_test.rs b/crates/nunciod/tests/no_secrets_canary_test.rs index 3113464..e4150a9 100644 --- a/crates/nunciod/tests/no_secrets_canary_test.rs +++ b/crates/nunciod/tests/no_secrets_canary_test.rs @@ -150,6 +150,8 @@ fn sentinel_inbound_email() -> Email { body_plain: Some(SENTINEL_BODY.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index 5fccdcd..933f84e 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -127,6 +127,8 @@ fn sample_email(subject: &str) -> Email { body_plain: Some("original body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index b15c2ae..93e28e1 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -88,6 +88,8 @@ fn mock_inbound_email(id: &str, subject: &str, body: &str) -> Email { body_plain: Some(body.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } -- 2.54.0