diff --git a/crates/nuncio-core/src/export.rs b/crates/nuncio-core/src/export.rs index d180291..79a71e4 100644 --- a/crates/nuncio-core/src/export.rs +++ b/crates/nuncio-core/src/export.rs @@ -87,10 +87,13 @@ impl ExportEngine { writer.write_all(from_line.as_bytes())?; total_bytes += from_line.len() as u64; - // Headers + // Headers. No X-Nuncio-Folder header: the folder is a property of + // a placement now, not of the message, so `Email` has none to + // name here. Restoring it would mean threading the relevant + // `Placement` into this function's signature. let headers = format!( - "Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nX-Nuncio-Folder: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n", - email.id, email.sender, email.recipient, email.subject, date_str, email.account_id, email.folder_id + "Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n", + email.id, email.sender, email.recipient, email.subject, date_str, email.account_id ); writer.write_all(headers.as_bytes())?; total_bytes += headers.len() as u64; @@ -184,32 +187,28 @@ mod tests { Email { id: "msg-100".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "100".to_string(), - uid_validity: "1".to_string(), subject: "Test Subject 1".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: Some("Hello world body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }, Email { id: "msg-101".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "101".to_string(), - uid_validity: "1".to_string(), subject: "Test Subject 2".to_string(), sender: "charlie@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000100, - read: true, body_plain: Some("Second body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }, ] } diff --git a/crates/nuncio-core/src/model.rs b/crates/nuncio-core/src/model.rs index b6e5034..0df4a26 100644 --- a/crates/nuncio-core/src/model.rs +++ b/crates/nuncio-core/src/model.rs @@ -61,33 +61,153 @@ pub struct Attachment { pub content: Bytes, } -/// Email message domain entity owned by Nuncio core. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct Email { - /// Opaque, deterministic surrogate identifier for the message. +/// Which tier of the identity precedence produced a [`MessageIdentity`]. +/// +/// Persisted alongside the key so a later pass can tell a server-assigned +/// identity from a locally-derived fallback without recomputing it, and so a +/// store can be audited for how much of it rests on the weakest tier. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum IdentitySource { + /// RFC 8474 OBJECTID `EMAILID` -- server-assigned and stable across folders. + EmailId, + /// Gmail's `X-GM-MSGID`, the same guarantee by a vendor extension. + GmailMsgId, + /// `Message-ID` **and** a content hash together. Never the header alone. + MessageIdContent, + /// Folder-scoped last resort: the message is only identified by where it + /// currently sits, so the same mail in another folder is a second message. + Surrogate, +} + +impl IdentitySource { + /// Stable token persisted in `messages.identity_source`. + pub fn as_str(&self) -> &'static str { + match self { + Self::EmailId => "emailid", + Self::GmailMsgId => "gmsgid", + Self::MessageIdContent => "msgid_content", + Self::Surrogate => "surrogate", + } + } + + /// Parse a token written by [`Self::as_str`]. Returns `None` for anything + /// else so an unrecognised value fails loudly rather than defaulting to a + /// tier that would overstate how trustworthy the key is. /// - /// Derived from [`Email::surrogate_id`] over the account, folder, - /// UIDVALIDITY, and protocol-native id. It carries no decodable internal - /// encoding (callers must treat it as opaque) and is stable across - /// re-syncs of the same message, so persisting it upserts the same row - /// rather than duplicating. Distinct messages that happen to share a - /// protocol id across folders or accounts hash to distinct surrogates, so - /// one can never silently overwrite another. - pub id: String, - /// Account identifier owning the message. + /// Deliberately not `std::str::FromStr`: that trait forces a `Result` and + /// therefore an error type, when the only failure here is "not one of four + /// known tokens" and `None` already says exactly that. + pub fn from_token(s: &str) -> Option { + match s { + "emailid" => Some(Self::EmailId), + "gmsgid" => Some(Self::GmailMsgId), + "msgid_content" => Some(Self::MessageIdContent), + "surrogate" => Some(Self::Surrogate), + _ => None, + } + } +} + +/// A content-addressed message key together with the tier that produced it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MessageIdentity { + /// Hex SHA-256 key identifying the message within its account. + pub key: String, + /// Which precedence tier produced `key`. + pub source: IdentitySource, +} + +/// Server-supplied identity hints for one message, in precedence order. +/// +/// Borrowed rather than owned because every caller already holds these as +/// fields on a parsed response and only needs them for the duration of the +/// key derivation. +#[derive(Debug, Clone, Copy, Default)] +pub struct RemoteIdentity<'a> { + /// RFC 8474 `EMAILID` from a `FETCH ... (EMAILID)`, if the server has OBJECTID. + pub email_id: Option<&'a str>, + /// Gmail `X-GM-MSGID`, if the server advertises `X-GM-EXT-1`. + pub gm_msgid: Option<&'a str>, + /// Normalized `Message-ID` (see [`Email::normalize_message_id`]). + pub message_id: Option<&'a str>, + /// Hex SHA-256 over the full RFC822 octets (see [`Email::content_hash_of`]). + pub content_hash: Option<&'a str>, +} + +/// One occupancy of a message in one mailbox. +/// +/// A message can sit in several folders at once (an IMAP `COPY`, a Gmail label, +/// a message that is both in a thread's folder and in `\All`). The addressing +/// coordinates and the read flag are properties of the *occupancy*, not of the +/// message: IMAP `\Seen` is per-mailbox, and a UID is only meaningful inside one +/// `uid_validity` scope of one folder. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Placement { + /// Account owning the mailbox. pub account_id: String, /// Mailbox folder identifier (e.g. "inbox"). pub folder_id: String, - /// Protocol-native message id used to address the message on its server: - /// the IMAP UID as a decimal string, or the JMAP Email object id. This is - /// what a remote mutation must use to identify the message on the wire -- - /// never the opaque [`Email::id`]. - pub remote_id: String, - /// The addressing scope the `remote_id` is only meaningful within: the IMAP - /// folder UIDVALIDITY as a decimal string. Protocols without a UIDVALIDITY - /// (JMAP) carry a stable sentinel instead, so the surrogate id stays - /// deterministic. + /// The IMAP UIDVALIDITY the `remote_id` was captured under, as a decimal + /// string; a stable sentinel for protocols without one (JMAP). In the + /// primary key because UIDs restart at 1 after a bump and would otherwise + /// collide with rows written before it. pub uid_validity: String, + /// Protocol-native id addressing the message in this mailbox: the IMAP UID + /// as a decimal string, or the JMAP Email object id. + pub remote_id: String, + /// Read/unread state **in this mailbox**. + pub read: bool, +} + +/// The four coordinates that address one mailbox occupancy. +/// +/// The addressing half of a [`Placement`], without the read flag: enough to +/// name an occupancy, not enough to describe it. A protocol backend reports +/// these when a folder stops mentioning a message, and the store deletes +/// exactly the rows they name -- which is why the type lives here in `core` +/// rather than in either crate. What a folder stops reporting is an occupancy, +/// never a message; a message goes only when its last placement does. +/// +/// `Ord` is derived because the sync path sorts and dedups batches of these +/// before deleting; `Hash`/`Eq` because it is also the key of the +/// already-present set used to classify a fetched chunk. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct PlacementKey { + /// Account owning the mailbox. + pub account_id: String, + /// Mailbox folder identifier. + pub folder_id: String, + /// The UIDVALIDITY generation `remote_id` was captured under. + pub uid_validity: String, + /// Protocol-native id addressing the message in this mailbox. + pub remote_id: String, +} + +impl Placement { + /// The addressing coordinates of this occupancy, dropping the read flag. + pub fn key(&self) -> PlacementKey { + PlacementKey { + account_id: self.account_id.clone(), + folder_id: self.folder_id.clone(), + uid_validity: self.uid_validity.clone(), + remote_id: self.remote_id.clone(), + } + } +} + +/// Email message domain entity owned by Nuncio core. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Email { + /// Opaque, account-scoped identity for the message itself. + /// + /// Derived by [`Email::derive_message_key`], so it is independent of which + /// folder the message currently occupies: moving a message between folders + /// does not change it. Where the message actually sits, and its read state + /// there, live in [`Placement`] rows instead. Callers must treat this as an + /// opaque token -- it carries no decodable structure. + pub id: String, + /// Account identifier owning the message. + pub account_id: String, /// Subject line. pub subject: String, /// Sender address (e.g. "alice@nuncio.mx"). @@ -96,14 +216,32 @@ pub struct Email { pub recipient: String, /// Unix timestamp of message arrival. pub received_at: i64, - /// Read/unread flag status. - pub read: bool, /// Plaintext message body. pub body_plain: Option, /// HTML message body. pub body_html: Option, /// List of attached files. pub attachments: Vec, + /// The message's RFC 5322 `Message-ID`, normalized by + /// [`Email::normalize_message_id`], or `None` when the message carried + /// none (the header is `SHOULD`, not `MUST`) or the transport did not + /// surface it. + /// + /// Captured but not yet used for identity. It is a *hint* toward a + /// folder-independent identity, never an identity on its own: the value is + /// public (it is echoed in the `References` of every reply) and forgeable, + /// so keying storage on it alone would let a crafted message address, and + /// overwrite, one the user already trusts. + pub message_id: Option, + /// Hex SHA-256 over the full RFC822 octets the message was parsed from, or + /// `None` when only headers or an envelope were available. + /// + /// Pairs with [`Email::message_id`]: the same `Message-ID` carrying + /// different content is two messages, not one. That case is ordinary -- + /// a mailing-list copy and a direct copy of the same mail share a + /// `Message-ID` while differing in headers and footer, and a draft keeps + /// one `Message-ID` across every save. + pub content_hash: Option, } impl Email { @@ -131,6 +269,139 @@ impl Email { } hex::encode(hasher.finalize()) } + + /// Canonicalize a raw `Message-ID` header value. + /// + /// Every engine syncing an account must derive byte-identical values from + /// the same header, or two caches that agree in substance will disagree in + /// bytes and no comparison between them can be trusted. The rules are + /// therefore fixed and deliberately small: + /// + /// 1. Remove **all** ASCII whitespace. A `msg-id` cannot legally contain + /// any, so this unfolds a wrapped header without needing to know how the + /// transport folded it -- and transports differ. + /// 2. Strip one surrounding pair of angle brackets, which servers include + /// or omit inconsistently between `ENVELOPE` and a raw header. + /// 3. Preserve case. Only the domain half is case-insensitive per RFC 5322 + /// §3.6.4; lowercasing the whole value would merge ids that the + /// originating server considers distinct, and the same server returns + /// the same bytes to every engine anyway. + /// + /// Returns `None` for an absent or empty value so a missing header stays + /// distinguishable from a present-but-empty one. + pub fn normalize_message_id(raw: &str) -> Option { + let stripped: String = raw.chars().filter(|c| !c.is_ascii_whitespace()).collect(); + let stripped = stripped + .strip_prefix('<') + .and_then(|s| s.strip_suffix('>')) + .unwrap_or(&stripped); + if stripped.is_empty() { + None + } else { + Some(stripped.to_string()) + } + } + + /// Hex SHA-256 over a message's full RFC822 octets, for the + /// `content_hash` field. + /// + /// Deterministic for identical bytes, which is what makes it usable as the + /// tiebreaker against a duplicated `Message-ID`. It is *not* stable across + /// servers that re-serialize a message, so it distinguishes content within + /// an account rather than identifying a message globally. + pub fn content_hash_of(raw_bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(raw_bytes); + hex::encode(hasher.finalize()) + } + + /// Domain-separated, length-prefixed digest over an identity tier's inputs. + /// + /// The tier label participates in the hash so that a value appearing in two + /// different tiers (a server whose `EMAILID` happens to equal another's + /// `X-GM-MSGID`) cannot produce one key. Length prefixes stop two distinct + /// field tuples serializing to the same byte stream, exactly as in + /// [`Email::surrogate_id`]. + fn identity_digest(tier: &str, account_id: &str, parts: &[&str]) -> String { + let mut hasher = Sha256::new(); + for field in [tier, account_id].into_iter().chain(parts.iter().copied()) { + hasher.update((field.len() as u64).to_le_bytes()); + hasher.update(field.as_bytes()); + } + hex::encode(hasher.finalize()) + } + + /// Derive the account-scoped message key by identity precedence. + /// + /// `EMAILID` → `X-GM-MSGID` → (`Message-ID` + content hash) → surrogate. + /// Every tier is account-scoped: identity is only ever claimed within the + /// account that fetched it, so one account's server cannot mint a key that + /// addresses another account's mail. + /// + /// The third tier requires **both** a `Message-ID` and a content hash. A + /// `Message-ID` is public -- it is echoed in the `References` of every reply + /// -- and forgeable, so keying on it alone would let a crafted message + /// address, and with an upsert overwrite, a message the user already trusts, + /// and would evade filtering by never being classified as new. Pairing it + /// with the content hash also gets the ordinary cases right: a mailing-list + /// copy and a direct copy share a `Message-ID` but differ in content, and so + /// are correctly two messages. + /// + /// The surrogate tier is folder-scoped and therefore *not* stable across a + /// move; it is the honest answer when the server offered nothing better. + /// + /// # Keys do not yet converge across engine types + /// + /// Precedence makes a key reproducible from the same inputs, not from the + /// same *message*. The IMAP engine does not currently request `EMAILID` or + /// `X-GM-MSGID`, so it can only ever reach the `Message-ID`+content tier or + /// the surrogate, while JMAP supplies a server id and enters at the top + /// tier. Two engines syncing one account therefore produce disjoint key + /// spaces: the same message is two identities, one per engine. This is a + /// known gap in the convergent multi-engine model, not a property to rely + /// on -- do not assume a key derived by one engine addresses the row + /// another engine wrote. + pub fn derive_message_key( + account_id: &str, + remote: RemoteIdentity<'_>, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + ) -> MessageIdentity { + if let Some(email_id) = remote.email_id.filter(|v| !v.is_empty()) { + return MessageIdentity { + key: Self::identity_digest("emailid", account_id, &[email_id]), + source: IdentitySource::EmailId, + }; + } + if let Some(gm_msgid) = remote.gm_msgid.filter(|v| !v.is_empty()) { + return MessageIdentity { + key: Self::identity_digest("gmsgid", account_id, &[gm_msgid]), + source: IdentitySource::GmailMsgId, + }; + } + if let (Some(message_id), Some(content_hash)) = ( + remote.message_id.filter(|v| !v.is_empty()), + remote.content_hash.filter(|v| !v.is_empty()), + ) { + return MessageIdentity { + key: Self::identity_digest( + "msgid_content", + account_id, + &[message_id, content_hash], + ), + source: IdentitySource::MessageIdContent, + }; + } + MessageIdentity { + key: Self::identity_digest( + "surrogate", + account_id, + &[folder_id, uid_validity, remote_id], + ), + source: IdentitySource::Surrogate, + } + } } /// Mailbox folder entity. @@ -247,4 +518,330 @@ mod tests { Email::surrogate_id("ab", "c", "1", "1"), ); } + + #[test] + fn normalize_message_id_unfolds_and_unwraps_to_one_canonical_form() { + // Every spelling a transport might hand us for the same id must + // collapse to identical bytes, or two engines holding the same message + // cannot be shown to agree. + let canonical = "abc123@mail.example.com"; + for raw in [ + "", + "abc123@mail.example.com", + " ", + "", + "", + ] { + assert_eq!( + Email::normalize_message_id(raw).as_deref(), + Some(canonical), + "input {raw:?} did not canonicalize" + ); + } + } + + #[test] + fn normalize_message_id_preserves_case_and_rejects_empty() { + // Only the domain half is case-insensitive per RFC 5322 3.6.4; + // lowercasing wholesale would merge ids the origin considers distinct. + assert_eq!( + Email::normalize_message_id("").as_deref(), + Some("AbC@Example.COM") + ); + assert_eq!(Email::normalize_message_id(""), None); + assert_eq!(Email::normalize_message_id("<>"), None); + assert_eq!(Email::normalize_message_id(" "), None); + } + + #[test] + fn content_hash_distinguishes_identical_message_ids_with_different_bodies() { + // The case the hash exists for: a list copy and a direct copy share a + // Message-ID but are not the same message. + let direct = b"Message-ID: \r\n\r\nbody"; + let via_list = b"Message-ID: \r\nList-Id: l\r\n\r\nbody\n--\nfooter"; + assert_ne!( + Email::content_hash_of(direct), + Email::content_hash_of(via_list) + ); + assert_eq!( + Email::content_hash_of(direct), + Email::content_hash_of(direct), + "the hash must be deterministic for identical octets" + ); + } + + #[test] + fn message_key_precedence_prefers_emailid_over_everything_below_it() { + let remote = RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: Some("1234567890"), + message_id: Some("a@b.example"), + content_hash: Some("deadbeef"), + }; + let id = Email::derive_message_key("acct-1", remote, "INBOX", "42", "5"); + assert_eq!(id.source, IdentitySource::EmailId); + + // Same EMAILID in a different folder is the SAME message. + let elsewhere = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: None, + message_id: None, + content_hash: None, + }, + "Archive", + "99", + "7", + ); + assert_eq!(id.key, elsewhere.key); + } + + #[test] + fn message_key_is_account_scoped_at_every_tier() { + let emailid = || RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: None, + message_id: None, + content_hash: None, + }; + let a = Email::derive_message_key("acct-1", emailid(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", emailid(), "INBOX", "42", "5"); + assert_ne!( + a.key, b.key, + "the same EMAILID in two accounts must not collide" + ); + + let gmsgid = || RemoteIdentity { + email_id: None, + gm_msgid: Some("1234567890"), + message_id: None, + content_hash: None, + }; + let a = Email::derive_message_key("acct-1", gmsgid(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", gmsgid(), "INBOX", "42", "5"); + assert_ne!( + a.key, b.key, + "the same X-GM-MSGID in two accounts must not collide" + ); + + let msgid_content = || RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some("deadbeef"), + }; + let a = Email::derive_message_key("acct-1", msgid_content(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", msgid_content(), "INBOX", "42", "5"); + assert_ne!( + a.key, b.key, + "the same Message-ID and content hash in two accounts must not collide" + ); + + let surrogate = RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: None, + content_hash: None, + }; + let a = Email::derive_message_key("acct-1", surrogate, "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", surrogate, "INBOX", "42", "5"); + assert_ne!( + a.key, b.key, + "the same folder/uidvalidity/remote-id in two accounts must not collide" + ); + } + + #[test] + fn a_message_id_without_a_content_hash_never_reaches_the_content_tier() { + // The non-negotiable: a Message-ID is public and forgeable, so it must + // never key storage on its own. Missing content hash falls through to the + // folder-scoped surrogate rather than trusting the header alone. + let id = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: None, + }, + "INBOX", + "42", + "5", + ); + assert_eq!(id.source, IdentitySource::Surrogate); + + let surrogate_only = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: None, + content_hash: None, + }, + "INBOX", + "42", + "5", + ); + assert_eq!(id.key, surrogate_only.key); + } + + #[test] + fn the_same_message_id_with_different_content_is_two_messages() { + let mk = |hash: &str| { + Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some(hash), + }, + "INBOX", + "42", + "5", + ) + }; + let list_copy = mk("1111"); + let direct_copy = mk("2222"); + assert_eq!(list_copy.source, IdentitySource::MessageIdContent); + assert_ne!(list_copy.key, direct_copy.key); + } + + #[test] + fn tiers_are_domain_separated_so_a_shared_value_cannot_collide_across_them() { + let as_emailid = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: Some("X"), + gm_msgid: None, + message_id: None, + content_hash: None, + }, + "INBOX", + "42", + "5", + ); + let as_gmsgid = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: Some("X"), + message_id: None, + content_hash: None, + }, + "INBOX", + "42", + "5", + ); + assert_ne!(as_emailid.key, as_gmsgid.key); + } + + fn sample_placement() -> Placement { + Placement { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "42".to_string(), + remote_id: "5".to_string(), + read: true, + } + } + + #[test] + fn placement_key_carries_every_addressing_coordinate_and_drops_the_read_flag() { + // The key must reproduce all four coordinates exactly. A key that + // dropped or reordered one would address a different occupancy, and a + // sync diffing reported-against-stored would delete live mail. + let placement = sample_placement(); + let key = placement.key(); + assert_eq!(key.account_id, "acct-1"); + assert_eq!(key.folder_id, "INBOX"); + assert_eq!(key.uid_validity, "42"); + assert_eq!(key.remote_id, "5"); + + // The read flag is the one field a key must NOT carry: it describes an + // occupancy rather than naming it, so including it would make the same + // occupancy compare unequal to itself after the user opened the message. + let mut unread = placement.clone(); + unread.read = false; + assert_ne!(placement, unread); + assert_eq!(placement.key(), unread.key()); + } + + #[test] + fn placement_keys_distinguish_every_coordinate_that_can_differ() { + let base = sample_placement().key(); + for mutated in [ + PlacementKey { + account_id: "acct-2".to_string(), + ..base.clone() + }, + PlacementKey { + folder_id: "Archive".to_string(), + ..base.clone() + }, + PlacementKey { + uid_validity: "43".to_string(), + ..base.clone() + }, + PlacementKey { + remote_id: "6".to_string(), + ..base.clone() + }, + ] { + assert_ne!( + base, mutated, + "changing one coordinate must yield a different occupancy" + ); + } + + // Usable as a set key and sortable, which is what the sync path needs + // to diff a reported UID set against what the store holds. + let mut set = std::collections::HashSet::new(); + assert!(set.insert(base.clone())); + assert!(!set.insert(base.clone()), "an equal key must not re-insert"); + let mut keys = [ + PlacementKey { + remote_id: "6".to_string(), + ..base.clone() + }, + base.clone(), + ]; + keys.sort(); + assert_eq!(keys[0], base); + } + + #[test] + fn placement_key_round_trips_through_serde() { + let key = sample_placement().key(); + let json = serde_json::to_string(&key).unwrap(); + let parsed: PlacementKey = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, key); + } + + #[test] + fn placement_round_trips_through_serde_including_its_read_flag() { + let placement = sample_placement(); + let json = serde_json::to_string(&placement).unwrap(); + let parsed: Placement = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, placement); + assert!(parsed.read); + } + + #[test] + fn identity_source_round_trips_through_its_stored_string() { + for source in [ + IdentitySource::EmailId, + IdentitySource::GmailMsgId, + IdentitySource::MessageIdContent, + IdentitySource::Surrogate, + ] { + assert_eq!(IdentitySource::from_token(source.as_str()), Some(source)); + } + assert_eq!( + IdentitySource::from_token("not_a_tier"), + None, + "an unrecognised token must fail loudly, never default to a tier" + ); + } } diff --git a/crates/nuncio-filter/src/engine.rs b/crates/nuncio-filter/src/engine.rs index 72605aa..9139b6c 100644 --- a/crates/nuncio-filter/src/engine.rs +++ b/crates/nuncio-filter/src/engine.rs @@ -6,13 +6,27 @@ use crate::ast::{ }; use arc_swap::ArcSwap; use hmac::{Hmac, Mac}; -use nuncio_core::model::Email; +use nuncio_core::model::{Email, Placement}; use regex::Regex; use sha2::Sha256; use std::sync::Arc; use std::time::{Duration, Instant}; use tracing::{debug, info, warn}; +/// A message together with the one mailbox occupancy being evaluated. +/// +/// `WHERE FOLDER = 'INBOX'` has no answer for a message that sits in three +/// folders at once, so evaluation is defined against a single placement and the +/// caller fans out over however many the message has. Borrowing both halves +/// keeps that fan-out free of clones. +#[derive(Debug, Clone, Copy)] +pub struct PlacedEmail<'a> { + /// Identity and content -- the same for every placement. + pub email: &'a Email, + /// The occupancy this evaluation is about. + pub placement: &'a Placement, +} + /// Pre-compiled single filter rule optimizing regex and predicate execution. #[derive(Clone)] pub struct CompiledFilter { @@ -55,31 +69,40 @@ impl CompiledFilter { } } - /// Evaluate email message against compiled rule condition tree. + /// Evaluate one placement against the compiled rule condition tree. /// /// This checks only the WHERE-clause conditions; it does NOT apply the /// rule's account scope. Callers must gate on /// [`FilterRule::matches_account`] first (as every `FilterEngine` entry /// point does), or a rule scoped to one account will fire on messages of /// another. - pub fn evaluate_condition(&self, email: &Email) -> bool { - Self::eval_node(&self.rule.conditions, email, &self.compiled_regexes) + pub fn evaluate_condition(&self, placed: PlacedEmail<'_>) -> bool { + Self::eval_node(&self.rule.conditions, placed, &self.compiled_regexes) } - fn eval_node(node: &ConditionNode, email: &Email, regexes: &[(String, Regex)]) -> bool { + fn eval_node( + node: &ConditionNode, + placed: PlacedEmail<'_>, + regexes: &[(String, Regex)], + ) -> bool { match node { - ConditionNode::Leaf(leaf) => Self::eval_leaf(leaf, email, regexes), + ConditionNode::Leaf(leaf) => Self::eval_leaf(leaf, placed, regexes), ConditionNode::And(children) => { - children.iter().all(|c| Self::eval_node(c, email, regexes)) + children.iter().all(|c| Self::eval_node(c, placed, regexes)) } ConditionNode::Or(children) => { - children.iter().any(|c| Self::eval_node(c, email, regexes)) + children.iter().any(|c| Self::eval_node(c, placed, regexes)) } - ConditionNode::Not(inner) => !Self::eval_node(inner, email, regexes), + ConditionNode::Not(inner) => !Self::eval_node(inner, placed, regexes), } } - fn eval_leaf(leaf: &ConditionLeaf, email: &Email, regexes: &[(String, Regex)]) -> bool { + fn eval_leaf( + leaf: &ConditionLeaf, + placed: PlacedEmail<'_>, + regexes: &[(String, Regex)], + ) -> bool { + let email = placed.email; match &leaf.field { FilterField::Subject => { Self::eval_string_op(&email.subject, &leaf.operator, &leaf.value, regexes) @@ -98,12 +121,18 @@ impl CompiledFilter { .unwrap_or(""); Self::eval_string_op(body, &leaf.operator, &leaf.value, regexes) } - FilterField::Folder => { - Self::eval_string_op(&email.folder_id, &leaf.operator, &leaf.value, regexes) - } - FilterField::Account => { - Self::eval_string_op(&email.account_id, &leaf.operator, &leaf.value, regexes) - } + FilterField::Folder => Self::eval_string_op( + &placed.placement.folder_id, + &leaf.operator, + &leaf.value, + regexes, + ), + FilterField::Account => Self::eval_string_op( + &placed.placement.account_id, + &leaf.operator, + &leaf.value, + regexes, + ), FilterField::HasAttachment => { let has = !email.attachments.is_empty(); if let FilterValue::Boolean(b) = leaf.value { @@ -277,18 +306,26 @@ impl FilterEngine { Ok(()) } - /// Evaluate email message returning matching rule actions. + /// Evaluate one placement of a message, returning the matching rules' actions. + /// + /// Defined per placement, not per message: `FOLDER` and `ACCOUNT` are + /// properties of where the message sits, and a message can sit in several + /// places at once. Callers holding a multi-placement message evaluate each + /// placement and are responsible for not acting twice on one message -- see + /// the fire-once claim in the store. /// /// A rule only ever fires for the account it was created against /// (`FilterRule::target_account`, set via `ON ACCOUNT` or `*` for all /// accounts) — condition matching alone is not account-scoped, so this /// check must happen before `evaluate_condition` runs. - pub fn evaluate(&self, email: &Email) -> Vec<(FilterRule, Vec)> { + pub fn evaluate(&self, placed: PlacedEmail<'_>) -> Vec<(FilterRule, Vec)> { let guard = self.cache.load(); let mut results = Vec::new(); for filter in &guard.filters { - if filter.rule.matches_account(&email.account_id) && filter.evaluate_condition(email) { + if filter.rule.matches_account(&placed.placement.account_id) + && filter.evaluate_condition(placed) + { debug!( rule_id = %filter.rule.id, rule_name = %filter.rule.name, @@ -315,21 +352,25 @@ impl FilterEngine { /// the remaining rules rather than aborting the whole batch. pub async fn evaluate_with_timeout( &self, - email: &Email, + placed: PlacedEmail<'_>, timeout_duration: Duration, ) -> Vec<(FilterRule, Vec)> { let guard = self.cache.load(); let mut results = Vec::new(); for filter in &guard.filters { - if !filter.rule.matches_account(&email.account_id) { + if !filter.rule.matches_account(&placed.placement.account_id) { continue; } let filter_for_task = filter.clone(); - let email_for_task = email.clone(); + let email_for_task = placed.email.clone(); + let placement_for_task = placed.placement.clone(); let eval_task = tokio::task::spawn_blocking(move || { - filter_for_task.evaluate_condition(&email_for_task) + filter_for_task.evaluate_condition(PlacedEmail { + email: &email_for_task, + placement: &placement_for_task, + }) }); match tokio::time::timeout(timeout_duration, eval_task).await { @@ -370,7 +411,7 @@ impl FilterEngine { /// Rules whose account scope does not match the message are reported as /// `SKIPPED (account scope mismatch)` rather than omitted, so the trace /// distinguishes "condition did not match" from "rule does not apply here". - pub fn preview(&self, email: &Email) -> FilterPreviewResult { + pub fn preview(&self, placed: PlacedEmail<'_>) -> FilterPreviewResult { let start = Instant::now(); let guard = self.cache.load(); let mut matched_rule_id = None; @@ -380,7 +421,7 @@ impl FilterEngine { let mut matched = false; for filter in &guard.filters { - if !filter.rule.matches_account(&email.account_id) { + if !filter.rule.matches_account(&placed.placement.account_id) { traces.push(format!( "Rule '{}' (priority {}): SKIPPED (account scope mismatch)", filter.rule.name, filter.rule.priority @@ -388,7 +429,7 @@ impl FilterEngine { continue; } - let is_match = filter.evaluate_condition(email); + let is_match = filter.evaluate_condition(placed); traces.push(format!( "Rule '{}' (priority {}): {}", filter.rule.name, @@ -406,7 +447,7 @@ impl FilterEngine { let elapsed = start.elapsed().as_micros() as u64; FilterPreviewResult { - message_id: email.id.clone(), + message_id: placed.email.id.clone(), matched, matched_rule_id, matched_rule_name, @@ -435,22 +476,31 @@ mod tests { use super::*; use nuncio_core::model::Email; - fn test_email(account_id: &str, subject: &str, folder_id: &str) -> Email { - Email { + /// Build an `Email`/`Placement` pair that shares one account between the + /// message and the mailbox it is placed in, matching how a message and + /// its own placement always agree on account. + fn test_placed_email(account_id: &str, subject: &str, folder_id: &str) -> (Email, Placement) { + let email = Email { id: "msg-1".to_string(), account_id: account_id.to_string(), - folder_id: folder_id.to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), - } + message_id: None, + content_hash: None, + }; + let placement = Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: "1".to_string(), + read: false, + }; + (email, placement) } #[test] @@ -459,27 +509,19 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Urgent", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let email = Email { - id: "msg-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "Urgent Meeting".to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1700000000, - read: false, - body_plain: Some("Hello".to_string()), - body_html: None, - attachments: Vec::new(), - }; + let (email, placement) = test_placed_email("acct-1", "Urgent Meeting", "inbox"); - let results = engine.evaluate(&email); + let results = engine.evaluate(PlacedEmail { + email: &email, + placement: &placement, + }); assert_eq!(results.len(), 1); assert_eq!(results[0].1.len(), 2); - let preview = engine.preview(&email); + let preview = engine.preview(PlacedEmail { + email: &email, + placement: &placement, + }); assert!(preview.matched); assert_eq!(preview.matched_rule_name, Some("Urgent".to_string())); } @@ -499,14 +541,27 @@ mod tests { // Condition matches the subject, but the message belongs to a // different account than the rule was created for. - let email_b = test_email("acct-b", "Weekly Report", "inbox"); + let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox"); assert!( - engine.evaluate(&email_b).is_empty(), + engine + .evaluate(PlacedEmail { + email: &email_b, + placement: &placement_b + }) + .is_empty(), "rule scoped to acct-a must not match a message from acct-b" ); - let email_a = test_email("acct-a", "Weekly Report", "inbox"); - assert_eq!(engine.evaluate(&email_a).len(), 1); + let (email_a, placement_a) = test_placed_email("acct-a", "Weekly Report", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email_a, + placement: &placement_a + }) + .len(), + 1 + ); } #[tokio::test] @@ -515,15 +570,27 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Scoped To A", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let email_b = test_email("acct-b", "Weekly Report", "inbox"); + let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox"); let results = engine - .evaluate_with_timeout(&email_b, Duration::from_secs(1)) + .evaluate_with_timeout( + PlacedEmail { + email: &email_b, + placement: &placement_b, + }, + Duration::from_secs(1), + ) .await; assert!(results.is_empty()); - let email_a = test_email("acct-a", "Weekly Report", "inbox"); + let (email_a, placement_a) = test_placed_email("acct-a", "Weekly Report", "inbox"); let results = engine - .evaluate_with_timeout(&email_a, Duration::from_secs(1)) + .evaluate_with_timeout( + PlacedEmail { + email: &email_a, + placement: &placement_a, + }, + Duration::from_secs(1), + ) .await; assert_eq!(results.len(), 1); } @@ -534,8 +601,11 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Scoped To A", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let email_b = test_email("acct-b", "Weekly Report", "inbox"); - let preview = engine.preview(&email_b); + let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox"); + let preview = engine.preview(PlacedEmail { + email: &email_b, + placement: &placement_b, + }); assert!(!preview.matched); assert!(preview .condition_traces @@ -549,8 +619,16 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Global", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let email = test_email("any-account", "Weekly Report", "inbox"); - assert_eq!(engine.evaluate(&email).len(), 1); + let (email, placement) = test_placed_email("any-account", "Weekly Report", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &placement + }) + .len(), + 1 + ); } #[test] @@ -560,20 +638,30 @@ mod tests { let engine = FilterEngine::new(vec![rule]).unwrap(); // Same folder_id, different account_id: a rule matching on the - // `account` condition field must key off email.account_id, never - // fall through to comparing folder_id. - let email_a = test_email("acct-a", "Anything", "shared-folder"); + // `account` condition field must key off the placement's account_id, + // never fall through to comparing folder_id. + let (email_a, placement_a) = test_placed_email("acct-a", "Anything", "shared-folder"); assert_eq!( - engine.evaluate(&email_a).len(), + engine + .evaluate(PlacedEmail { + email: &email_a, + placement: &placement_a + }) + .len(), 1, - "rule on account = 'acct-a' must match a message with account_id 'acct-a'" + "rule on account = 'acct-a' must match a placement with account_id 'acct-a'" ); - let email_b = test_email("acct-b", "Anything", "shared-folder"); + let (email_b, placement_b) = test_placed_email("acct-b", "Anything", "shared-folder"); assert!( - engine.evaluate(&email_b).is_empty(), - "rule on account = 'acct-a' must not match a message with account_id 'acct-b', \ - even when it shares the same folder_id as an acct-a message" + engine + .evaluate(PlacedEmail { + email: &email_b, + placement: &placement_b + }) + .is_empty(), + "rule on account = 'acct-a' must not match a placement with account_id 'acct-b', \ + even when it shares the same folder_id as an acct-a placement" ); } @@ -583,11 +671,24 @@ mod tests { let rule = crate::parser::NsqlParser::parse_rule("Not In Spam Or Trash", 1, nsql).unwrap(); let engine = FilterEngine::new(vec![rule]).unwrap(); - let inbox_email = test_email("acct-1", "Anything", "inbox"); - assert_eq!(engine.evaluate(&inbox_email).len(), 1); + let (inbox_email, inbox_placement) = test_placed_email("acct-1", "Anything", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &inbox_email, + placement: &inbox_placement + }) + .len(), + 1 + ); - let spam_email = test_email("acct-1", "Anything", "spam"); - assert!(engine.evaluate(&spam_email).is_empty()); + let (spam_email, spam_placement) = test_placed_email("acct-1", "Anything", "spam"); + assert!(engine + .evaluate(PlacedEmail { + email: &spam_email, + placement: &spam_placement + }) + .is_empty()); } #[test] @@ -612,8 +713,16 @@ mod tests { }; let engine = FilterEngine::new(vec![rule]).unwrap(); - let email = test_email("acct-1", "Anything", "inbox"); - assert_eq!(engine.evaluate(&email).len(), 1); + let (email, placement) = test_placed_email("acct-1", "Anything", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &placement + }) + .len(), + 1 + ); } #[test] @@ -640,8 +749,16 @@ mod tests { }; let engine = FilterEngine::new(vec![rule]).unwrap(); - let email = test_email("acct-1", "Anything", "inbox"); - assert_eq!(engine.evaluate(&email).len(), 1); + let (email, placement) = test_placed_email("acct-1", "Anything", "inbox"); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &placement + }) + .len(), + 1 + ); } #[tokio::test] @@ -672,11 +789,17 @@ mod tests { }; let engine = FilterEngine::new(vec![rule]).unwrap(); - let mut email = test_email("acct-1", "Anything", "inbox"); + let (mut email, placement) = test_placed_email("acct-1", "Anything", "inbox"); email.body_plain = Some("A".repeat(60_000_000)); let results = engine - .evaluate_with_timeout(&email, Duration::from_millis(1)) + .evaluate_with_timeout( + PlacedEmail { + email: &email, + placement: &placement, + }, + Duration::from_millis(1), + ) .await; assert!( @@ -693,4 +816,99 @@ mod tests { "expected a WARN log carrying the rule id for the timed-out evaluation, got: {captured:?}" ); } + + fn sample_email(key: &str) -> Email { + Email { + id: key.to_string(), + account_id: "acct-1".to_string(), + subject: "Anything".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1700000000, + body_plain: Some("Hello".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + } + } + + fn sample_placement(folder_id: &str, remote_id: &str) -> Placement { + Placement { + account_id: "acct-1".to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: remote_id.to_string(), + read: false, + } + } + + fn rule_where_folder_is(folder: &str) -> FilterRule { + let nsql = format!("WHERE folder = '{folder}' ACTION MARK READ"); + crate::parser::NsqlParser::parse_rule("Folder Filter", 1, &nsql).unwrap() + } + + fn rule_where_account_is(account: &str) -> FilterRule { + let nsql = format!("WHERE account = '{account}' ACTION MARK READ"); + crate::parser::NsqlParser::parse_rule("Account Filter", 1, &nsql).unwrap() + } + + #[test] + fn a_folder_condition_matches_only_the_placement_it_is_evaluated_against() { + let engine = FilterEngine::new(vec![rule_where_folder_is("INBOX")]).unwrap(); + let email = sample_email("key-1"); + let inbox = sample_placement("INBOX", "5"); + let archive = sample_placement("Archive", "9"); + + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &inbox + }) + .len(), + 1, + "the INBOX placement matches" + ); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &archive + }) + .len(), + 0, + "the same message's Archive placement does not" + ); + } + + #[test] + fn an_account_condition_reads_the_placement_account() { + let engine = FilterEngine::new(vec![rule_where_account_is("acct-1")]).unwrap(); + let email = sample_email("key-1"); + let mine = sample_placement("INBOX", "5"); + let theirs = Placement { + account_id: "acct-2".into(), + ..sample_placement("INBOX", "5") + }; + + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &mine + }) + .len(), + 1 + ); + assert_eq!( + engine + .evaluate(PlacedEmail { + email: &email, + placement: &theirs + }) + .len(), + 0 + ); + } } diff --git a/crates/nuncio-filter/src/validator.rs b/crates/nuncio-filter/src/validator.rs index 37a058d..8351c78 100644 --- a/crates/nuncio-filter/src/validator.rs +++ b/crates/nuncio-filter/src/validator.rs @@ -92,6 +92,10 @@ impl NsqlValidator { "header field conditions are not yet supported: message headers are not indexed" .to_string(), )), + // `Folder` and `Account` are validated the same as the + // content fields here: only the value's shape (string or + // string list) matters at this pass. At evaluation time they + // read a placement's coordinates, not the message's content. FilterField::Subject | FilterField::From | FilterField::To diff --git a/crates/nuncio-mail/src/backend.rs b/crates/nuncio-mail/src/backend.rs index f25e9bf..00e65cd 100644 --- a/crates/nuncio-mail/src/backend.rs +++ b/crates/nuncio-mail/src/backend.rs @@ -1,10 +1,28 @@ //! Protocol-agnostic async mail backend trait definitions. use async_trait::async_trait; -use nuncio_core::model::{Attachment, Email, Folder}; +use nuncio_core::model::{Attachment, Email, Folder, IdentitySource, Placement, PlacementKey}; use crate::parser::MailError; +/// One message as a backend surfaced it: identity, content, and the mailbox +/// occupancy it was found in. +/// +/// Backends emit this rather than a bare [`Email`] because a fetch observes two +/// separate facts at once: *which message this is* -- folder-independent, and +/// the same in every mailbox that holds a copy -- and *where this pass found +/// it*, which is per-mailbox and changes under a move. Collapsing them into one +/// record is what made a moved message read as a new one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlacedMessage { + /// Identity and content. `email.id` is the derived message key. + pub email: Email, + /// Which precedence tier produced `email.id`. + pub source: IdentitySource, + /// The occupancy this fetch found it in. + pub placement: Placement, +} + /// The remote change a [`RemoteMutationSpec`] applies to one addressed message. /// /// This is the closed vocabulary of genuinely-remote filter actions that the @@ -72,21 +90,28 @@ pub struct RemoteMutationSpec { /// or deleted by any other client -- another Nuncio daemon, a phone, webmail -- /// simply stays in the local store forever. Expressing absence is the whole /// point of the type. +/// +/// Absence is reported as [`PlacementKey`]s, not message keys, because what a +/// folder stops mentioning is an *occupancy*. The same message may still sit in +/// another mailbox, and deleting the message on the strength of one folder's +/// silence would destroy a copy the user still has. A message goes only when +/// its last placement does. #[derive(Debug, Default, Clone)] pub struct FolderChanges { - /// Messages to store, whether newly arrived or changed. - pub upserts: Vec, - /// Surrogate ids the server **explicitly reported as gone**, via QRESYNC + /// Messages to store, whether newly arrived or changed, each with the + /// occupancy this pass found it in. + pub upserts: Vec, + /// Occupancies the server **explicitly reported as gone**, via QRESYNC /// `VANISHED`. Always safe to delete. - pub removals: Vec, - /// Surrogate ids of everything the folder currently holds, when this pass - /// enumerated the complete UID set. The caller may delete anything it - /// stores for this folder that is absent from this list. + pub removals: Vec, + /// Every occupancy the folder currently holds, when this pass enumerated + /// the complete UID set. The caller may delete any placement it stores for + /// this folder that is absent from this list. /// /// `None` and `Some(vec![])` mean different things, and conflating them /// deletes a mailbox. `None` is "this pass was incremental and cannot /// speak to absence"; `Some(vec![])` is "the folder is genuinely empty". - pub present: Option>, + pub present: Option>, /// The checkpoint to resume from next time. pub next_state: String, } @@ -158,11 +183,16 @@ pub trait MailBackend: Send + Sync { /// /// Provided rather than required so no implementation can satisfy the /// trait by supplying this and quietly never reporting a removal. + /// + /// Every message comes back as a [`PlacedMessage`] so the caller can tell + /// a message it already stores in another folder from one it has never + /// seen: the derived key is the same in both folders, only the placement + /// differs. async fn sync_messages( &self, folder_id: &str, since_state: Option<&str>, - ) -> Result<(Vec, String), MailError> { + ) -> Result<(Vec, String), MailError> { let changes = self.sync_changes(folder_id, since_state).await?; Ok((changes.upserts, changes.next_state)) } diff --git a/crates/nuncio-mail/src/imap.rs b/crates/nuncio-mail/src/imap.rs index ac273d7..fe93558 100644 --- a/crates/nuncio-mail/src/imap.rs +++ b/crates/nuncio-mail/src/imap.rs @@ -1,7 +1,7 @@ //! IMAP4rev1 dual-socket connection engine and IDLE push stream manager. use async_trait::async_trait; -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, MessageIdentity, Placement, PlacementKey, RemoteIdentity}; use nuncio_core::TlsMode; use std::fmt::Debug; use std::pin::Pin; @@ -52,7 +52,10 @@ const MAX_MESSAGES_PER_FETCH_BATCH: usize = 200; /// would ever accept is pointless to transfer only to reject. const MAX_MESSAGE_BODY_BYTES: u32 = 25 * 1024 * 1024; -use crate::backend::{MailBackend, MutationOutcome, RemoteMutationKind, RemoteMutationSpec}; +use crate::backend::{ + FolderChanges, MailBackend, MutationOutcome, PlacedMessage, RemoteMutationKind, + RemoteMutationSpec, +}; use crate::parser::{MailError, MimeParserAdapter}; /// Recover a raw IMAP UID from a [`RemoteMutationSpec::remote_id`] (a decimal @@ -783,7 +786,8 @@ impl ImapEngine { } /// Fetch and parse the messages in a folder over an active IMAP session, - /// returning the fetched messages and the new sync checkpoint. + /// returning each message with the occupancy it was found in, plus the new + /// sync checkpoint. /// /// `since_state` is the checkpoint returned by a previous sync of this /// folder, encoded as `"{uidvalidity}:{uidnext}"` (see [`CHECKPOINT_DELIM`]). @@ -799,7 +803,7 @@ impl ImapEngine { folder_id: &str, since_state: Option<&str>, session: &mut async_imap::Session, - ) -> Result + ) -> Result where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -818,15 +822,21 @@ impl ImapEngine { /// exercise the multi-batch path with a small window instead of a huge /// fixture. /// - /// The sync runs in two phases so the transient working set stays bounded - /// regardless of mailbox size: + /// The sync runs in two phases: /// 1. A cheap enumeration ([`build_enumerate_command_query`]) reads every /// in-range message's UID and `RFC822.SIZE` with NO body. Messages at or /// below [`MAX_MESSAGE_BODY_BYTES`] are queued for a full-body fetch; /// oversized ones are queued for a headers-only fetch and WARN-logged. /// 2. Each queue is drained in explicit UID-set batches of at most - /// `batch_size`, so the full bodies held in memory at once never exceed - /// one batch. + /// `batch_size`, so a single `UID FETCH` request never asks the server + /// for more than that many bodies at a time. + /// + /// Batching bounds the size of each *request*, not the memory this function + /// holds: the parsed messages accumulate into one `Vec` across every batch, + /// so the returned [`FolderChanges`] carries the whole in-range set and the + /// peak working set still scales with the mailbox. That is the shape a + /// caller receiving `FolderChanges` depends on, so anything downstream must + /// be sized for a whole folder rather than a batch. #[tracing::instrument( skip(self, session), fields(account_id = %self.account_id, folder_id = %folder_id), @@ -838,7 +848,7 @@ impl ImapEngine { since_state: Option<&str>, session: &mut async_imap::Session, batch_size: usize, - ) -> Result + ) -> Result where S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static, { @@ -968,16 +978,44 @@ impl ImapEngine { .unwrap_or_else(|| "full-resync-required".to_string()), }; - let removals: Vec = plan - .vanished_uids - .iter() - .map(|uid| self.surrogate_for(folder_id, server_uid_validity, *uid)) - .collect(); - let present = plan.present_uids.as_ref().map(|uids| { - uids.iter() - .map(|uid| self.surrogate_for(folder_id, server_uid_validity, *uid)) - .collect() - }); + // Absence can only be reported against a known UIDVALIDITY generation. + // + // `[UIDVALIDITY]` is REQUIRED on a successful SELECT (RFC 3501 section + // 6.3.1), so this should not happen -- but a server that omits it would + // otherwise produce keys carrying an empty generation, and NONE of them + // would match placements stored under a real one. Under the documented + // contract ("delete any placement absent from `present`") that reads as + // "the folder holds nothing", and the caller deletes the whole mailbox. + // Saying nothing is the only safe answer, and it mirrors the checkpoint + // above, which likewise refuses to write a validity-less value. + let (removals, present) = match server_uid_validity { + Some(_) => { + // Both are occupancies, not messages: a UID that vanished from + // this folder says nothing about copies of the same message + // elsewhere. + let removals: Vec = plan + .vanished_uids + .iter() + .map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid)) + .collect(); + let present = plan.present_uids.as_ref().map(|uids| { + uids.iter() + .map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid)) + .collect() + }); + (removals, present) + } + None => { + tracing::warn!( + folder_id, + vanished = plan.vanished_uids.len(), + enumerated = plan.present_uids.as_ref().map_or(0, Vec::len), + "mailbox reported no UIDVALIDITY, so this pass cannot key an occupancy and \ + reports no absence; upserts are still surfaced" + ); + (Vec::new(), None) + } + }; tracing::info!( account_id = %self.account_id, @@ -985,13 +1023,13 @@ impl ImapEngine { rung = ?plan.rung, fetched = emails.len(), removed = removals.len(), - enumerated = plan.present_uids.as_ref().map_or(0, Vec::len), + enumerated = present.as_ref().map_or(0, Vec::len), oversized = oversized_uids.len(), skipped = skipped_no_uid, "imap folder sync complete" ); - Ok(crate::backend::FolderChanges { + Ok(FolderChanges { upserts: emails, removals, present, @@ -1179,45 +1217,126 @@ impl ImapEngine { Ok(uids) } - /// Surrogate id for a UID in this account/folder/UIDVALIDITY scope, so - /// removals name messages the same way stored rows do. - fn surrogate_for(&self, folder_id: &str, uid_validity: Option, uid: u32) -> String { - let validity = uid_validity.map(|v| v.to_string()).unwrap_or_default(); - Email::surrogate_id(&self.account_id, folder_id, &validity, &uid.to_string()) + /// The placement key a UID addresses in this account/folder/UIDVALIDITY + /// scope, so absence reports name occupancies the same way stored rows do. + /// + /// Deliberately NOT a message key. A UID is only ever evidence about one + /// mailbox: `VANISHED` says this folder no longer holds this UID, not that + /// the message is gone -- another mailbox may still hold a copy. It also + /// could not be a message key even in principle, because the key a message + /// is stored under depends on which identity tier the server supported, + /// and a UID cannot reconstruct that. + fn placement_key_for( + &self, + folder_id: &str, + uid_validity: Option, + uid: u32, + ) -> PlacementKey { + PlacementKey { + account_id: self.account_id.clone(), + folder_id: folder_id.to_string(), + uid_validity: uid_validity.map(|v| v.to_string()).unwrap_or_default(), + remote_id: uid.to_string(), + } } - /// Build an [`Email`] from a full-body (or envelope-only) FETCH item. When - /// the item carries a `BODY[]` it is parsed as a full MIME message; when it - /// does not (a server that returned only the ENVELOPE), a minimal record is - /// synthesised from the envelope so the message is still surfaced honestly. + /// Build the identity and placement for one fetched message. + /// + /// `EMAILID` and `X-GM-MSGID` are only present when the server advertised + /// `OBJECTID` / `X-GM-EXT-1` and the FETCH asked for them; when neither is + /// available the pair of `Message-ID` and content hash carries the identity, + /// and only if both are missing does this fall back to the folder-scoped + /// surrogate -- which is the one tier that does not survive a move. + /// + /// The server-supplied hints arrive as one [`RemoteIdentity`] rather than as + /// four loose arguments: they are already a single value at every call site, + /// and spreading them out buys no clarity while pushing this function past + /// the argument-count lint. + /// + /// In practice every IMAP call site passes `email_id: None` and + /// `gm_msgid: None` today, because the FETCH never asks for them, so this + /// engine reaches only the `Message-ID`+content tier or the surrogate. The + /// JMAP engine enters at the top tier, which means an IMAP-derived key and + /// a JMAP-derived key for the *same* message will not match. Syncing one + /// account over both engines does not converge on a shared identity yet. + fn place( + &self, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + remote: RemoteIdentity<'_>, + read: bool, + ) -> (MessageIdentity, Placement) { + let identity = + Email::derive_message_key(&self.account_id, remote, folder_id, uid_validity, remote_id); + let placement = Placement { + account_id: self.account_id.clone(), + folder_id: folder_id.to_string(), + uid_validity: uid_validity.to_string(), + remote_id: remote_id.to_string(), + read, + }; + (identity, placement) + } + + /// Build a [`PlacedMessage`] from a full-body (or envelope-only) FETCH item. + /// When the item carries a `BODY[]` it is parsed as a full MIME message; + /// when it does not (a server that returned only the ENVELOPE), a minimal + /// record is synthesised from the envelope so the message is still surfaced + /// honestly. fn build_email_from_fetch( &self, folder_id: &str, server_uid_validity: Option, fetch_data: &async_imap::types::Fetch, - ) -> Result { + ) -> Result { let uid_num = fetch_data.uid.unwrap_or(0); // The message is addressed on the wire by its UID within the folder's - // UIDVALIDITY scope; the persisted id is an opaque surrogate hashed over - // both plus the account and folder, so the same UID in another - // folder/account can never collide. + // UIDVALIDITY scope. That pair, with the folder and account, is the + // *placement*; the message's own identity is derived separately so that + // moving it between folders does not make it a different message. let remote_id = uid_num.to_string(); let uid_validity = server_uid_validity .map(|v| v.to_string()) .unwrap_or_default(); - let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id); let is_read = fetch_data .flags() .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); + // EMAILID (RFC 8474) and X-GM-MSGID are not requested by the current + // FETCH, so identity rests on the Message-ID/content pair where the + // message carried a Message-ID and on the folder-scoped surrogate + // otherwise. + // + // Adding those FETCH items is not a local change. Every already-stored + // message would derive a DIFFERENT key on its next resync, so each + // re-reported placement repoints to the new key -- and the old message + // row, now referenced by nothing, is never reaped. Its decrypted body + // stays in the plaintext FTS index forever, which is exactly the leak + // the reap on placement removal exists to prevent. The FETCH items can + // only be requested once the repoint path reaps the message it + // abandons; until then, requesting them leaks plaintext. if let Some(raw_bytes) = fetch_data.body() { - let mut email = - MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, raw_bytes)?; - email.remote_id = remote_id; - email.uid_validity = uid_validity; - email.read = is_read; - return Ok(email); + let mut email = MimeParserAdapter::parse_mime(&self.account_id, raw_bytes)?; + let (identity, placement) = self.place( + folder_id, + &uid_validity, + &remote_id, + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: email.message_id.as_deref(), + content_hash: email.content_hash.as_deref(), + }, + is_read, + ); + email.id = identity.key; + return Ok(PlacedMessage { + email, + source: identity.source, + placement, + }); } let subject = fetch_data @@ -1241,24 +1360,51 @@ impl ImapEngine { let received_at = fetch_data.internal_date().map_or(0, |dt| dt.timestamp()); - Ok(Email { - id: email_id, - account_id: self.account_id.clone(), - folder_id: folder_id.to_string(), - remote_id, - uid_validity, - subject, - sender, - recipient, - received_at, - read: is_read, - body_plain: None, - body_html: None, - attachments: Vec::new(), + // No body was returned, so there are no octets to hash -- but the + // ENVELOPE carries `message-id` (RFC 3501 section 7.4.2), so identity + // capture does not have to be skipped on this path. + let message_id = fetch_data + .envelope() + .and_then(|env| env.message_id.as_ref()) + .and_then(|raw| Email::normalize_message_id(&String::from_utf8_lossy(raw))); + + // With no octets there is no content hash, and the `Message-ID` alone + // is never enough to key on, so this path lands on the surrogate tier. + // That is the honest answer: nothing was fetched that could prove two + // folders hold the same message. + let (identity, placement) = self.place( + folder_id, + &uid_validity, + &remote_id, + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: message_id.as_deref(), + content_hash: None, + }, + is_read, + ); + + Ok(PlacedMessage { + email: Email { + id: identity.key, + account_id: self.account_id.clone(), + subject, + sender, + recipient, + received_at, + body_plain: None, + body_html: None, + attachments: Vec::new(), + message_id, + content_hash: None, + }, + source: identity.source, + placement, }) } - /// Build an [`Email`] for a message whose body exceeded + /// Build a [`PlacedMessage`] for a message whose body exceeded /// [`MAX_MESSAGE_BODY_BYTES`]: its headers (`BODY[HEADER]`) are parsed for /// honest metadata (subject/sender/date) but the oversized body is left /// unfetched, so the record carries an EMPTY body rather than a fabricated @@ -1269,30 +1415,50 @@ impl ImapEngine { folder_id: &str, server_uid_validity: Option, fetch_data: &async_imap::types::Fetch, - ) -> Email { + ) -> PlacedMessage { let uid_num = fetch_data.uid.unwrap_or(0); let remote_id = uid_num.to_string(); let uid_validity = server_uid_validity .map(|v| v.to_string()) .unwrap_or_default(); - let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id); let is_read = fetch_data .flags() .any(|flag| matches!(flag, async_imap::types::Flag::Seen)); if let Some(header_bytes) = fetch_data.header() { - if let Ok(mut email) = - MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, header_bytes) - { + if let Ok(mut email) = MimeParserAdapter::parse_mime(&self.account_id, header_bytes) { // A headers-only parse yields no body, but be explicit that the // oversized body was intentionally not fetched. - email.remote_id = remote_id; - email.uid_validity = uid_validity; - email.read = is_read; email.body_plain = None; email.body_html = None; email.attachments = Vec::new(); - return email; + // `parse_mime` hashed the header section it was handed, but + // `content_hash` is defined over a message's FULL octets. + // Keeping that value would publish a hash that silently means + // something else, so drop it; the `Message-ID` it recovered + // from those headers is still correct and is kept. Dropping it + // also costs this message the content tier -- a `Message-ID` + // never keys on its own -- so an oversized message identifies + // by its folder-scoped surrogate until the octets are fetched. + email.content_hash = None; + let (identity, placement) = self.place( + folder_id, + &uid_validity, + &remote_id, + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: email.message_id.as_deref(), + content_hash: None, + }, + is_read, + ); + email.id = identity.key; + return PlacedMessage { + email, + source: identity.source, + placement, + }; } } @@ -1300,20 +1466,31 @@ impl ImapEngine { // `build_email_from_fetch` cannot error on this path (no body to parse), // but if it ever did, surface a minimal honest record rather than panic. self.build_email_from_fetch(folder_id, server_uid_validity, fetch_data) - .unwrap_or_else(|_| Email { - id: email_id, - account_id: self.account_id.clone(), - folder_id: folder_id.to_string(), - remote_id, - uid_validity, - subject: "No Subject".to_string(), - sender: "unknown@nuncio.mx".to_string(), - recipient: "me@nuncio.mx".to_string(), - received_at: 0, - read: is_read, - body_plain: None, - body_html: None, - attachments: Vec::new(), + .unwrap_or_else(|_| { + let (identity, placement) = self.place( + folder_id, + &uid_validity, + &remote_id, + RemoteIdentity::default(), + is_read, + ); + PlacedMessage { + email: Email { + id: identity.key, + account_id: self.account_id.clone(), + subject: "No Subject".to_string(), + sender: "unknown@nuncio.mx".to_string(), + recipient: "me@nuncio.mx".to_string(), + received_at: 0, + body_plain: None, + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + source: identity.source, + placement, + } }) } @@ -1325,7 +1502,7 @@ impl ImapEngine { &self, folder_id: &str, since_state: Option<&str>, - ) -> Result { + ) -> Result { let (username, password) = match (&self.username, &self.password) { (Some(u), Some(p)) => (u.as_str(), p.as_str()), _ => { @@ -1793,7 +1970,7 @@ impl MailBackend for ImapEngine { &self, folder_id: &str, since_state: Option<&str>, - ) -> Result { + ) -> Result { self.sync_folder_messages(folder_id, since_state).await } @@ -1808,6 +1985,7 @@ impl MailBackend for ImapEngine { #[cfg(test)] mod tests { use super::*; + use nuncio_core::model::IdentitySource; use tracing_test::traced_test; #[tokio::test] @@ -2193,17 +2371,25 @@ mod tests { } /// The RFC822 body of a scripted fixture message. + /// + /// Carries a `Message-ID` so the full-body path reaches the content tier. + /// That is what makes the headers-only path's tier meaningful to assert: + /// the same message identifies differently there, and only because the + /// header-section hash is deliberately discarded. fn fixture_body(uid: u32) -> String { format!( "Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\ + Message-ID: \r\n\ To: me@example.test\r\n\r\nBody of message {uid}.\r\n" ) } - /// The header block (no body) of a scripted fixture message. + /// The header block (no body) of a scripted fixture message. Same headers + /// as [`fixture_body`], including the `Message-ID`. fn fixture_header(uid: u32) -> String { format!( "Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\ + Message-ID: \r\n\ To: me@example.test\r\n\r\n" ) } @@ -2237,13 +2423,17 @@ mod tests { /// and records every `UID FETCH` command line the client issued so a test /// can assert on the exact batching. Returns the synced emails, the new /// checkpoint, and the captured `UID FETCH` command lines. + /// + /// `uidvalidity` is an `Option` so a test can script the one thing RFC 3501 + /// section 6.3.1 says a SELECT must never do -- omit `[UIDVALIDITY]` -- and + /// check the sync stays safe when a server does it anyway. async fn run_batched_sync( messages: Vec<(u32, u32)>, - uidvalidity: u32, + uidvalidity: Option, uidnext: u32, since_state: Option, batch_size: usize, - ) -> (Vec, String, Vec) { + ) -> (FolderChanges, Vec) { let (client_io, server_io) = tokio::io::duplex(65536); let fetch_cmds = Arc::new(std::sync::Mutex::new(Vec::::new())); let server_cmds = fetch_cmds.clone(); @@ -2305,9 +2495,13 @@ mod tests { .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) .await; } else if upper.contains("SELECT") { + let validity_line = match uidvalidity { + Some(v) => format!("* OK [UIDVALIDITY {v}] ok\r\n"), + None => String::new(), + }; let resp = format!( "* FLAGS (\\Seen)\r\n* {exists} EXISTS\r\n* 0 RECENT\r\n\ - * OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\ + {validity_line}* OK [UIDNEXT {uidnext}] ok\r\n\ {tag} OK [READ-WRITE] SELECT done\r\n" ); let _ = io.write_all(resp.as_bytes()).await; @@ -2331,12 +2525,11 @@ mod tests { .sync_folder_messages_batched("INBOX", since_state.as_deref(), &mut session, batch_size) .await .expect("batched sync succeeds"); - let (emails, checkpoint) = (changes.upserts, changes.next_state); drop(session); let _ = server.await; let cmds = fetch_cmds.lock().expect("lock captured commands").clone(); - (emails, checkpoint, cmds) + (changes, cmds) } /// The captured `UID FETCH` command lines that pulled FULL bodies (a @@ -2358,10 +2551,11 @@ mod tests { // `1:*` body fetch. The enumeration discovers the UIDs; the bodies are // then fetched in explicit UID-set chunks of at most the batch size. let messages: Vec<(u32, u32)> = (1..=5).map(|u| (u, 100)).collect(); - let (emails, checkpoint, cmds) = run_batched_sync(messages, 1, 6, None, 2).await; + let (changes, cmds) = run_batched_sync(messages, Some(1), 6, None, 2).await; + let emails = &changes.upserts; assert_eq!(emails.len(), 5, "every message must be synced"); - assert_eq!(checkpoint, "v1:uidnext:1:6"); + assert_eq!(changes.next_state, "v1:uidnext:1:6"); let body_cmds = body_fetch_commands(&cmds); assert_eq!( @@ -2391,7 +2585,8 @@ mod tests { // normal messages get full bodies. let huge = MAX_MESSAGE_BODY_BYTES + 1; let messages: Vec<(u32, u32)> = vec![(1, 100), (2, huge), (3, 100)]; - let (emails, _checkpoint, cmds) = run_batched_sync(messages, 1, 4, None, 200).await; + let (changes, cmds) = run_batched_sync(messages, Some(1), 4, None, 200).await; + let emails = &changes.upserts; assert_eq!(emails.len(), 3, "no message may be silently dropped"); @@ -2416,27 +2611,54 @@ mod tests { // body -- never a fabricated one. let oversized = emails .iter() - .find(|e| e.remote_id == "2") + .find(|e| e.placement.remote_id == "2") .expect("oversized message must be present"); assert!( - oversized.body_plain.is_none() && oversized.body_html.is_none(), + oversized.email.body_plain.is_none() && oversized.email.body_html.is_none(), "oversized message must carry no body, got: {oversized:?}" ); assert!( - oversized.subject.contains('2'), + oversized.email.subject.contains('2'), "oversized message must keep honest header metadata, got: {oversized:?}" ); + // The header section it WAS given hashes to something, but that hash + // covers only the headers, not the message. Publishing it would key the + // message on the wrong octets -- and then the same message would key + // differently once its body was finally fetched, resurrecting it as a + // duplicate. So the hash is dropped, which costs this message the + // content tier even though it carried a Message-ID. + assert!( + oversized.email.message_id.is_some(), + "the headers-only parse must still recover the Message-ID" + ); + assert_eq!( + oversized.email.content_hash, None, + "a hash over only the header section must never be published" + ); + assert_eq!( + oversized.source, + IdentitySource::Surrogate, + "without octets to hash, a Message-ID alone must not reach the content tier" + ); + // The normal messages DID get full bodies. for uid in ["1", "3"] { let email = emails .iter() - .find(|e| e.remote_id == uid) + .find(|e| e.placement.remote_id == uid) .expect("normal message present"); assert!( - email.body_plain.is_some(), + email.email.body_plain.is_some(), "normal message {uid} must have a body, got: {email:?}" ); + // The contrast that gives the assertion above its force: the same + // headers, plus the octets, DO reach the content tier. + assert_eq!( + email.source, + IdentitySource::MessageIdContent, + "a full-body fetch with a Message-ID must reach the content tier" + ); } } @@ -2447,24 +2669,24 @@ mod tests { // checkpoint enumerates only the narrowed `{uidnext}:*` range (so a // fire-once-per-new-message consumer never re-sees old mail). let messages: Vec<(u32, u32)> = (1..=3).map(|u| (u, 100)).collect(); - let (first, checkpoint, first_cmds) = - run_batched_sync(messages.clone(), 1, 4, None, 2).await; - assert_eq!(first.len(), 3); + let (first, first_cmds) = run_batched_sync(messages.clone(), Some(1), 4, None, 2).await; + let checkpoint = first.next_state.clone(); + assert_eq!(first.upserts.len(), 3); assert_eq!(checkpoint, "v1:uidnext:1:4"); assert!( first_cmds.iter().any(|c| c.contains("1:*")), "first sync enumerates the full range, got: {first_cmds:?}" ); - let (second, checkpoint2, second_cmds) = - run_batched_sync(messages, 1, 4, Some(checkpoint.clone()), 2).await; + let (second, second_cmds) = + run_batched_sync(messages, Some(1), 4, Some(checkpoint.clone()), 2).await; assert_eq!( - second.len(), + second.upserts.len(), 0, "no new mail arrived, so an incremental sync returns nothing" ); assert_eq!( - checkpoint2, "v1:uidnext:1:4", + second.next_state, "v1:uidnext:1:4", "checkpoint stays stable across an empty pass" ); let second_bodies = body_fetch_ranges(&second_cmds); @@ -3258,4 +3480,259 @@ mod tests { ); Ok(()) } + + /// Drive a folder sync over a scripted server that serves exactly ONE + /// message: `uid`, with `raw` as its full RFC822 octets and `seen` + /// controlling its `\Seen` flag. Holding the octets fixed while varying the + /// folder, UIDVALIDITY and UID is the shape of "the same message, found in + /// two mailboxes" -- which is what the identity/placement split exists for. + /// + /// Passing `raw = None` scripts the envelope-only path instead: the server + /// answers the body fetch with an `ENVELOPE` and no `BODY[]`, which is what + /// a server that will not hand over octets looks like. + async fn run_single_message_sync( + folder_id: &str, + uidvalidity: u32, + uid: u32, + seen: bool, + raw: Option<&str>, + ) -> PlacedMessage { + let (client_io, server_io) = tokio::io::duplex(65536); + let raw = raw.map(str::to_string); + let uidnext = uid.saturating_add(1); + let flags = if seen { "\\Seen" } else { "" }; + + let server = tokio::spawn(async move { + let mut io = server_io; + while let Some(line) = read_scripted_line(&mut io).await { + let tag = line.split_whitespace().next().unwrap_or("").to_string(); + let upper = line.to_ascii_uppercase(); + if upper.contains("UID FETCH") { + let size = raw.as_ref().map_or(64, String::len); + let resp = if !upper.contains("BODY.PEEK[]") { + // Enumeration pass: UID + size only, no body. + format!( + "* 1 FETCH (UID {uid} RFC822.SIZE {size})\r\n\ + {tag} OK FETCH completed\r\n" + ) + } else if let Some(raw) = raw.as_ref() { + format!( + "* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \ + BODY[] {{{size}}}\r\n{raw})\r\n{tag} OK FETCH completed\r\n" + ) + } else { + // A server that answers with metadata but no octets. + // ENVELOPE still carries `message-id` (RFC 3501 + // section 7.4.2), so identity capture is not skipped -- + // but there is nothing to hash. + format!( + "* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \ + ENVELOPE (\"Mon, 1 Jan 2024 00:00:00 +0000\" \"Env subject\" \ + ((\"A\" NIL \"alice\" \"b.example\")) \ + ((\"A\" NIL \"alice\" \"b.example\")) \ + ((\"A\" NIL \"alice\" \"b.example\")) \ + ((\"B\" NIL \"bob\" \"b.example\")) \ + NIL NIL NIL \"\"))\r\n\ + {tag} OK FETCH completed\r\n" + ) + }; + let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("CAPABILITY") { + // The planner negotiates capabilities before selecting; a + // script that never answers this blocks the client forever. + let _ = io + .write_all( + format!( + "* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n" + ) + .as_bytes(), + ) + .await; + } else if upper.contains("LOGIN") { + let _ = io + .write_all(format!("{tag} OK Logged in\r\n").as_bytes()) + .await; + } else if upper.contains("SELECT") { + let resp = format!( + "* FLAGS (\\Seen)\r\n* 1 EXISTS\r\n* 0 RECENT\r\n\ + * OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\ + {tag} OK [READ-WRITE] SELECT done\r\n" + ); + let _ = io.write_all(resp.as_bytes()).await; + } else if upper.contains("LOGOUT") { + let _ = io + .write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes()) + .await; + break; + } + } + }); + + let client = async_imap::Client::new(client_io); + let mut session = client + .login("user", "pass") + .await + .map_err(|(e, _)| e) + .expect("scripted login succeeds"); + let engine = ImapEngine::new("acct-1", "example.test", 993); + let changes = engine + .sync_folder_messages_batched(folder_id, None, &mut session, 8) + .await + .expect("scripted sync succeeds"); + drop(session); + let _ = server.await; + + changes + .upserts + .into_iter() + .next() + .expect("the scripted message must be surfaced") + } + + #[tokio::test] + async fn the_imap_backend_gives_one_identity_to_a_message_found_in_two_folders() { + // Byte-identical octets under two different UIDs in two different + // mailboxes -- an IMAP COPY, or a Gmail label. Identity comes from the + // Message-ID/content pair, so the derived key is ONE value while the + // placements stay distinct. Before the split this was two messages. + let raw = "Message-ID: \r\nSubject: Hi\r\n\ + From: alice@b.example\r\n\r\nbody\r\n"; + let inbox = run_single_message_sync("INBOX", 42, 5, true, Some(raw)).await; + let archive = run_single_message_sync("Archive", 77, 9, false, Some(raw)).await; + + assert_eq!(inbox.source, IdentitySource::MessageIdContent); + assert_eq!(archive.source, IdentitySource::MessageIdContent); + assert_eq!( + inbox.email.id, archive.email.id, + "the same octets in two folders must derive one message key" + ); + + assert_eq!(inbox.placement.account_id, "acct-1"); + assert_eq!(inbox.placement.folder_id, "INBOX"); + assert_eq!(inbox.placement.uid_validity, "42"); + assert_eq!(inbox.placement.remote_id, "5"); + assert_eq!(archive.placement.folder_id, "Archive"); + assert_eq!(archive.placement.uid_validity, "77"); + assert_eq!(archive.placement.remote_id, "9"); + + // IMAP `\Seen` is per-mailbox (RFC 3501 section 2.3.2), so read state + // belongs to the occupancy, not to the message. + assert!(inbox.placement.read); + assert!(!archive.placement.read); + } + + #[tokio::test] + async fn a_message_with_no_message_id_stays_on_the_folder_scoped_surrogate() { + // RFC 5322 section 3.6.4 makes Message-ID a SHOULD, so a message may + // carry none. Without it there is no content tier, and the backend must + // report two honest folder-scoped keys rather than claim a stability + // the server gave it no basis for. + let raw = "Subject: No id\r\nFrom: alice@b.example\r\n\r\nbody\r\n"; + let inbox = run_single_message_sync("INBOX", 42, 5, false, Some(raw)).await; + let archive = run_single_message_sync("Archive", 77, 9, false, Some(raw)).await; + + assert_eq!(inbox.source, IdentitySource::Surrogate); + assert_ne!( + inbox.email.id, archive.email.id, + "a surrogate is folder-scoped and must not be claimed to survive a move" + ); + } + + #[tokio::test] + async fn an_envelope_only_fetch_keeps_its_message_id_but_stays_on_the_surrogate() { + // A server that returns metadata and no octets. ENVELOPE carries + // `message-id` (RFC 3501 section 7.4.2) so the header is still + // captured, but there is nothing to hash -- and a Message-ID alone is + // public and forgeable, so it must never key storage by itself. + let placed = run_single_message_sync("INBOX", 42, 5, false, None).await; + + assert_eq!(placed.email.subject, "Env subject"); + assert_eq!(placed.email.message_id.as_deref(), Some("env@b.example")); + assert_eq!( + placed.email.content_hash, None, + "no octets were fetched, so there is nothing honest to hash" + ); + assert_eq!( + placed.source, + IdentitySource::Surrogate, + "an envelope-only fetch must not reach the content tier" + ); + assert_eq!(placed.placement.remote_id, "5"); + assert_eq!(placed.placement.uid_validity, "42"); + } + + #[traced_test] + #[tokio::test] + async fn a_select_without_uidvalidity_reports_no_absence_at_all() { + // RFC 3501 section 6.3.1 makes `[UIDVALIDITY]` REQUIRED on SELECT, so a + // server omitting it is broken -- but the failure mode is a wiped + // mailbox, not a bad sync. Every placement key this pass could build + // would carry an empty generation and match nothing the store holds, + // and under the `present` contract that reads as "the folder is empty". + // The pass must therefore decline to speak about absence, while still + // surfacing what it fetched. + let messages: Vec<(u32, u32)> = (1..=2).map(|u| (u, 100)).collect(); + let (changes, _cmds) = run_batched_sync(messages, None, 3, None, 200).await; + + assert_eq!( + changes.upserts.len(), + 2, + "the messages themselves are still surfaced" + ); + assert_eq!( + changes.present, None, + "a pass that cannot key an occupancy must not claim to know what is present" + ); + assert!( + changes.removals.is_empty(), + "nor may it report a removal, got: {:?}", + changes.removals + ); + // And it must say why, rather than failing silently. + assert!(logs_contain("mailbox reported no UIDVALIDITY")); + + // The same pass against a compliant server DOES report presence, so the + // assertions above are about the missing UIDVALIDITY and nothing else. + let messages: Vec<(u32, u32)> = (1..=2).map(|u| (u, 100)).collect(); + let (compliant, _cmds) = run_batched_sync(messages, Some(7), 3, None, 200).await; + let present = compliant + .present + .expect("a compliant SELECT must let the pass report presence"); + assert_eq!(present.len(), 2); + assert!(present.iter().all(|k| k.uid_validity == "7")); + } + + #[tokio::test] + async fn a_message_fetched_from_two_folders_keeps_one_identity() { + // Both folders return the same Message-ID and the same octets, so the + // content tier must produce one key from two different UIDs. + let raw = b"Message-ID: \r\nSubject: Hi\r\n\r\nbody"; + let hash = Email::content_hash_of(raw); + let inbox = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some(&hash), + }, + "INBOX", + "42", + "5", + ); + let archive = Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some(&hash), + }, + "Archive", + "77", + "9", + ); + assert_eq!(inbox.key, archive.key); + assert_eq!(inbox.source, IdentitySource::MessageIdContent); + } } diff --git a/crates/nuncio-mail/src/jmap.rs b/crates/nuncio-mail/src/jmap.rs index d09d12f..83d5227 100644 --- a/crates/nuncio-mail/src/jmap.rs +++ b/crates/nuncio-mail/src/jmap.rs @@ -1,19 +1,22 @@ //! JMAP (RFC 8620 / RFC 8621) protocol engine, session discovery, and differential update parser. use async_trait::async_trait; -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, Placement, RemoteIdentity}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use crate::backend::{MailBackend, MutationOutcome, RemoteMutationKind, RemoteMutationSpec}; +use crate::backend::{ + FolderChanges, MailBackend, MutationOutcome, PlacedMessage, RemoteMutationKind, + RemoteMutationSpec, +}; use crate::parser::MailError; /// Stand-in for the UIDVALIDITY scope on JMAP, which has no such concept: a -/// JMAP Email object id is already globally stable within its account (RFC -/// 8621), so a fixed sentinel keeps the surrogate-id computation uniform with -/// IMAP without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats -/// this non-numeric value as "no validity to compare" and is never engaged for -/// a JMAP mutation, which addresses messages by object id. +/// JMAP Email object id is already stable within its account (RFC 8621), so a +/// fixed sentinel keeps a placement key the same shape on both protocols +/// without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats this +/// non-numeric value as "no validity to compare" and is never engaged for a +/// JMAP mutation, which addresses messages by object id. const JMAP_UID_VALIDITY_SENTINEL: &str = "jmap"; /// JMAP Session Object (RFC 8620 Section 2). @@ -92,6 +95,10 @@ pub struct JmapEmail { pub received_at: Option, pub is_unread: Option, pub body_snippet: Option, + /// RFC 8621 models `messageId` as a list, because a message may carry more + /// than one `Message-ID` header. The first entry is the one used. + #[serde(default)] + pub message_id: Option>, } /// JMAP email address object. @@ -184,7 +191,7 @@ impl JmapEngine { { "accountId": account_id, "ids": ids, - "properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet"] + "properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet", "messageId"] }, "c1" ] @@ -364,11 +371,12 @@ impl JmapEngine { } } - /// Parse raw JMAP `Email/get` JSON response payload into domain [`Email`] list and new state string. + /// Parse a raw JMAP `Email/get` JSON response payload into a list of + /// [`PlacedMessage`] and the new state string. pub fn parse_email_get_response( &self, raw_json: &str, - ) -> Result<(Vec, String), MailError> { + ) -> Result<(Vec, String), MailError> { let val: Value = serde_json::from_str(raw_json) .map_err(|e| MailError::ParseFailed(format!("invalid JMAP JSON response: {e}")))?; let payload = Self::extract_method_response_payload(&val)?; @@ -394,30 +402,58 @@ impl JmapEngine { .unwrap_or_else(|| "me@nuncio.mx".to_string()); let folder_id = "inbox".to_string(); - // The JMAP object id is the protocol-native id; the persisted - // id is an opaque surrogate hashed over it plus the account, - // folder, and (sentinel) scope so it stays uniform with IMAP. let remote_id = item.id; - let id = Email::surrogate_id( + let message_id = item + .message_id + .as_ref() + .and_then(|ids| ids.first()) + .and_then(|raw| Email::normalize_message_id(raw)); + + // A JMAP Email id is server-assigned and stable across mailboxes + // within the account, which is the same guarantee RFC 8474 + // EMAILID gives, so it enters at the top tier rather than + // through the surrogate. + let identity = Email::derive_message_key( &self.account_id, + RemoteIdentity { + email_id: Some(&remote_id), + gm_msgid: None, + message_id: message_id.as_deref(), + // `Email/get` returns a `bodySnippet`, not the + // message's octets, so there is nothing to hash here + // without an extra download per message. + content_hash: None, + }, &folder_id, JMAP_UID_VALIDITY_SENTINEL, &remote_id, ); - Email { - id, + let placement = Placement { account_id: self.account_id.clone(), folder_id, - remote_id, + // JMAP has no UIDVALIDITY; the sentinel keeps the placement + // key the same shape as IMAP's without implying a guard. uid_validity: JMAP_UID_VALIDITY_SENTINEL.to_string(), - subject: item.subject.unwrap_or_else(|| "No Subject".to_string()), - sender, - recipient, - received_at: item.received_at.unwrap_or(0), + remote_id: remote_id.clone(), read: !item.is_unread.unwrap_or(false), - body_plain: item.body_snippet, - body_html: None, - attachments: Vec::new(), + }; + + PlacedMessage { + email: Email { + id: identity.key, + account_id: self.account_id.clone(), + subject: item.subject.unwrap_or_else(|| "No Subject".to_string()), + sender, + recipient, + received_at: item.received_at.unwrap_or(0), + body_plain: item.body_snippet, + body_html: None, + attachments: Vec::new(), + message_id, + content_hash: None, + }, + source: identity.source, + placement, } }) .collect(); @@ -565,7 +601,7 @@ impl MailBackend for JmapEngine { &self, folder_id: &str, _since_state: Option<&str>, - ) -> Result { + ) -> Result { if !self.has_credentials() { return Err(MailError::AuthError( "JMAP message sync requires credentials".to_string(), @@ -589,7 +625,7 @@ impl MailBackend for JmapEngine { "jmap message sync complete" ); - Ok(crate::backend::FolderChanges { + Ok(FolderChanges { upserts: emails, removals: Vec::new(), // `Email/query` was filtered to one mailbox, but this engine does @@ -722,6 +758,35 @@ mod tests { assert_eq!(new_state, "s-2"); } + #[test] + fn parse_email_get_response_captures_normalized_message_id() { + // RFC 8621 models `messageId` as a list; the first entry is taken and + // normalized to the same canonical form the IMAP paths produce, so two + // engines syncing one account via different protocols agree. + let raw = r#"{ + "methodResponses": [ + ["Email/get", {"state": "s-1", "list": [ + {"id": "m-1", "subject": "Hi", "messageId": [""]}, + {"id": "m-2", "subject": "No id"} + ]}, "c1"] + ] + }"#; + let engine = JmapEngine::new("acct-1"); + let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get"); + + assert_eq!( + emails[0].email.message_id.as_deref(), + Some("Abc.123@mail.nuncio.mx") + ); + assert_eq!( + emails[1].email.message_id, None, + "an absent messageId must stay None, not become empty" + ); + // `Email/get` returns a snippet, never the octets, so there is nothing + // honest to hash on this path. + assert_eq!(emails[0].email.content_hash, None); + } + #[test] fn parse_invalid_jmap_json_fails() { let engine = JmapEngine::new("acct-1"); diff --git a/crates/nuncio-mail/src/lib.rs b/crates/nuncio-mail/src/lib.rs index 9fc19ad..6155b1d 100644 --- a/crates/nuncio-mail/src/lib.rs +++ b/crates/nuncio-mail/src/lib.rs @@ -18,7 +18,7 @@ pub mod smtp; pub mod test_server; pub use backend::{ - FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage, + FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage, PlacedMessage, RemoteMutationKind, RemoteMutationSpec, }; pub use imap::{IdleSocketState, ImapDualSocketManager, ImapEngine}; diff --git a/crates/nuncio-mail/src/mock.rs b/crates/nuncio-mail/src/mock.rs index f592192..ebcb10b 100644 --- a/crates/nuncio-mail/src/mock.rs +++ b/crates/nuncio-mail/src/mock.rs @@ -1,11 +1,12 @@ //! Deterministic mock mail backend for offline testing and integration verification. use async_trait::async_trait; -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, IdentitySource, Placement, PlacementKey}; use std::sync::{Arc, Mutex}; use crate::backend::{ - MailBackend, MessageSender, MutationOutcome, OutboundMessage, RemoteMutationSpec, + FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage, PlacedMessage, + RemoteMutationSpec, }; use crate::parser::MailError; @@ -13,7 +14,7 @@ use crate::parser::MailError; #[derive(Debug, Clone, Default)] pub struct MockMailBackend { folders: Arc>>, - messages: Arc>>, + messages: Arc>>, should_fail: Arc>, /// Every `since_state` argument this mock's `sync_messages` was called /// with, in call order, so tests can prove a caller threads the returned @@ -25,6 +26,10 @@ pub struct MockMailBackend { /// so a daemon E2E can prove the outbox executor genuinely invoked the /// backend op (not a fabricated completion). applied_mutations: Arc>>, + /// Occupancies this mock reports as explicitly gone, the way QRESYNC + /// `VANISHED` does. Handed back on the pass for the folder each key names, + /// so a test can prove a caller acts on a server-confirmed removal. + removals: Arc>>, } impl MockMailBackend { @@ -56,6 +61,15 @@ impl MockMailBackend { .unwrap_or_default() } + /// Stage occupancies this mock reports as explicitly gone, as a QRESYNC + /// `VANISHED` would. Each key is handed back on the pass for the folder it + /// names, so staging a removal for `Archive` does not leak into `INBOX`. + pub fn set_removals(&self, removals: Vec) { + if let Ok(mut guard) = self.removals.lock() { + *guard = removals; + } + } + /// Configure the mock to simulate network failure errors. pub fn set_should_fail(&self, fail: bool) { if let Ok(mut flag) = self.should_fail.lock() { @@ -70,13 +84,81 @@ impl MockMailBackend { } } - /// Add a mock email message to the storage. - pub fn add_message(&self, email: Email) { + /// Add a mock message to the storage, as the occupancy a backend would + /// have found it in. Taking a [`PlacedMessage`] rather than a bare `Email` + /// lets a test stage the same message key in two folders and prove the + /// caller treats it as one message in two places. + pub fn add_message(&self, message: PlacedMessage) { if let Ok(mut guard) = self.messages.lock() { - guard.push(email); + guard.push(message); } } + /// Account every message staged by [`Self::shared_message_in`] and + /// [`Self::with_same_message_in`] belongs to. + pub const SHARED_ACCOUNT_ID: &'static str = "acct-1"; + + /// The one derived message key every occupancy staged by + /// [`Self::shared_message_in`] carries, whatever folder it sits in. + pub const SHARED_MESSAGE_KEY: &'static str = "msg-shared-identity"; + + /// One occupancy of the single shared message, in `folder_id` under + /// `remote_id`. + /// + /// Identity is byte-identical across every folder -- that is the whole + /// point: a caller cannot tell a first arrival in a second mailbox from a + /// message it already stores unless it compares occupancies rather than + /// message keys. + pub fn shared_message_in(folder_id: &str, remote_id: &str) -> PlacedMessage { + PlacedMessage { + email: Email { + id: Self::SHARED_MESSAGE_KEY.to_string(), + account_id: Self::SHARED_ACCOUNT_ID.to_string(), + subject: "Shared across folders".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("one message, several mailboxes".to_string()), + body_html: None, + message_id: Some("shared@nuncio.mx".to_string()), + content_hash: None, + attachments: Vec::new(), + }, + // EmailId, not Surrogate: a folder-independent key is exactly what + // a server-assigned identity buys, and it is the tier under which + // one message legitimately reports from several mailboxes. + source: IdentitySource::EmailId, + placement: Placement { + account_id: Self::SHARED_ACCOUNT_ID.to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: remote_id.to_string(), + read: false, + }, + } + } + + /// A backend serving `folders`, each holding the same message under its own + /// `remote_id` -- one identity, one occupancy per folder. + /// + /// Stages the folders as well as the messages, so a caller that enumerates + /// folders and then syncs each one sees every occupancy. + pub fn with_same_message_in(folders: &[&str]) -> Self { + let backend = Self::new(); + for (index, folder_id) in folders.iter().enumerate() { + backend.add_folder(Folder { + id: (*folder_id).to_string(), + name: (*folder_id).to_string(), + total_messages: 1, + unread_messages: 1, + }); + // A distinct UID per folder: UIDs are folder-scoped, so the same + // mail in two mailboxes is addressed by two different ones. + backend.add_message(Self::shared_message_in(folder_id, &(index + 1).to_string())); + } + backend + } + /// Every [`RemoteMutationSpec`] this mock's `apply_mutation` was called /// with, in call order. A failing configuration (`set_should_fail(true)`) /// records nothing, so a test can prove a failed op is never counted as @@ -112,7 +194,7 @@ impl MailBackend for MockMailBackend { &self, folder_id: &str, since_state: Option<&str>, - ) -> Result { + ) -> Result { // Record the checkpoint arg before any early return, so even a failing // call is visible to a test asserting on how the caller resumes. if let Ok(mut calls) = self.since_state_calls.lock() { @@ -134,11 +216,25 @@ impl MailBackend for MockMailBackend { .map_err(|e| MailError::ParseFailed(e.to_string()))? .clone(); + // Staged removals are scoped to the folder each key names, and are + // reported on both the full and the incremental pass -- a `VANISHED` + // report is exactly the kind of thing an incremental resync carries. + let removals: Vec = self + .removals + .lock() + .map_err(|e| MailError::ParseFailed(e.to_string()))? + .iter() + .filter(|key| key.folder_id == folder_id) + .cloned() + .collect(); + // Simulate a since_state-aware backend: with a prior checkpoint there // is nothing new to hand back, so the incremental fetch is genuinely // narrower (empty) rather than a full re-report of every message. + // `present` stays `None`: an incremental pass cannot speak to absence. if since_state.is_some() { - return Ok(crate::backend::FolderChanges { + return Ok(FolderChanges { + removals, next_state: returned_state, ..Default::default() }); @@ -150,15 +246,16 @@ impl MailBackend for MockMailBackend { .map_err(|e| MailError::ParseFailed(e.to_string()))?; let matches: Vec<_> = messages .iter() - .filter(|m| m.folder_id == folder_id) + .filter(|m| m.placement.folder_id == folder_id) .cloned() .collect(); // A full pass over the mock's store did see everything in the folder, - // so it can honestly report what is present. - let present = matches.iter().map(|m| m.id.clone()).collect(); - Ok(crate::backend::FolderChanges { + // so it can honestly report what is present -- as occupancies, since + // that is what a folder can speak to. + let present = matches.iter().map(|m| m.placement.key()).collect(); + Ok(FolderChanges { upserts: matches, - removals: Vec::new(), + removals, present: Some(present), next_state: returned_state, }) @@ -256,22 +353,30 @@ mod tests { unread_messages: 1, }); - let email = Email { - id: "msg-mock-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "Mock Test".to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1700000000, - read: false, - body_plain: Some("Mock body".to_string()), - body_html: None, - attachments: Vec::new(), + let placed = PlacedMessage { + email: Email { + id: "msg-mock-1".to_string(), + account_id: "acct-1".to_string(), + subject: "Mock Test".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1700000000, + body_plain: Some("Mock body".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + source: IdentitySource::Surrogate, + placement: Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: "1".to_string(), + read: false, + }, }; - mock.add_message(email.clone()); + mock.add_message(placed.clone()); let folders = mock.sync_folders().await.expect("sync folders succeeds"); assert_eq!(folders.len(), 1); @@ -281,6 +386,10 @@ mod tests { .await .expect("sync messages succeeds"); assert_eq!(msgs.len(), 1); + assert_eq!( + msgs[0], placed, + "the mock must hand back exactly the placement it was staged with" + ); assert_eq!(state, "mock-state-token-100"); // Test error simulation @@ -289,6 +398,39 @@ mod tests { assert!(mock.sync_messages("inbox", None).await.is_err()); } + /// The staging helper is only useful if it really produces one identity in + /// two places: same message key, different occupancies. If it ever drifted + /// into two keys, every test built on it would pass for the wrong reason. + #[tokio::test] + async fn with_same_message_in_stages_one_identity_across_every_folder() { + let mock = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]); + + let folders = mock.sync_folders().await.expect("sync folders succeeds"); + let folder_ids: Vec<&str> = folders.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(folder_ids, vec!["INBOX", "Archive"]); + + let inbox = mock + .sync_changes("INBOX", None) + .await + .expect("inbox pass succeeds"); + let archive = mock + .sync_changes("Archive", None) + .await + .expect("archive pass succeeds"); + assert_eq!(inbox.upserts.len(), 1); + assert_eq!(archive.upserts.len(), 1); + + assert_eq!( + inbox.upserts[0].email.id, archive.upserts[0].email.id, + "both occupancies must carry the same derived message key" + ); + assert_ne!( + inbox.upserts[0].placement.key(), + archive.upserts[0].placement.key(), + "the occupancies themselves must differ, or there is nothing to classify" + ); + } + fn sample_outbound_message() -> OutboundMessage { OutboundMessage { from: "alice@nuncio.mx".to_string(), diff --git a/crates/nuncio-mail/src/parser.rs b/crates/nuncio-mail/src/parser.rs index 95e19f7..25e7ef3 100644 --- a/crates/nuncio-mail/src/parser.rs +++ b/crates/nuncio-mail/src/parser.rs @@ -61,12 +61,15 @@ impl MimeParserAdapter { pub const MAX_PAYLOAD_BYTES: usize = 25 * 1024 * 1024; /// Parse a raw RFC 5322 byte slice into an [`Email`] domain entity. - pub fn parse_mime( - id: &str, - account_id: &str, - folder_id: &str, - raw_bytes: &[u8], - ) -> Result { + /// + /// The returned `Email` carries an **empty** `id`, and it is the caller's + /// job to fill it from [`Email::derive_message_key`]. The derivation needs + /// inputs this parser cannot see -- the transport's addressing coordinates + /// and any server-assigned identity (RFC 8474 `EMAILID`, `X-GM-MSGID`) -- + /// alongside the `Message-ID` and content hash recovered here, so the id + /// can only be settled once both halves are in hand. Minting a provisional + /// value here would risk one reaching the store as though it were derived. + pub fn parse_mime(account_id: &str, raw_bytes: &[u8]) -> Result { if raw_bytes.len() > Self::MAX_PAYLOAD_BYTES { return Err(MailError::ParseFailed( "MIME payload exceeds maximum allowed limit of 25MB".to_string(), @@ -95,6 +98,9 @@ impl MimeParserAdapter { let received_at = msg.date().map_or(0, |d| d.to_timestamp()); + let message_id = msg.message_id().and_then(Email::normalize_message_id); + let content_hash = Some(Email::content_hash_of(raw_bytes)); + let body_plain = msg.body_text(0).map(|b| b.to_string()); let body_html = msg.body_html(0).map(|b| b.to_string()); @@ -126,21 +132,19 @@ impl MimeParserAdapter { } Ok(Email { - id: id.to_string(), + // Assigned by the caller from the derived message key; see the + // method doc for why it cannot be settled here. + id: String::new(), account_id: account_id.to_string(), - folder_id: folder_id.to_string(), - // The MIME body carries no protocol addressing; the caller (the - // IMAP sync) fills these from the transport before persisting. - remote_id: String::new(), - uid_validity: String::new(), subject, sender, recipient, received_at, - read: false, body_plain, body_html, attachments, + message_id, + content_hash, }) } } @@ -158,9 +162,12 @@ mod tests { \r\n\ Hello Bob, this is a plain text email."; - let email = - MimeParserAdapter::parse_mime("msg-1", "acct-1", "inbox", raw).expect("parse succeeds"); - assert_eq!(email.id, "msg-1"); + let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds"); + assert_eq!( + email.id, "", + "the parser must not mint an id; the caller derives it" + ); + assert_eq!(email.account_id, "acct-1"); assert_eq!(email.subject, "Test Subject"); assert_eq!(email.sender, "alice@nuncio.mx"); assert_eq!(email.recipient, "bob@nuncio.mx"); @@ -170,11 +177,41 @@ mod tests { ); } + #[test] + fn parse_captures_normalized_message_id_and_content_hash() { + let raw = b"From: Alice \r\n\ + To: Bob \r\n\ + Message-ID: \r\n\ + Subject: Test\r\n\ + \r\n\ + body"; + + let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds"); + assert_eq!(email.message_id.as_deref(), Some("Abc.123@mail.nuncio.mx")); + assert_eq!( + email.content_hash, + Some(nuncio_core::model::Email::content_hash_of(raw)), + "the hash must cover the octets actually parsed" + ); + } + + #[test] + fn parse_reports_no_message_id_when_the_header_is_absent() { + // RFC 5322 3.6.4 makes Message-ID a SHOULD. An absent header must read + // as "none present", never as an invented or empty value. + let raw = b"From: Alice \r\n\ + Subject: No id\r\n\ + \r\n\ + body"; + let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds"); + assert_eq!(email.message_id, None); + assert!(email.content_hash.is_some()); + } + #[test] fn parse_invalid_mime_returns_error() { let raw = b""; - let err = MimeParserAdapter::parse_mime("msg-2", "acct-1", "inbox", raw) - .expect_err("should fail"); + let err = MimeParserAdapter::parse_mime("acct-1", raw).expect_err("should fail"); assert_eq!( err, MailError::ParseFailed("invalid RFC 5322 MIME structure".to_string()) @@ -188,7 +225,7 @@ mod tests { #[test] fn parse_oversized_mime_returns_error() { let oversized = vec![0u8; MimeParserAdapter::MAX_PAYLOAD_BYTES + 1]; - let err = MimeParserAdapter::parse_mime("msg-huge", "acct-1", "inbox", &oversized) + let err = MimeParserAdapter::parse_mime("acct-1", &oversized) .expect_err("should fail for oversized payload"); assert!(err .to_string() diff --git a/crates/nuncio-mail/src/smtp.rs b/crates/nuncio-mail/src/smtp.rs index 894ae6c..673d963 100644 --- a/crates/nuncio-mail/src/smtp.rs +++ b/crates/nuncio-mail/src/smtp.rs @@ -357,17 +357,15 @@ mod tests { Email { id: "msg-1".to_string(), account_id: "acct-1".to_string(), - folder_id: "outbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), subject: "Status Update".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: true, body_plain: Some("Plaintext status update".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } diff --git a/crates/nuncio-mail/tests/imap_raw_wire_test.rs b/crates/nuncio-mail/tests/imap_raw_wire_test.rs index 42a67a7..97ed5f2 100644 --- a/crates/nuncio-mail/tests/imap_raw_wire_test.rs +++ b/crates/nuncio-mail/tests/imap_raw_wire_test.rs @@ -11,6 +11,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use nuncio_core::model::PlacementKey; use nuncio_mail::imap_raw; use nuncio_mail::test_server::{MockImapServer, ServerProfile}; @@ -345,14 +346,17 @@ async fn every_rung_reports_a_message_that_disappeared() { let second = sync_folder(&server, Some(&first.next_state)).await; // However the rung learned it, the caller must end up able to identify - // the removed message and only that one. - let mut reported: Vec = second.removals.clone(); + // the occupancy that went and only that one. Absence is reported per + // mailbox, so what is compared is the placement each upsert was found + // under, not the message key -- the same message may still sit + // elsewhere and must not be swept up by this folder's silence. + let mut reported: Vec = second.removals.clone(); if let Some(present) = &second.present { let stored_gone = first .upserts .iter() - .map(|e| e.id.clone()) - .filter(|id| !present.contains(id)); + .map(|e| e.placement.key()) + .filter(|key| !present.contains(key)); reported.extend(stored_gone); } reported.sort(); @@ -366,12 +370,17 @@ async fn every_rung_reports_a_message_that_disappeared() { let surviving = first .upserts .iter() - .find(|e| e.remote_id == keep.to_string()) + .find(|e| e.placement.remote_id == keep.to_string()) .expect("the kept message was in the first pass"); assert!( - !reported.contains(&surviving.id), + !reported.contains(&surviving.placement.key()), "{profile:?}: the surviving message must not be reported gone" ); + assert_eq!( + reported[0].remote_id, + gone.to_string(), + "{profile:?}: the reported occupancy must be the expunged UID" + ); } } @@ -474,9 +483,17 @@ async fn a_uidvalidity_change_refetches_rather_than_deleting_the_folder() { second.removals.is_empty(), "a renumbering is not a removal report" ); - // The re-fetched message has a different surrogate, because UIDVALIDITY is - // part of identity -- so the caller replaces rather than merges. - assert_ne!(second.upserts[0].id, first.upserts[0].id); + // The re-fetched message lands under a new occupancy, because UIDVALIDITY + // is part of a placement key -- so the caller writes a new placement rather + // than updating the old one, and the stale UID never addresses live mail. + assert_ne!( + second.upserts[0].placement.uid_validity, + first.upserts[0].placement.uid_validity + ); + assert_ne!(second.upserts[0].placement, first.upserts[0].placement); + // These two are also genuinely different messages, so their derived keys + // differ as well. + assert_ne!(second.upserts[0].email.id, first.upserts[0].email.id); } /// Build an engine pointed at the mock and apply one mutation through the diff --git a/crates/nuncio-mail/tests/wiremock_jmap_test.rs b/crates/nuncio-mail/tests/wiremock_jmap_test.rs index f809590..46b9415 100644 --- a/crates/nuncio-mail/tests/wiremock_jmap_test.rs +++ b/crates/nuncio-mail/tests/wiremock_jmap_test.rs @@ -2,6 +2,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use nuncio_core::model::IdentitySource; use nuncio_mail::{JmapEngine, MailBackend, RemoteMutationKind, RemoteMutationSpec}; use serde_json::json; use wiremock::matchers::{body_partial_json, method, path}; @@ -84,13 +85,16 @@ async fn wiremock_jmap_session_discovery_and_email_get_sync() { .expect("parse email get"); assert_eq!(emails.len(), 1); - // The JMAP object id is preserved as the protocol-native `remote_id`, while - // the persisted id is an opaque 64-char surrogate hash (never the raw id). - assert_eq!(emails[0].remote_id, "msg-wm-100"); - assert_ne!(emails[0].id, "msg-wm-100"); - assert_eq!(emails[0].id.len(), 64); - assert_eq!(emails[0].subject, "WireMock JMAP E2E Test"); - assert_eq!(emails[0].sender, "sender@nuncio.mx"); + // The JMAP object id is preserved as the placement's protocol-native + // `remote_id`, while the message's own id is an opaque 64-char derived key + // (never the raw id). The object id is account-stable, so it enters the + // precedence at the EMAILID tier rather than as a folder-scoped surrogate. + assert_eq!(emails[0].placement.remote_id, "msg-wm-100"); + assert_eq!(emails[0].source, IdentitySource::EmailId); + assert_ne!(emails[0].email.id, "msg-wm-100"); + assert_eq!(emails[0].email.id.len(), 64); + assert_eq!(emails[0].email.subject, "WireMock JMAP E2E Test"); + assert_eq!(emails[0].email.sender, "sender@nuncio.mx"); assert_eq!(new_state, "sync-state-500"); } @@ -204,15 +208,18 @@ async fn jmap_engine_sync_folders_and_messages_through_mail_backend_trait() { .await .expect("sync_messages succeeds"); assert_eq!(emails.len(), 1); - assert_eq!(emails[0].remote_id, "msg-wm-200"); - assert_ne!(emails[0].id, "msg-wm-200"); - assert_eq!(emails[0].id.len(), 64); - assert_eq!(emails[0].subject, "Trait-Level JMAP Sync"); - assert_eq!(emails[0].sender, "sender@nuncio.mx"); - assert_eq!(emails[0].received_at, 1700002000); - assert!(!emails[0].read); + assert_eq!(emails[0].placement.remote_id, "msg-wm-200"); + assert_eq!(emails[0].placement.folder_id, "inbox"); + assert_eq!(emails[0].source, IdentitySource::EmailId); + assert_ne!(emails[0].email.id, "msg-wm-200"); + assert_eq!(emails[0].email.id.len(), 64); + assert_eq!(emails[0].email.subject, "Trait-Level JMAP Sync"); + assert_eq!(emails[0].email.sender, "sender@nuncio.mx"); + assert_eq!(emails[0].email.received_at, 1700002000); + // Read state is a property of the occupancy, not of the message. + assert!(!emails[0].placement.read); assert_eq!( - emails[0].body_plain, + emails[0].email.body_plain, Some("Fetched through the MailBackend trait.".to_string()) ); assert_eq!(new_state, "sync-state-900"); diff --git a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto index 41302bf..2abaec2 100644 --- a/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto +++ b/crates/nuncio-proto/proto/nuncio/v1/nuncio.proto @@ -519,10 +519,19 @@ message Attachment { // (rather than `Email`) to avoid colliding with common generated-code // identifiers in hand-written Rust using this module. message Message { - // Unique message identifier. OPAQUE and server-assigned: a deterministic - // surrogate derived from the message's stable protocol identity, never a - // raw upstream protocol id (e.g. an IMAP UID, which is only stable within - // a single IMAP session and is NOT a durable cross-session identifier). + // Unique message identifier. OPAQUE and server-assigned: content-addressed + // from the strongest identity the message's server offered. Never a raw + // upstream protocol id (e.g. an IMAP UID, which is only stable within a + // single IMAP session and is NOT a durable cross-session identifier). + // + // Stability across a move depends on what the server offered. Where a + // server-assigned object id or a usable Message-ID was available, the id is + // folder-independent and survives a move between folders. Where none was -- + // a message carrying no Message-ID, or one fetched headers-only -- the id + // falls back to a derivation that includes the folder, and moving such a + // message yields a different id. Clients MUST NOT assume an id outlives a + // move. + // // Clients MUST treat this as an opaque token -- pass it back verbatim in // later requests, but never parse, construct, or assume any structure in // its value. @@ -530,6 +539,11 @@ message Message { // Account identifier owning the message. string account_id = 2; // Mailbox folder identifier (e.g. "inbox"). + // + // A message can occupy several folders at once. This names the one the + // message was reached through: the requested folder for a listing, and the + // lowest-sorting folder id for a lookup by id. A field carrying the complete + // placement set is not yet part of this contract. string folder_id = 3; // Subject line. string subject = 4; @@ -777,6 +791,13 @@ service Filters { // applied retroactively to mail that was synced before those rules // existed. Streams cumulative progress as it works, ending with a final // `done: true` message once the whole store has been scanned. + // + // Each (rule, message) pair fires at most once, ever -- the same claim + // ledger live sync uses, keyed by rule id and message id. This is not a + // repeatable rescan: running Triage again with the same rules finds + // nothing left to claim and reports `matched_count: 0`. Only messages + // stored since the last run, or rules created since then, produce further + // matches; editing an existing rule's contents does not reset its claims. rpc Triage(TriageRequest) returns (stream TriageProgress); } diff --git a/crates/nuncio-store/src/db.rs b/crates/nuncio-store/src/db.rs index 1c53ea7..10f874d 100644 --- a/crates/nuncio-store/src/db.rs +++ b/crates/nuncio-store/src/db.rs @@ -241,6 +241,107 @@ pub struct WormChainReport { pub first_broken_seq: Option, } +/// What a [`DatabaseEngine::save_email_at`] call actually changed. +/// +/// The two flags answer different questions and a caller usually needs both: +/// `message_is_new` gates work that should happen once per message (indexing, +/// side-effecting filter actions), `placement_is_new` gates work that should +/// happen once per mailbox occupancy (per-folder counters, per-placement +/// evaluation). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SaveOutcome { + /// The message identity had not been stored before this call. + pub message_is_new: bool, + /// This mailbox occupancy had not been stored before this call. + pub placement_is_new: bool, +} + +/// The four coordinates that address one mailbox occupancy. +/// +/// Re-exported from `nuncio-core` rather than defined here: a protocol backend +/// reports these when a folder stops mentioning a message, and this store +/// deletes exactly the rows they name, so both crates must be able to name one +/// type. `nuncio-mail` must not depend on `nuncio-store`, which leaves `core` +/// as the only place both can reach. +pub use nuncio_core::model::PlacementKey; + +/// A message paired with the mailbox occupancy it was read through. +/// +/// The two travel together out of every folder-scoped read: the caller almost +/// always needs the per-mailbox facts (the read flag, the addressing UID) as +/// well as the message, and re-deriving them would cost a query per row. +/// +/// Deliberately not called `PlacedMessage`: `nuncio_mail::PlacedMessage` is a +/// different type carrying a third field (the identity tier a backend derived +/// the key from), and the sync path imports both. One name for two shapes in +/// one call site is a trap worth a longer name to avoid. +pub type MessageWithPlacement = (nuncio_core::model::Email, nuncio_core::model::Placement); + +/// Keyset cursor for [`DatabaseEngine::list_messages_page`]: the +/// `(received_at, message_key, uidvalidity, uid)` of the last row of a page. +/// +/// It addresses a placement, not a message, because a message can occupy one +/// folder more than once -- see that method for why a message-only cursor +/// silently drops rows. +pub type MessagePageCursor = (i64, String, String, String); + +/// Column order of the identity-only `messages` projection shared by every +/// whole-store read path: key, account, subject, sender, recipient, +/// received_at, body_plain, body_html, message_id, content_hash. +type MessageRow = ( + String, + String, + String, + String, + String, + i64, + Option, + Option, + Option, + Option, +); + +/// [`MessageRow`] widened with the joined placement's `uidvalidity`, `uid` and +/// `read_flag`, in that order. +type MessageWithPlacementRow = ( + String, + String, + String, + String, + String, + i64, + Option, + Option, + Option, + Option, + String, + String, + i64, +); + +/// What a [`DatabaseEngine::delete_placements`] call removed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeleteOutcome { + /// Mailbox occupancies removed. + pub placements_removed: u64, + /// Messages that lost their last placement and were removed with it. + pub messages_reaped: u64, +} + +/// How many keys one generated `IN (...)` list may carry. +/// +/// SQLite caps a statement's bind parameters at `SQLITE_MAX_VARIABLE_NUMBER`, +/// 32766 in the bundled build. The widest key built here spends four +/// parameters per entry (account, folder, uidvalidity, uid), so 4000 keys is +/// 16000 parameters -- comfortably under the cap with room for the limit to +/// halve, or for a future key to grow a fifth column, without silently +/// becoming a runtime error again. +/// +/// The cap is worth this margin because overrunning it is not a slow query but +/// a hard failure of the whole statement: the enclosing folder sync aborts, no +/// checkpoint is written, and every retry fails in exactly the same place. +const MAX_KEYS_PER_IN_LIST: usize = 4_000; + /// Serialize a [`nuncio_core::TlsMode`] to its stable on-disk `accounts` /// column form. Kept as a bare snake_case token (matching the column's /// `'implicit_tls'` default) rather than JSON, so a row inserted by the @@ -467,11 +568,16 @@ impl DatabaseEngine { } /// Total unread messages across every folder, read live from the store. + /// + /// Counted over placements, not messages: `\Seen` is per-mailbox, so a + /// message sitting unread in two folders is genuinely two unread items to + /// clear. pub async fn count_unread_messages(&self) -> Result { - let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages WHERE read_flag = 0") - .fetch_one(&self.pool) - .await - .map_err(DatabaseError::Query)?; + let (count,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM placements WHERE read_flag = 0") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; Ok(count.max(0) as u64) } @@ -522,7 +628,7 @@ impl DatabaseEngine { /// /// Bump this only for a change of that kind, and only together with the /// reset it implies (see [`DatabaseEngine::reconcile_schema_version`]). - pub const IDENTITY_SCHEMA_VERSION: i64 = 1; + pub const IDENTITY_SCHEMA_VERSION: i64 = 2; /// Reconcile the on-disk identity-schema generation with this binary's. /// @@ -582,11 +688,12 @@ impl DatabaseEngine { /// Drop everything derived from the server, keeping everything the user /// configured. /// - /// Cleared: messages and their FTS index, per-folder sync checkpoints, and - /// the filter execution ledger. The ledger goes because its hash chain is - /// keyed on message ids that will no longer resolve, and a chain pointing - /// at absent rows has no audit value -- reinitialising it is more honest - /// than carrying a permanently mixed-namespace log. + /// Cleared: messages and their FTS index, mailbox placements, the filter + /// fire-once ledger, per-folder sync checkpoints, and the filter execution + /// ledger. The execution ledger goes because its hash chain is keyed on + /// message ids that will no longer resolve, and a chain pointing at absent + /// rows has no audit value -- reinitialising it is more honest than + /// carrying a permanently mixed-namespace log. /// /// Kept: accounts, credentials (which live in the OS keyring regardless), /// filter rules, and the WORM audit records, whose triggers forbid deletion @@ -596,6 +703,8 @@ impl DatabaseEngine { for statement in [ "DELETE FROM messages", "DELETE FROM messages_fts", + "DELETE FROM placements", + "DELETE FROM filter_fired", "DELETE FROM folder_sync_state", "DELETE FROM filter_execution_logs", ] { @@ -608,23 +717,117 @@ impl DatabaseEngine { Ok(()) } + /// Atomically claim the right to run `rule_id`'s actions against + /// `message_key`, returning `true` only for the claim that won. + /// + /// The claim is the fire-once guard for side-effecting actions. It is keyed + /// on message identity rather than on a placement because the same message + /// legitimately arrives in several folders -- and, under the old + /// folder-scoped identity, each arrival looked like a different message and + /// fired the rule again. `INSERT ... ON CONFLICT DO NOTHING` makes winning + /// the claim a single atomic statement, so two passes racing over the same + /// message cannot both act. + pub async fn claim_filter_fire( + &self, + rule_id: &str, + message_key: &str, + ) -> Result { + let result = sqlx::query( + "INSERT INTO filter_fired (rule_id, message_key, fired_at) VALUES (?, ?, ?) + ON CONFLICT (rule_id, message_key) DO NOTHING", + ) + .bind(rule_id) + .bind(message_key) + .bind(chrono::Utc::now().timestamp()) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(result.rows_affected() == 1) + } + + /// Whether `rule_id` has already acted on `message_key`. + /// + /// Public rather than test-only: the claim ledger answers a question + /// operators and the outbox both have a real reason to ask -- "did this rule + /// already act on this message" -- and callers in other crates cannot reach + /// the pool directly. + pub async fn has_filter_fired( + &self, + rule_id: &str, + message_key: &str, + ) -> Result { + let found: Option<(i64,)> = + sqlx::query_as("SELECT 1 FROM filter_fired WHERE rule_id = ? AND message_key = ?") + .bind(rule_id) + .bind(message_key) + .fetch_optional(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(found.is_some()) + } + + /// How many distinct (rule, message) fires the ledger records. + /// + /// Counterpart to [`Self::has_filter_fired`] for callers that need to assert + /// on the ledger as a whole rather than one entry. + pub async fn filter_fire_count(&self) -> Result { + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(count) + } + /// Execute initial database migrations creating core envelope tables. pub async fn migrate(&self) -> Result<(), DatabaseError> { sqlx::query( r#" + -- Identity and immutable content, one row per message. Where the + -- message sits lives in `placements`; nothing here is folder-scoped. CREATE TABLE IF NOT EXISTS messages ( - id TEXT PRIMARY KEY NOT NULL, + message_key TEXT PRIMARY KEY NOT NULL, account_id TEXT NOT NULL, - folder_id TEXT NOT NULL, - remote_id TEXT NOT NULL DEFAULT '', - uid_validity TEXT NOT NULL DEFAULT '', subject TEXT NOT NULL, sender TEXT NOT NULL, recipient TEXT NOT NULL, received_at INTEGER NOT NULL, - read_flag INTEGER NOT NULL DEFAULT 0, body_plain TEXT, - body_html TEXT + body_html TEXT, + -- Captured at ingest, not yet read. Both are nullable because + -- neither is always available: `Message-ID` is SHOULD, not + -- MUST (RFC 5322 3.6.4), and a headers-only or envelope-only + -- fetch has no full octets to hash. + message_id TEXT, + content_hash TEXT, + identity_source TEXT NOT NULL DEFAULT 'surrogate' + ); + + -- One row per (mailbox, message) occupancy. `uidvalidity` is in the + -- primary key because UIDs restart at 1 after a bump: without it the + -- first message of the new generation would overwrite the row of + -- whichever old message shared its UID. + CREATE TABLE IF NOT EXISTS placements ( + account_id TEXT NOT NULL, + folder_id TEXT NOT NULL, + uidvalidity TEXT NOT NULL, + uid TEXT NOT NULL, + message_key TEXT NOT NULL, + read_flag INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (account_id, folder_id, uidvalidity, uid) + ); + + -- Fire-once ledger for filter actions. Keyed on the message + -- *identity*, not a placement, so a message arriving in a second + -- folder does not re-fire a rule that already acted on it -- which + -- matters most for FORWARD and CALL WEBHOOK, whose effects land on + -- third parties and cannot be undone by convergence. + CREATE TABLE IF NOT EXISTS filter_fired ( + rule_id TEXT NOT NULL, + message_key TEXT NOT NULL, + fired_at INTEGER NOT NULL, + PRIMARY KEY (rule_id, message_key) ); CREATE TABLE IF NOT EXISTS calendar_events ( @@ -731,6 +934,8 @@ impl DatabaseEngine { CREATE INDEX IF NOT EXISTS idx_filter_logs_rule ON filter_execution_logs(rule_id, matched_at DESC); CREATE INDEX IF NOT EXISTS idx_pending_mutations_status ON pending_remote_mutations(status, created_at ASC); CREATE INDEX IF NOT EXISTS idx_worm_audit_seq ON worm_audit_records(sequence ASC); + CREATE INDEX IF NOT EXISTS idx_placements_message ON placements(message_key); + CREATE INDEX IF NOT EXISTS idx_placements_folder ON placements(account_id, folder_id); -- Full-text search indexes are created eagerly at migration time (not lazily on -- first search) so no message or event saved before the first search call is ever @@ -743,7 +948,7 @@ impl DatabaseEngine { -- ciphertext and a trigger-based mirror would index that ciphertext verbatim -- (defeating search entirely). Instead `messages_fts` is populated explicitly from -- the plaintext body in application code, at the moment of encryption in - -- `DatabaseEngine::save_email` (see there) and via `backfill_message_fts` below for + -- `DatabaseEngine::save_email_at` (see there) and via `backfill_message_fts` below for -- any pre-existing rows. This means the trigram index now contains -- plaintext-derived body text: the FTS index itself is NOT encrypted, so message -- body content is recoverable from `messages_fts` by anyone with filesystem access @@ -753,7 +958,7 @@ impl DatabaseEngine { -- encrypted search index, or whole-database encryption) is future work and is NOT -- provided today. CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( - id UNINDEXED, + message_key UNINDEXED, subject, sender, body_plain, @@ -761,12 +966,19 @@ impl DatabaseEngine { ); -- Subject/sender are never encrypted in `messages`, so a delete-only trigger is - -- sufficient here: it just keeps the FTS index free of orphaned rows. Insertion and - -- update of `messages_fts` content happens explicitly in `save_email`, never via an + -- sufficient here: it just keeps the FTS index free of orphaned rows. Insertion of + -- `messages_fts` content happens explicitly in `save_email_at`, never via an -- AFTER INSERT/UPDATE trigger, because such a trigger would only ever see the -- ciphertext body column. + -- + -- This trigger MUST stay on `messages` and never move to `placements`. Dropping one + -- placement of a message that still sits in another folder must leave the body + -- searchable; only losing the last placement removes the `messages` row, and that is + -- what reaps the FTS row here. Firing on `placements` instead would delete the index + -- entry while the body remained -- and, inverted, leaving it off `messages` entirely + -- would strand plaintext bodies in `messages_fts` after the message itself was gone. CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN - DELETE FROM messages_fts WHERE id = old.id; + DELETE FROM messages_fts WHERE message_key = old.message_key; END; -- Calendar event summary/location are never encrypted at rest, so trigger-based @@ -887,16 +1099,16 @@ impl DatabaseEngine { async fn backfill_message_fts(&self) -> Result<(), DatabaseError> { let rows: Vec<(String, String, String, Option)> = sqlx::query_as( r#" - SELECT m.id, m.subject, m.sender, m.body_plain + SELECT m.message_key, m.subject, m.sender, m.body_plain FROM messages m - WHERE NOT EXISTS (SELECT 1 FROM messages_fts f WHERE f.id = m.id) + WHERE NOT EXISTS (SELECT 1 FROM messages_fts f WHERE f.message_key = m.message_key) "#, ) .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - for (id, subject, sender, body_plain) in rows { + for (message_key, subject, sender, body_plain) in rows { let dec_plain = body_plain .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) .transpose() @@ -904,9 +1116,9 @@ impl DatabaseEngine { .unwrap_or_default(); sqlx::query( - "INSERT INTO messages_fts (id, subject, sender, body_plain) VALUES (?, ?, ?, ?)", + "INSERT INTO messages_fts (message_key, subject, sender, body_plain) VALUES (?, ?, ?, ?)", ) - .bind(&id) + .bind(&message_key) .bind(&subject) .bind(&sender) .bind(&dec_plain) @@ -1166,7 +1378,19 @@ impl DatabaseEngine { Ok(()) } - /// Save an [`nuncio_core::model::Email`] to SQLite (INSERT OR REPLACE). + /// Persist a message's identity and content, and record that it occupies + /// `placement`, in one transaction. + /// + /// The message row is **first-write-wins**: a second sighting of the same + /// key -- which is the normal case for a message that also exists in another + /// folder -- leaves the stored subject and body untouched. That is what + /// keeps a message whose identity rests on `Message-ID` + content hash from + /// being rewritten by a later fetch claiming the same key, and it is why the + /// FTS row is written only on the insert that actually created the message. + /// + /// The placement row is upserted rather than ignored, because its read flag + /// is genuinely mutable: `\Seen` changes in the mailbox and the store must + /// follow it. /// /// The message body is encrypted (AES-256-GCM) before being written to the `messages` /// table, but the *plaintext* body is also indexed into the standalone `messages_fts` @@ -1176,7 +1400,12 @@ impl DatabaseEngine { /// `CREATE VIRTUAL TABLE` statement in [`DatabaseEngine::migrate`] -- the FTS index itself /// is not encrypted, so this intentionally trades some body confidentiality for working /// search. - pub async fn save_email(&self, email: &nuncio_core::model::Email) -> Result<(), DatabaseError> { + pub async fn save_email_at( + &self, + email: &nuncio_core::model::Email, + source: nuncio_core::model::IdentitySource, + placement: &nuncio_core::model::Placement, + ) -> Result { // Encryption failures MUST surface before any SQL runs: a swallowed error here would // otherwise leave a row with an empty/placeholder body column, indistinguishable from // a genuinely empty body on read-back. @@ -1191,280 +1420,456 @@ impl DatabaseEngine { .map(|h| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, h)) .transpose()?; - let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + // `BEGIN IMMEDIATE` takes the write lock before the first statement runs. This + // transaction reads (the placement existence check) and then writes what it read, and in + // WAL mode a deferred transaction that upgrades to a writer after another connection has + // committed aborts with SQLITE_BUSY_SNAPSHOT -- which `busy_timeout` does not retry, + // because there is no lock to wait for, only a stale snapshot. Acquiring the lock up + // front makes a concurrent save block instead of failing. + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(DatabaseError::Query)?; - sqlx::query( + // Ask before writing. SQLite reports one row affected for both arms of + // an upsert, so after the fact there is no way to tell an inserted + // placement from an updated one -- and the caller needs that distinction + // to decide whether this is a first arrival worth filtering. + let (already_placed,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&placement.account_id) + .bind(&placement.folder_id) + .bind(&placement.uid_validity) + .bind(&placement.remote_id) + .fetch_one(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + let placement_is_new = already_placed == 0; + + let inserted = sqlx::query( r#" - INSERT OR REPLACE INTO messages - (id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO messages + (message_key, account_id, subject, sender, recipient, received_at, + body_plain, body_html, message_id, content_hash, identity_source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (message_key) DO NOTHING "#, ) .bind(&email.id) .bind(&email.account_id) - .bind(&email.folder_id) - .bind(&email.remote_id) - .bind(&email.uid_validity) .bind(&email.subject) .bind(&email.sender) .bind(&email.recipient) .bind(email.received_at) - .bind(if email.read { 1i64 } else { 0i64 }) .bind(&enc_plain) .bind(&enc_html) + .bind(&email.message_id) + .bind(&email.content_hash) + .bind(source.as_str()) .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; - // Re-indexing: delete any prior FTS row for this message id, then insert the fresh - // plaintext-derived row. FTS5 has no natural "INSERT OR REPLACE" semantics for a - // standalone (non-external-content) table, so this is done explicitly rather than via - // trigger. - sqlx::query("DELETE FROM messages_fts WHERE id = ?") + let message_is_new = inserted.rows_affected() == 1; + + // Index only on the insert that created the message. Re-indexing on + // every placement would either duplicate the row (FTS5 standalone tables + // have no upsert) or rewrite it with a body the first-write-wins rule + // just rejected. + if message_is_new { + sqlx::query( + "INSERT INTO messages_fts (message_key, subject, sender, body_plain) VALUES (?, ?, ?, ?)", + ) .bind(&email.id) + .bind(&email.subject) + .bind(&email.sender) + .bind(email.body_plain.as_deref().unwrap_or("")) .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; + } sqlx::query( - "INSERT INTO messages_fts (id, subject, sender, body_plain) VALUES (?, ?, ?, ?)", + r#" + INSERT INTO placements + (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (account_id, folder_id, uidvalidity, uid) + DO UPDATE SET message_key = excluded.message_key, read_flag = excluded.read_flag + "#, ) + .bind(&placement.account_id) + .bind(&placement.folder_id) + .bind(&placement.uid_validity) + .bind(&placement.remote_id) .bind(&email.id) - .bind(&email.subject) - .bind(&email.sender) - .bind(email.body_plain.as_deref().unwrap_or("")) + .bind(if placement.read { 1i64 } else { 0i64 }) .execute(&mut *tx) .await .map_err(DatabaseError::Query)?; tx.commit().await.map_err(DatabaseError::Query)?; - Ok(()) + Ok(SaveOutcome { + message_is_new, + placement_is_new, + }) } - /// Query synced email messages for a specific folder. - #[allow(clippy::type_complexity)] + /// Remove mailbox occupancies, reaping any message left with none. + /// + /// A server that stops reporting a UID has told us the message left *that + /// mailbox*, not that it ceased to exist -- a move reports exactly this in + /// the source folder while the message continues in the destination. So the + /// placement goes unconditionally and the message goes only when its last + /// placement does. + /// + /// The reap is what keeps the plaintext FTS index honest. `messages_fts` + /// holds decrypted bodies keyed on `message_key`, reaped by the + /// `messages_ad` trigger on `messages`. Deleting the message row is + /// therefore the only thing that clears the index: skip the reap and every + /// fully-deleted message leaves its body readable to anyone with filesystem + /// access to the database. + pub async fn delete_placements( + &self, + keys: &[PlacementKey], + ) -> Result { + if keys.is_empty() { + return Ok(DeleteOutcome { + placements_removed: 0, + messages_reaped: 0, + }); + } + + // `BEGIN IMMEDIATE` for the same reason as in [`Self::save_email_at`]: this reads each + // placement's owning message key and then deletes based on what it read, and a deferred + // WAL transaction that upgrades to a writer mid-flight aborts with SQLITE_BUSY_SNAPSHOT + // rather than waiting out `busy_timeout`. + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(DatabaseError::Query)?; + let mut placements_removed = 0u64; + let mut touched: Vec = Vec::with_capacity(keys.len()); + + for key in keys { + // Remember which message each placement pointed at before removing + // it -- afterwards there is nothing left to join through. + let owner: Option<(String,)> = sqlx::query_as( + "SELECT message_key FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .fetch_optional(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + let Some((message_key,)) = owner else { + continue; + }; + + let result = sqlx::query( + "DELETE FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + placements_removed += result.rows_affected(); + touched.push(message_key); + } + + touched.sort_unstable(); + touched.dedup(); + + let mut messages_reaped = 0u64; + for message_key in touched { + // Reap only where nothing is left pointing at the message. The + // `messages_ad` trigger clears the FTS row as a consequence. + let result = sqlx::query( + "DELETE FROM messages WHERE message_key = ? + AND NOT EXISTS (SELECT 1 FROM placements WHERE message_key = ?)", + ) + .bind(&message_key) + .bind(&message_key) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + messages_reaped += result.rows_affected(); + } + + tx.commit().await.map_err(DatabaseError::Query)?; + Ok(DeleteOutcome { + placements_removed, + messages_reaped, + }) + } + + /// Every mailbox this message currently occupies. + pub async fn placements_of( + &self, + message_key: &str, + ) -> Result, DatabaseError> { + let rows: Vec<(String, String, String, String, i64)> = sqlx::query_as( + "SELECT account_id, folder_id, uidvalidity, uid, read_flag + FROM placements WHERE message_key = ? ORDER BY folder_id ASC", + ) + .bind(message_key) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map( + |(account_id, folder_id, uid_validity, remote_id, read_flag)| { + nuncio_core::model::Placement { + account_id, + folder_id, + uid_validity, + remote_id, + read: read_flag != 0, + } + }, + ) + .collect()) + } + + /// Messages occupying one folder of one account, newest first, paired with + /// the placement they were found through. + /// + /// The placement travels with the message because the caller almost always + /// needs the per-mailbox facts -- the read flag and the addressing UID -- + /// and re-deriving them would mean a second query per row. pub async fn list_messages( &self, + account_id: &str, folder_id: &str, limit: usize, - ) -> Result, DatabaseError> { - let rows: Vec<( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option, - Option, - String, - String, - )> = sqlx::query_as( + ) -> Result, DatabaseError> { + let rows: Vec = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity - FROM messages - WHERE folder_id = ? - ORDER BY received_at DESC + SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, + m.received_at, m.body_plain, m.body_html, m.message_id, m.content_hash, + p.uidvalidity, p.uid, p.read_flag + FROM messages m + JOIN placements p ON p.message_key = m.message_key + WHERE p.account_id = ? AND p.folder_id = ? + ORDER BY m.received_at DESC LIMIT ? "#, ) + .bind(account_id) .bind(folder_id) .bind(limit as i64) .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - rows.into_iter() - .map( - |( - id, - account_id, - folder_id, - subject, - sender, - recipient, - received_at, - read_flag, - body_plain, - body_html, - remote_id, - uid_validity, - )| { - let dec_plain = body_plain - .map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest( - &self.storage_key, - &p, - ) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - let dec_html = body_html - .map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest( - &self.storage_key, - &h, - ) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - Ok(nuncio_core::model::Email { - id, - account_id, - folder_id, - remote_id, - uid_validity, - subject, - sender, - recipient, - received_at, - read: read_flag != 0, - body_plain: dec_plain, - body_html: dec_html, - attachments: Vec::new(), - }) + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + + out.push(( + nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, }, - ) - .collect::, DatabaseError>>() + nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: row.10, + remote_id: row.11, + read: row.12 != 0, + }, + )); + } + Ok(out) } - /// Keyset-paginated listing of a folder's messages, newest first - /// (`received_at DESC, id DESC`). `after` is the `(received_at, id)` of - /// the last message of the previous page; `None` starts from the newest. - /// Fetches `page_size + 1` rows to detect a following page: returns at - /// most `page_size` messages plus the `(received_at, id)` cursor of the - /// last returned message when more remain (else `None`). - #[allow(clippy::type_complexity)] + /// Keyset-paginated listing of one folder's placements, newest first + /// (`received_at DESC, message_key DESC, uidvalidity DESC, uid DESC`), each + /// paired with the message it holds. `after` is the cursor of the last row + /// of the previous page; `None` starts from the newest. Fetches + /// `page_size + 1` rows to detect a following page: returns at most + /// `page_size` pairs plus the cursor of the last returned row when more + /// remain (else `None`). + /// + /// The cursor carries the full placement address, not just the message + /// coordinates, because the join is **not** one-to-one within a folder: the + /// placements primary key is `(account, folder, uidvalidity, uid)`, so one + /// message legitimately occupies a single mailbox twice -- two `COPY`s of + /// the same mail into one folder land under different UIDs. With only + /// `(received_at, message_key)` in the keyset the ordering is not total, and + /// two such rows straddling a page boundary make the second unreachable: + /// the next page resumes strictly after the message key and skips its twin. + /// `uidvalidity`/`uid` are compared as the text they are stored as, which is + /// an arbitrary but total order -- and matching the `ORDER BY` exactly is + /// all a keyset needs. pub async fn list_messages_page( &self, + account_id: &str, folder_id: &str, - after: Option<(i64, String)>, + after: Option, page_size: usize, - ) -> Result<(Vec, Option<(i64, String)>), DatabaseError> { + ) -> Result<(Vec, Option), DatabaseError> { let fetch = page_size.saturating_add(1); let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages WHERE folder_id = ", + "SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, \ + m.received_at, m.body_plain, m.body_html, m.message_id, m.content_hash, \ + p.uidvalidity, p.uid, p.read_flag \ + FROM messages m JOIN placements p ON p.message_key = m.message_key \ + WHERE p.account_id = ", ); + builder.push_bind(account_id.to_string()); + builder.push(" AND p.folder_id = "); builder.push_bind(folder_id.to_string()); - if let Some((ts, id)) = &after { - builder.push(" AND (received_at < "); + if let Some((ts, key, uid_validity, remote_id)) = &after { + builder.push(" AND (m.received_at < "); builder.push_bind(*ts); - builder.push(" OR (received_at = "); + builder.push(" OR (m.received_at = "); builder.push_bind(*ts); - builder.push(" AND id < "); - builder.push_bind(id.clone()); + builder.push(" AND m.message_key < "); + builder.push_bind(key.clone()); + builder.push(") OR (m.received_at = "); + builder.push_bind(*ts); + builder.push(" AND m.message_key = "); + builder.push_bind(key.clone()); + builder.push(" AND p.uidvalidity < "); + builder.push_bind(uid_validity.clone()); + builder.push(") OR (m.received_at = "); + builder.push_bind(*ts); + builder.push(" AND m.message_key = "); + builder.push_bind(key.clone()); + builder.push(" AND p.uidvalidity = "); + builder.push_bind(uid_validity.clone()); + builder.push(" AND p.uid < "); + builder.push_bind(remote_id.clone()); builder.push("))"); } - builder.push(" ORDER BY received_at DESC, id DESC LIMIT "); + builder.push( + " ORDER BY m.received_at DESC, m.message_key DESC, \ + p.uidvalidity DESC, p.uid DESC LIMIT ", + ); builder.push_bind(fetch as i64); let rows = builder - .build_query_as::<( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option, - Option, - String, - String, - )>() + .build_query_as::() .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; let has_more = rows.len() > page_size; - let mut emails = rows - .into_iter() - .take(page_size) - .map(|r| { - let dec_plain = - r.8.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - let dec_html = - r.9.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - Ok(nuncio_core::model::Email { - id: r.0, - account_id: r.1, - folder_id: r.2, - remote_id: r.10, - uid_validity: r.11, - subject: r.3, - sender: r.4, - recipient: r.5, - received_at: r.6, - read: r.7 != 0, + let mut placed = Vec::with_capacity(rows.len().min(page_size)); + for row in rows.into_iter().take(page_size) { + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + placed.push(( + nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), - }) - }) - .collect::, DatabaseError>>()?; + message_id: row.8, + content_hash: row.9, + }, + nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: row.10, + remote_id: row.11, + read: row.12 != 0, + }, + )); + } let next = if has_more { - emails - .last() - .map(|e| (e.received_at, e.id.clone())) - .filter(|_| !emails.is_empty()) + placed.last().map(|(e, p)| { + ( + e.received_at, + e.id.clone(), + p.uid_validity.clone(), + p.remote_id.clone(), + ) + }) } else { None }; - emails.shrink_to_fit(); - Ok((emails, next)) + placed.shrink_to_fit(); + Ok((placed, next)) } - /// Retrieve a single message by ID. - #[allow(clippy::type_complexity)] + /// Retrieve a message's identity and content by its key. + /// + /// Returns the message alone. Where it sits, and whether it is read there, + /// are per-mailbox facts -- ask [`Self::placements_of`] for those. pub async fn get_message( &self, - message_id: &str, + message_key: &str, ) -> Result { - let row: ( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option, - Option, - String, - String, - ) = sqlx::query_as( + let row: MessageRow = sqlx::query_as( r#" - SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity + SELECT message_key, account_id, subject, sender, recipient, received_at, + body_plain, body_html, message_id, content_hash FROM messages - WHERE id = ? + WHERE message_key = ? "#, ) - .bind(message_id) + .bind(message_key) .fetch_one(&self.pool) .await .map_err(DatabaseError::Query)?; let dec_plain = row - .8 + .6 .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) .transpose() .map_err(|e| DatabaseError::Decryption(e.to_string()))?; let dec_html = row - .9 + .7 .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) .transpose() .map_err(|e| DatabaseError::Decryption(e.to_string()))?; @@ -1472,99 +1877,151 @@ impl DatabaseEngine { Ok(nuncio_core::model::Email { id: row.0, account_id: row.1, - folder_id: row.2, - remote_id: row.10, - uid_validity: row.11, - subject: row.3, - sender: row.4, - recipient: row.5, - received_at: row.6, - read: row.7 != 0, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, body_plain: dec_plain, body_html: dec_html, attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, }) } - /// Return the subset of `ids` already present in the `messages` table, as a single - /// `SELECT id FROM messages WHERE id IN (...)` query. + /// Every placement currently stored for one folder of one account. /// - /// Lets a caller classify a whole fetched batch as new-vs-seen with one round trip instead - /// of one existence lookup per id. Passing an empty slice short-circuits to an empty set - /// without querying at all, since `IN ()` is not valid SQL. - /// Every message id currently stored for one folder of one account. - /// - /// The local half of a UID-set diff: a sync that enumerated the folder's - /// complete contents can subtract what the server reported from this to - /// find what disappeared. Scoped by account as well as folder because - /// folder ids are not globally unique -- two accounts both have an - /// `INBOX`, and diffing across them would delete one account's mail - /// because the other's server did not mention it. - pub async fn message_ids_in_folder( + /// The local half of a UID-set diff. This returns placements rather than + /// message keys because what a server stops reporting is an occupancy, not a + /// message: the same mail may still exist in another folder and must survive + /// there. + pub async fn placements_in_folder( &self, account_id: &str, folder_id: &str, - ) -> Result, DatabaseError> { - let rows: Vec<(String,)> = - sqlx::query_as("SELECT id FROM messages WHERE account_id = ? AND folder_id = ?") - .bind(account_id) - .bind(folder_id) + ) -> Result, DatabaseError> { + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT uidvalidity, uid FROM placements WHERE account_id = ? AND folder_id = ?", + ) + .bind(account_id) + .bind(folder_id) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(uid_validity, remote_id)| PlacementKey { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity, + remote_id, + }) + .collect()) + } + + /// The subset of `keys` already stored. + /// + /// SQLite supports row-value `IN` lists, so a whole batch is one query + /// rather than one existence check per placement. An empty slice + /// short-circuits, since `IN ()` is not valid SQL. + /// + /// The batch is split internally at [`MAX_KEYS_PER_IN_LIST`] and the + /// results unioned, so callers may pass an arbitrarily large set: an entire + /// mailbox's worth of placements is exactly what a first sync produces, and + /// pushing the split onto callers means the ceiling is only ever one + /// forgetful caller away from aborting every retry of that sync + /// identically. Chunking here makes the limit unreachable from outside. + pub async fn existing_placements( + &self, + keys: &[PlacementKey], + ) -> Result, DatabaseError> { + let mut found = std::collections::HashSet::new(); + + for chunk in keys.chunks(MAX_KEYS_PER_IN_LIST) { + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT account_id, folder_id, uidvalidity, uid FROM placements + WHERE (account_id, folder_id, uidvalidity, uid) IN (", + ); + for (index, key) in chunk.iter().enumerate() { + if index > 0 { + builder.push(", "); + } + builder.push("("); + builder.push_bind(key.account_id.clone()); + builder.push(", "); + builder.push_bind(key.folder_id.clone()); + builder.push(", "); + builder.push_bind(key.uid_validity.clone()); + builder.push(", "); + builder.push_bind(key.remote_id.clone()); + builder.push(")"); + } + builder.push(")"); + + let rows: Vec<(String, String, String, String)> = builder + .build_query_as() .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - Ok(rows.into_iter().map(|(id,)| id).collect()) + + found.extend(rows.into_iter().map( + |(account_id, folder_id, uid_validity, remote_id)| PlacementKey { + account_id, + folder_id, + uid_validity, + remote_id, + }, + )); + } + + Ok(found) } - /// Delete stored messages by id, returning how many rows went. + /// Return the subset of `keys` whose message identity is already present in the + /// `messages` table, via `SELECT message_key FROM messages WHERE message_key + /// IN (...)`. /// - /// The FTS index follows via the `messages_ad` trigger, so a deleted - /// message cannot survive as a searchable plaintext orphan. - pub async fn delete_messages(&self, ids: &[String]) -> Result { - if ids.is_empty() { - return Ok(0); - } - let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; - let mut deleted = 0u64; - for id in ids { - let result = sqlx::query("DELETE FROM messages WHERE id = ?") - .bind(id) - .execute(&mut *tx) - .await - .map_err(DatabaseError::Query)?; - deleted += result.rows_affected(); - } - tx.commit().await.map_err(DatabaseError::Query)?; - Ok(deleted) - } - + /// Lets a caller classify a whole fetched batch as new-vs-seen with one round trip instead + /// of one existence lookup per key. Answers a strictly different question from + /// [`Self::existing_placements`]: a message already known from another folder is *not* new + /// here, but its arrival in this folder still is. Passing an empty slice short-circuits to + /// an empty set without querying at all, since `IN ()` is not valid SQL. + /// + /// Chunked at [`MAX_KEYS_PER_IN_LIST`] for the same reason as + /// [`Self::existing_placements`]: the bind-parameter ceiling is a property + /// of the statement, so the only place it can be enforced once and for all + /// is where the statement is built. pub async fn existing_message_ids( &self, - ids: &[String], + keys: &[String], ) -> Result, DatabaseError> { - if ids.is_empty() { - return Ok(std::collections::HashSet::new()); + let mut found = std::collections::HashSet::new(); + + for chunk in keys.chunks(MAX_KEYS_PER_IN_LIST) { + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = + sqlx::QueryBuilder::new("SELECT message_key FROM messages WHERE message_key IN ("); + let mut separated = builder.separated(", "); + for key in chunk { + separated.push_bind(key); + } + builder.push(")"); + + let rows: Vec<(String,)> = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + found.extend(rows.into_iter().map(|(key,)| key)); } - let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = - sqlx::QueryBuilder::new("SELECT id FROM messages WHERE id IN ("); - let mut separated = builder.separated(", "); - for id in ids { - separated.push_bind(id); - } - builder.push(")"); - - let rows: Vec<(String,)> = builder - .build_query_as() - .fetch_all(&self.pool) - .await - .map_err(DatabaseError::Query)?; - - Ok(rows.into_iter().map(|(id,)| id).collect()) + Ok(found) } /// Save a [`nuncio_core::model::CalendarEvent`] to SQLite (INSERT OR REPLACE). /// - /// Unlike [`Self::save_email`], calendar event summary/location are never encrypted at + /// Unlike [`Self::save_email_at`], calendar event summary/location are never encrypted at /// rest (see the confidentiality note above the `events_fts` `CREATE VIRTUAL TABLE` /// statement in [`Self::migrate`]), so the `events_ai`/`events_ad`/`events_au` triggers /// created there keep `events_fts` in sync automatically -- there is no separate manual @@ -1900,23 +2357,30 @@ impl DatabaseEngine { Ok(contact_from_row(&row)) } - /// Update a single message's read/unread flag in place. + /// Update one placement's read flag in place. /// - /// Returns `DatabaseError::Query(sqlx::Error::RowNotFound)` if no message - /// with `message_id` exists, so callers can distinguish "flag flipped" - /// from "message never existed" rather than silently succeeding on a - /// no-op update. - pub async fn set_message_read( + /// Scoped to a single mailbox occupancy because IMAP `\Seen` is per-mailbox: + /// a message read in `INBOX` is not thereby read in `Archive`. Returns + /// `DatabaseError::Query(sqlx::Error::RowNotFound)` when no such placement + /// exists, so callers can still distinguish "flag flipped" from "never + /// there" rather than silently succeeding on a no-op update. + pub async fn set_placement_read( &self, - message_id: &str, + key: &PlacementKey, read: bool, ) -> Result<(), DatabaseError> { - let result = sqlx::query("UPDATE messages SET read_flag = ? WHERE id = ?") - .bind(if read { 1i64 } else { 0i64 }) - .bind(message_id) - .execute(&self.pool) - .await - .map_err(DatabaseError::Query)?; + let result = sqlx::query( + "UPDATE placements SET read_flag = ? + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(if read { 1i64 } else { 0i64 }) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; if result.rows_affected() == 0 { return Err(DatabaseError::Query(sqlx::Error::RowNotFound)); @@ -1924,15 +2388,28 @@ impl DatabaseEngine { Ok(()) } - /// Query available folders with message counts. - pub async fn list_folders(&self) -> Result, DatabaseError> { + /// Query one account's folders with message counts. + /// + /// Folders are derived from the placements that reference them rather than + /// stored in their own table, so a folder exists exactly as long as it holds + /// mail. Scoping by account is required, not cosmetic: folder ids are not + /// globally unique -- every account has an `INBOX` -- and grouping across + /// accounts would report one merged folder holding both accounts' mail. + pub async fn list_folders( + &self, + account_id: &str, + ) -> Result, DatabaseError> { let rows: Vec<(String, i64, i64)> = sqlx::query_as( r#" - SELECT folder_id, COUNT(*) as total, SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread - FROM messages + SELECT folder_id, COUNT(*) as total, + SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread + FROM placements + WHERE account_id = ? GROUP BY folder_id + ORDER BY folder_id ASC "#, ) + .bind(account_id) .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; @@ -1948,21 +2425,24 @@ impl DatabaseEngine { .collect()) } - /// Keyset-paginated listing of folders with message counts, folder id - /// ascending. `after` is the id of the last folder of the previous page. - /// Fetches `page_size + 1` groups to detect a following page and returns - /// the id of the last returned folder as the cursor when more remain. + /// Keyset-paginated listing of one account's folders with message counts, + /// folder id ascending. `after` is the id of the last folder of the previous + /// page. Fetches `page_size + 1` groups to detect a following page and + /// returns the id of the last returned folder as the cursor when more + /// remain. Account-scoped for the same reason as [`Self::list_folders`]. pub async fn list_folders_page( &self, + account_id: &str, after: Option, page_size: usize, ) -> Result<(Vec, Option), DatabaseError> { let fetch = page_size.saturating_add(1); let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT folder_id, COUNT(*) as total, SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread FROM messages ", + "SELECT folder_id, COUNT(*) as total, SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread FROM placements WHERE account_id = ", ); + builder.push_bind(account_id.to_string()); if let Some(id) = &after { - builder.push("WHERE folder_id > "); + builder.push(" AND folder_id > "); builder.push_bind(id.clone()); } builder.push(" GROUP BY folder_id ORDER BY folder_id ASC LIMIT "); @@ -2374,157 +2854,122 @@ impl DatabaseEngine { Ok(true) } - /// Fetch a page of messages using Keyset Chunking (`WHERE id > ? ORDER BY id ASC LIMIT ?`). + /// Fetch a page of messages using Keyset Chunking + /// (`WHERE message_key > ? ORDER BY message_key ASC LIMIT ?`). + /// + /// Walks message identities, not placements: a chunked pass over the store + /// wants each message exactly once, however many folders it occupies. pub async fn get_message_chunk( &self, - last_id: &str, + last_key: &str, limit: usize, ) -> Result, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages " + "SELECT message_key, account_id, subject, sender, recipient, received_at, \ + body_plain, body_html, message_id, content_hash FROM messages ", ); - if !last_id.is_empty() { - builder.push("WHERE id > "); - builder.push_bind(last_id); + if !last_key.is_empty() { + builder.push("WHERE message_key > "); + builder.push_bind(last_key); } - builder.push(" ORDER BY id ASC LIMIT "); + builder.push(" ORDER BY message_key ASC LIMIT "); builder.push_bind(limit as i64); - let query = builder.build_query_as::<( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option, - Option, - String, - String, - )>(); + let query = builder.build_query_as::(); let rows = query .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - rows.into_iter() - .map(|r| { - let dec_plain = - r.8.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - let dec_html = - r.9.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - Ok(nuncio_core::model::Email { - id: r.0, - account_id: r.1, - folder_id: r.2, - remote_id: r.10, - uid_validity: r.11, - subject: r.3, - sender: r.4, - recipient: r.5, - received_at: r.6, - read: r.7 != 0, - body_plain: dec_plain, - body_html: dec_html, - attachments: Vec::new(), - }) - }) - .collect::, DatabaseError>>() + rows.into_iter().map(|r| self.email_from_row(r)).collect() } /// Query messages across the WHOLE store for export purposes, /// optionally narrowed to a single account or a /// single folder. Passing `None` for both returns every message in the - /// store. Ordered by `id` ascending, mirroring [`Self::get_message_chunk`]'s - /// deterministic keyset ordering. - #[allow(clippy::type_complexity)] + /// store. Ordered by `message_key` ascending, mirroring + /// [`Self::get_message_chunk`]'s deterministic keyset ordering. + /// + /// The folder filter is an `EXISTS` over `placements` rather than a join: + /// a message occupying two folders must still export as one message, and a + /// join would emit it once per matching placement. pub async fn list_messages_for_export( &self, account_id: Option<&str>, folder_id: Option<&str>, ) -> Result, DatabaseError> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( - "SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages" + "SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, \ + m.received_at, m.body_plain, m.body_html, m.message_id, m.content_hash \ + FROM messages m", ); let mut has_filter = false; if let Some(account_id) = account_id { - builder.push(" WHERE account_id = "); + builder.push(" WHERE m.account_id = "); builder.push_bind(account_id); has_filter = true; } if let Some(folder_id) = folder_id { - builder.push(if has_filter { - " AND folder_id = " - } else { - " WHERE folder_id = " - }); + builder.push(if has_filter { " AND " } else { " WHERE " }); + // Scoped to the message's own account, not just the folder name: folder ids are not + // globally unique, so an unscoped `p.folder_id` match would export another account's + // INBOX alongside this one -- the very collision this model exists to remove. + builder.push( + "EXISTS (SELECT 1 FROM placements p \ + WHERE p.message_key = m.message_key \ + AND p.account_id = m.account_id \ + AND p.folder_id = ", + ); builder.push_bind(folder_id); + builder.push(")"); } - builder.push(" ORDER BY id ASC"); + builder.push(" ORDER BY m.message_key ASC"); - let query = builder.build_query_as::<( - String, - String, - String, - String, - String, - String, - i64, - i64, - Option, - Option, - String, - String, - )>(); + let query = builder.build_query_as::(); let rows = query .fetch_all(&self.pool) .await .map_err(DatabaseError::Query)?; - rows.into_iter() - .map(|r| { - let dec_plain = - r.8.map(|p| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - let dec_html = - r.9.map(|h| { - crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h) - }) - .transpose() - .map_err(|e| DatabaseError::Decryption(e.to_string()))?; - Ok(nuncio_core::model::Email { - id: r.0, - account_id: r.1, - folder_id: r.2, - remote_id: r.10, - uid_validity: r.11, - subject: r.3, - sender: r.4, - recipient: r.5, - received_at: r.6, - read: r.7 != 0, - body_plain: dec_plain, - body_html: dec_html, - attachments: Vec::new(), - }) - }) - .collect::, DatabaseError>>() + rows.into_iter().map(|r| self.email_from_row(r)).collect() + } + + /// Rebuild an [`nuncio_core::model::Email`] from the identity-column row + /// shape shared by the whole-store read paths, decrypting the body columns + /// with this engine's storage key. + /// + /// Decryption failures propagate as [`DatabaseError::Decryption`] rather + /// than degrading to an empty body: a body that will not authenticate has + /// been tampered with or corrupted, and silently returning nothing would + /// hide exactly the event the AEAD exists to detect. + fn email_from_row(&self, row: MessageRow) -> Result { + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + + Ok(nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, + }) } /// Append a new immutable WORM audit record to the log ledger, signed with the WORM @@ -2997,17 +3442,18 @@ mod tests { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); sqlx::query( - "INSERT INTO messages (id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain) - VALUES ('msg-1', 'acct-1', 'inbox', 'Hello', 'alice@nuncio.mx', 'bob@nuncio.mx', 1700000000, 0, 'Hi Bob')", + "INSERT INTO messages (message_key, account_id, subject, sender, recipient, received_at, body_plain) + VALUES ('msg-1', 'acct-1', 'Hello', 'alice@nuncio.mx', 'bob@nuncio.mx', 1700000000, 'Hi Bob')", ) .execute(engine.pool()) .await .unwrap(); - let subject: (String,) = sqlx::query_as("SELECT subject FROM messages WHERE id = 'msg-1'") - .fetch_one(engine.pool()) - .await - .unwrap(); + let subject: (String,) = + sqlx::query_as("SELECT subject FROM messages WHERE message_key = 'msg-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); assert_eq!(subject.0, "Hello"); } @@ -3019,21 +3465,30 @@ mod tests { let email = nuncio_core::model::Email { id: "msg-db-100".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "100".to_string(), - uid_validity: "1".to_string(), subject: "Database Sync Test".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: Some("Plaintext content".to_string()), body_html: Some("

HTML content

".to_string()), attachments: Vec::new(), + message_id: None, + content_hash: None, + }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "1".to_string(), + remote_id: "100".to_string(), + read: false, }; engine - .save_email(&email) + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) .await .expect("save email succeeds"); @@ -3042,16 +3497,23 @@ mod tests { .await .expect("get message succeeds"); assert_eq!(fetched.subject, "Database Sync Test"); - assert!(!fetched.read); + assert!( + !engine.placements_of("msg-db-100").await.unwrap()[0].read, + "read state now lives on the placement, not the message" + ); let msgs = engine - .list_messages("INBOX", 10) + .list_messages("acct-1", "INBOX", 10) .await .expect("list messages succeeds"); assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0].id, "msg-db-100"); + assert_eq!(msgs[0].0.id, "msg-db-100"); + assert_eq!(msgs[0].1.remote_id, "100"); - let folders = engine.list_folders().await.expect("list folders succeeds"); + let folders = engine + .list_folders("acct-1") + .await + .expect("list folders succeeds"); assert_eq!(folders.len(), 1); assert_eq!(folders[0].id, "INBOX"); assert_eq!(folders[0].unread_messages, 1); @@ -3068,36 +3530,46 @@ mod tests { let email = nuncio_core::model::Email { id: "msg-db-corrupt".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "corrupt".to_string(), - uid_validity: "1".to_string(), subject: "Corrupted At Rest".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: Some("Sensitive plaintext body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, + }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "1".to_string(), + remote_id: "corrupt".to_string(), + read: false, }; engine - .save_email(&email) + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) .await .expect("save email succeeds"); // Flip a byte in the stored (still valid-hex) ciphertext to simulate tampering or // storage bit-rot. - let (stored,): (String,) = sqlx::query_as("SELECT body_plain FROM messages WHERE id = ?") - .bind("msg-db-corrupt") - .fetch_one(&engine.pool) - .await - .expect("fetch stored ciphertext"); + let (stored,): (String,) = + sqlx::query_as("SELECT body_plain FROM messages WHERE message_key = ?") + .bind("msg-db-corrupt") + .fetch_one(&engine.pool) + .await + .expect("fetch stored ciphertext"); let mut bytes = hex::decode(&stored).expect("stored ciphertext is valid hex"); let last_idx = bytes.len() - 1; bytes[last_idx] ^= 0xFF; let tampered = hex::encode(bytes); - sqlx::query("UPDATE messages SET body_plain = ? WHERE id = ?") + sqlx::query("UPDATE messages SET body_plain = ? WHERE message_key = ?") .bind(&tampered) .bind("msg-db-corrupt") .execute(&engine.pool) @@ -3111,10 +3583,19 @@ mod tests { assert!(matches!(err, DatabaseError::Decryption(_))); let err = engine - .list_messages("INBOX", 10) + .list_messages("acct-1", "INBOX", 10) .await .expect_err("list_messages must fail closed on corrupted ciphertext"); assert!(matches!(err, DatabaseError::Decryption(_))); + + // The export path reads the same ciphertext column and must fail closed too -- + // an export that silently dropped an unauthenticated body would write a file + // the user could not tell apart from a genuinely empty message. + let err = engine + .list_messages_for_export(None, None) + .await + .expect_err("export must fail closed on corrupted ciphertext"); + assert!(matches!(err, DatabaseError::Decryption(_))); } fn sample_calendar_event(id: &str, account_id: &str) -> nuncio_core::model::CalendarEvent { @@ -3361,55 +3842,69 @@ mod tests { )); } - /// `set_message_read` flips the persisted - /// `read_flag` in place (both directions), and reports - /// `sqlx::Error::RowNotFound` for a message ID that was never saved, - /// rather than silently succeeding on a no-op update. + /// `set_placement_read` flips the persisted `read_flag` in place (both + /// directions), and reports `sqlx::Error::RowNotFound` for a placement that + /// was never saved, rather than silently succeeding on a no-op update. #[tokio::test] - async fn set_message_read_flips_flag_and_reports_not_found_for_missing_message() { + async fn set_placement_read_flips_flag_and_reports_not_found_for_missing_placement() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); let email = nuncio_core::model::Email { id: "msg-mark-1".to_string(), account_id: "acct-1".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "mark-1".to_string(), - uid_validity: "1".to_string(), subject: "Mark Read Test".to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000, - read: false, body_plain: None, body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, + }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "1".to_string(), + remote_id: "mark-1".to_string(), + read: false, + }; + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) + .await + .expect("save email"); + + let key = PlacementKey { + account_id: "acct-1".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "1".to_string(), + remote_id: "mark-1".to_string(), }; - engine.save_email(&email).await.expect("save email"); engine - .set_message_read("msg-mark-1", true) + .set_placement_read(&key, true) .await .expect("mark read succeeds"); - let fetched = engine - .get_message("msg-mark-1") - .await - .expect("get message succeeds"); - assert!(fetched.read); + assert!(engine.placements_of("msg-mark-1").await.unwrap()[0].read); engine - .set_message_read("msg-mark-1", false) + .set_placement_read(&key, false) .await .expect("mark unread succeeds"); - let fetched = engine - .get_message("msg-mark-1") - .await - .expect("get message succeeds"); - assert!(!fetched.read); + assert!(!engine.placements_of("msg-mark-1").await.unwrap()[0].read); + let missing = PlacementKey { + remote_id: "never-placed".to_string(), + ..key.clone() + }; let err = engine - .set_message_read("msg-does-not-exist", true) + .set_placement_read(&missing, true) .await - .expect_err("marking an unknown message must fail"); + .expect_err("marking an unknown placement must fail"); assert!(matches!( err, DatabaseError::Query(sqlx::Error::RowNotFound) @@ -3840,19 +4335,31 @@ mod tests { let email = nuncio_core::model::Email { id: format!("msg-{i:03}"), account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: format!("{i}"), - uid_validity: "1".to_string(), subject: format!("Subject {i}"), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1700000000 + i, - read: false, body_plain: Some("Hello".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, }; - engine.save_email(&email).await.unwrap(); + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: format!("{i}"), + read: false, + }; + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) + .await + .unwrap(); } let chunk1 = engine.get_message_chunk("", 3).await.unwrap(); @@ -3950,40 +4457,62 @@ mod tests { ); } - /// Build a message exactly as a real sync would: an opaque surrogate id - /// hashed over its addressing coordinates, with the protocol-native id and - /// UIDVALIDITY scope carried in their own columns. + /// Build a message and its placement exactly as a real sync would when the + /// server offered no stable identity: an opaque surrogate key hashed over + /// the addressing coordinates, and a placement carrying those coordinates. fn synced_email( account_id: &str, folder_id: &str, uid_validity: &str, remote_id: &str, - ) -> nuncio_core::model::Email { - nuncio_core::model::Email { - id: nuncio_core::model::Email::surrogate_id( - account_id, - folder_id, - uid_validity, - remote_id, - ), - account_id: account_id.to_string(), - folder_id: folder_id.to_string(), - remote_id: remote_id.to_string(), - uid_validity: uid_validity.to_string(), - subject: format!("{folder_id}/{remote_id}"), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some("body".to_string()), - body_html: None, - attachments: Vec::new(), - } + ) -> (nuncio_core::model::Email, nuncio_core::model::Placement) { + ( + nuncio_core::model::Email { + id: nuncio_core::model::Email::surrogate_id( + account_id, + folder_id, + uid_validity, + remote_id, + ), + account_id: account_id.to_string(), + subject: format!("{folder_id}/{remote_id}"), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("body".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: uid_validity.to_string(), + remote_id: remote_id.to_string(), + read: false, + }, + ) + } + + /// Persist a `synced_email` pair under the surrogate identity tier. + async fn save_synced( + engine: &DatabaseEngine, + pair: &(nuncio_core::model::Email, nuncio_core::model::Placement), + ) -> SaveOutcome { + engine + .save_email_at( + &pair.0, + nuncio_core::model::IdentitySource::Surrogate, + &pair.1, + ) + .await + .expect("save synced message") } /// The C1 regression: the same protocol UID reused across folders or /// accounts must persist as DISTINCT rows (never silently overwrite), and a - /// re-sync of the SAME message must upsert the one row it already owns. + /// re-sync of the SAME message must not duplicate the row it already owns. #[tokio::test] async fn message_identity_prevents_cross_folder_and_cross_account_collision() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); @@ -3994,41 +4523,77 @@ mod tests { let sent = synced_email("acct-1", "Sent", "7", "5"); let other_account = synced_email("acct-2", "INBOX", "99", "5"); - // Every surrogate id is distinct, so none can overwrite another. - assert_ne!(inbox.id, sent.id); - assert_ne!(inbox.id, other_account.id); - assert_ne!(sent.id, other_account.id); + // Every surrogate key is distinct, so none can overwrite another. + assert_ne!(inbox.0.id, sent.0.id); + assert_ne!(inbox.0.id, other_account.0.id); + assert_ne!(sent.0.id, other_account.0.id); - engine.save_email(&inbox).await.expect("save inbox"); - engine.save_email(&sent).await.expect("save sent"); - engine - .save_email(&other_account) - .await - .expect("save other account"); + save_synced(&engine, &inbox).await; + save_synced(&engine, &sent).await; + save_synced(&engine, &other_account).await; - // All three coexist as separate rows, addressed by their own folder. - assert_eq!(engine.list_messages("INBOX", 10).await.unwrap().len(), 2); - assert_eq!(engine.list_messages("Sent", 10).await.unwrap().len(), 1); - - // Re-syncing the SAME message recomputes the SAME id, so the upsert - // updates the one row rather than duplicating it. - let inbox_resynced = synced_email("acct-1", "INBOX", "42", "5"); - assert_eq!(inbox_resynced.id, inbox.id); - engine - .save_email(&inbox_resynced) - .await - .expect("re-sync upserts"); + // All three coexist as separate messages, each reachable only through + // its own account's folder -- the second account's INBOX is not folded + // into the first's. assert_eq!( - engine.list_messages("INBOX", 10).await.unwrap().len(), - 2, - "a re-sync must upsert, not duplicate" + engine + .list_messages("acct-1", "INBOX", 10) + .await + .unwrap() + .len(), + 1 + ); + assert_eq!( + engine + .list_messages("acct-1", "Sent", 10) + .await + .unwrap() + .len(), + 1 + ); + assert_eq!( + engine + .list_messages("acct-2", "INBOX", 10) + .await + .unwrap() + .len(), + 1 + ); + let (messages,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(messages, 3); + + // Re-syncing the SAME message recomputes the SAME key, so it lands on + // the row it already owns rather than duplicating it. + let inbox_resynced = synced_email("acct-1", "INBOX", "42", "5"); + assert_eq!(inbox_resynced.0.id, inbox.0.id); + let outcome = save_synced(&engine, &inbox_resynced).await; + assert!( + !outcome.message_is_new, + "a re-sync must not create a message" + ); + assert!( + !outcome.placement_is_new, + "nor a second placement in the same mailbox" + ); + assert_eq!( + engine + .list_messages("acct-1", "INBOX", 10) + .await + .unwrap() + .len(), + 1, + "a re-sync must not duplicate" ); - // The recovered row round-trips its protocol addressing columns. - let fetched = engine.get_message(&inbox.id).await.expect("get message"); - assert_eq!(fetched.remote_id, "5"); - assert_eq!(fetched.uid_validity, "42"); - assert_eq!(fetched.folder_id, "INBOX"); + // The placement round-trips the protocol addressing coordinates. + let placements = engine.placements_of(&inbox.0.id).await.expect("placements"); + assert_eq!(placements.len(), 1); + assert_eq!(placements[0].remote_id, "5"); + assert_eq!(placements[0].uid_validity, "42"); + assert_eq!(placements[0].folder_id, "INBOX"); } #[tokio::test] @@ -4076,10 +4641,7 @@ mod tests { }), }; engine.save_account(&account).await.expect("save account"); - engine - .save_email(&synced_email("acct-reset", "INBOX", "42", "7")) - .await - .expect("save email"); + save_synced(&engine, &synced_email("acct-reset", "INBOX", "42", "7")).await; engine .save_folder_sync_state("acct-reset", "INBOX", "v1:uidnext:42:8") .await @@ -4129,49 +4691,867 @@ mod tests { .expect("re-running at the current version is a no-op"); } - fn export_test_email(id: &str, account_id: &str, folder_id: &str) -> nuncio_core::model::Email { - nuncio_core::model::Email { + #[tokio::test] + async fn placements_primary_key_includes_uidvalidity() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Same folder, same UID, different UIDVALIDITY: two distinct placements. + // After a UIDVALIDITY bump the server restarts UIDs at 1, so without + // uidvalidity in the key the new mail would overwrite the old. + for validity in ["42", "43"] { + sqlx::query( + "INSERT INTO placements (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES (?, ?, ?, ?, ?, 0)", + ) + .bind("acct-1") + .bind("INBOX") + .bind(validity) + .bind("1") + .bind(format!("key-{validity}")) + .execute(engine.pool()) + .await + .expect("both placements must insert"); + } + + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(count, 2); + } + + #[tokio::test] + async fn a_filter_fired_claim_is_won_exactly_once() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + assert!(engine.claim_filter_fire("rule-1", "key-1").await.unwrap()); + assert!( + !engine.claim_filter_fire("rule-1", "key-1").await.unwrap(), + "a second claim on the same (rule, message) must lose" + ); + assert!( + engine.claim_filter_fire("rule-2", "key-1").await.unwrap(), + "a different rule still gets its own claim" + ); + } + + #[tokio::test] + async fn bumping_the_identity_schema_rebuilds_placements_and_claims_too() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nuncio.db"); + let secrets = crate::vault::SecretManager::mock(); + + { + let engine = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .unwrap(); + sqlx::query( + "INSERT INTO placements (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES ('a', 'INBOX', '1', '1', 'k', 0)", + ) + .execute(engine.pool()) + .await + .unwrap(); + engine.claim_filter_fire("rule-1", "k").await.unwrap(); + // Pretend this file was written by the previous identity generation. + sqlx::query("PRAGMA user_version = 1") + .execute(engine.pool()) + .await + .unwrap(); + engine.close().await; + } + + let engine = DatabaseEngine::connect_file(&db_path, &secrets) + .await + .unwrap(); + let (placements,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + let (claims,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!( + placements, 0, + "placements are derived from the server and must be rebuilt" + ); + assert_eq!( + claims, 0, + "fire-once claims name message keys that no longer resolve" + ); + engine.close().await; + } + + /// An `(Email, Placement)` pair for `account_id = "acct-1"`, + /// `uid_validity = "42"` -- the shape a sync hands the store once identity + /// and occupancy are separate. + fn sample_message_and_placement( + key: &str, + folder: &str, + uid: &str, + ) -> (nuncio_core::model::Email, nuncio_core::model::Placement) { + ( + nuncio_core::model::Email { + id: key.into(), + account_id: "acct-1".into(), + subject: "Subject".into(), + sender: "a@nuncio.mx".into(), + recipient: "b@nuncio.mx".into(), + received_at: 1_000, + body_plain: Some("body".into()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::Placement { + account_id: "acct-1".into(), + folder_id: folder.into(), + uid_validity: "42".into(), + remote_id: uid.into(), + read: false, + }, + ) + } + + #[tokio::test] + async fn a_second_placement_does_not_overwrite_the_stored_body() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let email = nuncio_core::model::Email { + id: "key-1".into(), + account_id: "acct-1".into(), + subject: "Original".into(), + sender: "a@nuncio.mx".into(), + recipient: "b@nuncio.mx".into(), + received_at: 1_000, + body_plain: Some("the trusted body".into()), + body_html: None, + attachments: Vec::new(), + message_id: Some("a@b.example".into()), + content_hash: Some("1111".into()), + }; + let inbox = nuncio_core::model::Placement { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + read: false, + }; + let outcome = engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::MessageIdContent, + &inbox, + ) + .await + .unwrap(); + assert!(outcome.message_is_new && outcome.placement_is_new); + + // The same key arriving from another folder, carrying a different body. + let impostor = nuncio_core::model::Email { + subject: "Replaced".into(), + body_plain: Some("attacker body".into()), + ..email.clone() + }; + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + let outcome = engine + .save_email_at( + &impostor, + nuncio_core::model::IdentitySource::MessageIdContent, + &archive, + ) + .await + .unwrap(); + assert!(!outcome.message_is_new, "identity already existed"); + assert!(outcome.placement_is_new, "but the Archive placement is new"); + + let stored = engine.get_message("key-1").await.unwrap(); + assert_eq!(stored.subject, "Original", "first write must win"); + assert_eq!(stored.body_plain.as_deref(), Some("the trusted body")); + } + + #[tokio::test] + async fn re_placing_the_same_message_leaves_exactly_one_fts_row() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let (rows,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!( + rows, 1, + "the FTS index holds one row per message, not per placement" + ); + } + + #[tokio::test] + async fn read_state_is_tracked_per_placement() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + read: true, + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let placements = engine.placements_of("key-1").await.unwrap(); + let inbox_read = placements + .iter() + .find(|p| p.folder_id == "INBOX") + .unwrap() + .read; + let archive_read = placements + .iter() + .find(|p| p.folder_id == "Archive") + .unwrap() + .read; + assert!(!inbox_read); + assert!( + archive_read, + "IMAP \\Seen is per-mailbox, so the flags differ" + ); + } + + #[tokio::test] + async fn dropping_one_placement_keeps_the_body_searchable_from_the_other() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let inbox_key = PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }; + let outcome = engine.delete_placements(&[inbox_key]).await.unwrap(); + assert_eq!(outcome.placements_removed, 1); + assert_eq!( + outcome.messages_reaped, 0, + "the message still lives in Archive" + ); + + engine + .get_message("key-1") + .await + .expect("the message must survive"); + let (fts,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(fts, 1, "search must still find a message that still exists"); + } + + #[tokio::test] + async fn dropping_the_last_placement_reaps_the_message_and_its_plaintext_index() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + + let inbox_key = PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }; + let outcome = engine.delete_placements(&[inbox_key]).await.unwrap(); + assert_eq!(outcome.messages_reaped, 1); + + engine + .get_message("key-1") + .await + .expect_err("the message is gone"); + let (fts,): (i64,) = + sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!( + fts, 0, + "an orphaned FTS row would leave the plaintext body readable to anyone \ + with filesystem access after the message itself was deleted" + ); + } + + #[tokio::test] + async fn folders_are_derived_from_placements_and_scoped_per_account() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + for (account, folder, uid) in [ + ("acct-1", "INBOX", "1"), + ("acct-1", "Archive", "2"), + ("acct-2", "INBOX", "1"), + ] { + let (mut email, mut placement) = + sample_message_and_placement(&format!("key-{account}-{folder}"), folder, uid); + email.account_id = account.into(); + placement.account_id = account.into(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &placement, + ) + .await + .unwrap(); + } + + let folders = engine.list_folders("acct-1").await.unwrap(); + let names: Vec<&str> = folders.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(names, vec!["Archive", "INBOX"]); + assert!( + !folders.iter().any(|f| f.total_messages > 1), + "acct-2's INBOX must not be folded into acct-1's" + ); + } + + #[tokio::test] + async fn one_message_in_two_folders_counts_once_in_each() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let folders = engine.list_folders("acct-1").await.unwrap(); + assert_eq!(folders.len(), 2); + for folder in &folders { + assert_eq!( + folder.total_messages, 1, + "{} should hold the message once", + folder.id + ); + } + let (messages,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(messages, 1, "but it is still one message"); + } + + #[tokio::test] + async fn existing_placements_reports_the_occupancy_not_the_message() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + + // A *different* message really does occupy Archive. Without this the + // "Archive is not present" assertion below would hold even if the + // `IN (...)` filter were dropped entirely, since the table would have + // no Archive row to over-report in the first place. + let (other, other_in_archive) = sample_message_and_placement("key-2", "Archive", "77"); + engine + .save_email_at( + &other, + nuncio_core::model::IdentitySource::EmailId, + &other_in_archive, + ) + .await + .unwrap(); + + let in_inbox = PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }; + let in_archive = PlacementKey { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..in_inbox.clone() + }; + let never_placed = PlacementKey { + folder_id: "Sent".into(), + remote_id: "404".into(), + ..in_inbox.clone() + }; + + let found = engine + .existing_placements(&[in_inbox.clone(), in_archive.clone(), never_placed.clone()]) + .await + .unwrap(); + assert!(found.contains(&in_inbox)); + assert!( + !found.contains(&in_archive), + "the same message arriving in a new folder is a new placement and must be seen as such" + ); + assert!( + !found.contains(&never_placed), + "over-reporting would make genuinely new mail look already-seen and be skipped" + ); + assert_eq!(found.len(), 1, "and nothing else may be reported either"); + + // The message identity, by contrast, IS already known -- the two + // questions must not be conflated. + let known = engine + .existing_message_ids(&["key-1".to_string(), "key-3-never-saved".to_string()]) + .await + .unwrap(); + assert!(known.contains("key-1")); + assert!( + !known.contains("key-3-never-saved"), + "a key that was never persisted must never be reported as known" + ); + assert_eq!(known.len(), 1, "and the query must not echo its own input"); + } + + /// A first sync of a real mailbox hands `existing_placements` the entire + /// folder in one call. Each key spends four bind parameters, and SQLite + /// caps a statement at 32766 of them, so an unchunked `IN (...)` list dies + /// somewhere past 8191 keys -- and dies the same way on every retry, because + /// the folder sync aborts before writing a checkpoint. 10000 keys is + /// comfortably over that ceiling, and 6000 of them are really placed, so + /// this also fails if chunking loses or double-counts a chunk's results + /// rather than unioning them. + #[tokio::test] + async fn existing_placements_handles_a_batch_past_the_bind_parameter_ceiling() { + const PLACED: usize = 6_000; + const QUERIED: usize = 10_000; + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Insert the placement rows directly: this test is about the read + // path's statement size, and 6000 encrypt-and-commit round trips + // through `save_email_at` would test nothing extra at real cost. + let mut tx = engine.pool().begin().await.unwrap(); + for uid in 0..PLACED { + sqlx::query( + "INSERT INTO placements + (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES ('acct-1', 'INBOX', '42', ?, ?, 0)", + ) + .bind(uid.to_string()) + .bind(format!("key-{uid}")) + .execute(&mut *tx) + .await + .unwrap(); + } + tx.commit().await.unwrap(); + + let keys: Vec = (0..QUERIED) + .map(|uid| PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: uid.to_string(), + }) + .collect(); + + let found = engine.existing_placements(&keys).await.unwrap(); + + assert_eq!( + found.len(), + PLACED, + "every stored placement in the batch must be reported exactly once" + ); + assert!( + found.contains(&keys[0]) && found.contains(&keys[PLACED - 1]), + "hits at both ends of the stored range must survive the split" + ); + assert!( + !found.contains(&keys[PLACED]), + "and a key that was never placed must not be invented by the union" + ); + } + + /// The same ceiling in the one-parameter-per-key shape. 32766 parameters is + /// far away here, but the chunk boundary is not, and a batch that straddles + /// several chunks must still come back as one answer. + #[tokio::test] + async fn existing_message_ids_handles_a_batch_spanning_several_chunks() { + const QUERIED: usize = 10_000; + // Deliberately placed in different chunks of the split. + const STORED: [usize; 3] = [0, 5_000, 9_999]; + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + for (nth, index) in STORED.iter().enumerate() { + let (email, placement) = + sample_message_and_placement(&format!("key-{index}"), "INBOX", &nth.to_string()); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &placement, + ) + .await + .unwrap(); + } + + let keys: Vec = (0..QUERIED).map(|n| format!("key-{n}")).collect(); + let found = engine.existing_message_ids(&keys).await.unwrap(); + + assert_eq!(found.len(), STORED.len(), "no chunk's result may be lost"); + for index in STORED { + assert!(found.contains(&format!("key-{index}"))); + } + } + + /// `save_email_at` reads (the placement existence check) and then writes what + /// it read, so it must serialize rather than error under concurrency: in WAL + /// mode a deferred transaction upgrading to a writer aborts with + /// SQLITE_BUSY_SNAPSHOT, which `busy_timeout` does not retry. Twenty racing + /// saves of the SAME message and placement must therefore all succeed, and + /// exactly one of them must claim each of the two "is new" flags -- if two + /// callers both saw a first arrival, a fire-once filter action would run + /// twice. + #[tokio::test] + async fn concurrent_saves_of_one_placement_serialize_and_elect_one_winner() { + const CONCURRENT_SAVES: usize = 20; + + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let engine = std::sync::Arc::new(engine); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + + let mut handles = Vec::with_capacity(CONCURRENT_SAVES); + for _ in 0..CONCURRENT_SAVES { + let engine = std::sync::Arc::clone(&engine); + let email = email.clone(); + let inbox = inbox.clone(); + handles.push(tokio::spawn(async move { + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + })); + } + + let mut new_messages = 0; + let mut new_placements = 0; + for handle in handles { + let outcome = handle.await.expect("save task must not panic").expect( + "a concurrent save must never fail outright -- it must serialize, not error", + ); + new_messages += usize::from(outcome.message_is_new); + new_placements += usize::from(outcome.placement_is_new); + } + assert_eq!(new_messages, 1, "exactly one caller created the message"); + assert_eq!( + new_placements, 1, + "exactly one caller created the placement" + ); + + let (messages,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages") + .fetch_one(engine.pool()) + .await + .unwrap(); + let (fts,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages_fts") + .fetch_one(engine.pool()) + .await + .unwrap(); + let (placements,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!((messages, fts, placements), (1, 1, 1)); + } + + /// Paging a folder must yield every placement in it EXACTLY once, with no + /// dupes and no gaps -- including the case the keyset exists to survive: one + /// message occupying a single folder more than once (two `COPY`s of the same + /// mail land under different UIDs). Every row here also shares one + /// `received_at`, so the tiebreaker columns carry the whole ordering. + #[tokio::test] + async fn list_messages_page_returns_every_placement_once_no_gaps() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let mut expected: Vec<(String, String)> = Vec::new(); + for (key, uid) in [ + ("key-a", "1"), + ("key-a", "2"), + ("key-b", "3"), + ("key-b", "4"), + ("key-c", "5"), + ] { + let (email, placement) = sample_message_and_placement(key, "INBOX", uid); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &placement, + ) + .await + .unwrap(); + expected.push((key.to_string(), uid.to_string())); + } + + let mut seen: Vec<(String, String)> = Vec::new(); + let mut after: Option = None; + let mut pages = 0; + loop { + let (rows, next) = engine + .list_messages_page("acct-1", "INBOX", after.clone(), 2) + .await + .unwrap(); + pages += 1; + assert!(rows.len() <= 2, "page must not exceed page_size"); + seen.extend( + rows.iter() + .map(|(e, p)| (e.id.clone(), p.remote_id.clone())), + ); + match next { + Some(cursor) => after = Some(cursor), + None => break, + } + assert!(pages < 100, "pagination must terminate"); + } + + let mut deduped = seen.clone(); + deduped.sort(); + deduped.dedup(); + assert_eq!(deduped.len(), seen.len(), "no placement returned twice"); + + expected.sort(); + assert_eq!( + deduped, expected, + "every placement in the folder must be reachable across pages" + ); + assert!(pages >= 3, "5 rows at page_size 2 must span multiple pages"); + } + + #[tokio::test] + async fn existing_placements_with_empty_input_returns_empty_set_without_querying() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let found = engine.existing_placements(&[]).await.unwrap(); + + assert!(found.is_empty()); + } + + #[tokio::test] + async fn placements_in_folder_reports_only_that_mailbox() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let local = engine + .placements_in_folder("acct-1", "INBOX") + .await + .unwrap(); + assert_eq!( + local, + vec![PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }], + "the local half of a UID-set diff is scoped to the folder being synced" + ); + } + + #[tokio::test] + async fn marking_read_in_one_folder_leaves_the_other_unread() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { + folder_id: "Archive".into(), + remote_id: "9".into(), + ..inbox.clone() + }; + engine + .save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox) + .await + .unwrap(); + engine + .save_email_at( + &email, + nuncio_core::model::IdentitySource::EmailId, + &archive, + ) + .await + .unwrap(); + + let inbox_key = PlacementKey { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + }; + engine.set_placement_read(&inbox_key, true).await.unwrap(); + + let placements = engine.placements_of("key-1").await.unwrap(); + assert!( + placements + .iter() + .find(|p| p.folder_id == "INBOX") + .unwrap() + .read + ); + assert!( + !placements + .iter() + .find(|p| p.folder_id == "Archive") + .unwrap() + .read + ); + } + + #[tokio::test] + async fn message_provenance_round_trips_through_every_read_path() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + let mut pair = synced_email("acct-1", "INBOX", "42", "7"); + pair.0.message_id = Some("abc123@mail.nuncio.mx".to_string()); + pair.0.content_hash = Some(nuncio_core::model::Email::content_hash_of(b"raw octets")); + save_synced(&engine, &pair).await; + + let by_key = engine.get_message(&pair.0.id).await.expect("get_message"); + assert_eq!(by_key.message_id, pair.0.message_id); + assert_eq!(by_key.content_hash, pair.0.content_hash); + + let listed = engine + .list_messages("acct-1", "INBOX", 10) + .await + .expect("list"); + let found = listed + .iter() + .find(|(m, _)| m.id == pair.0.id) + .expect("the saved message is listed"); + assert_eq!(found.0.message_id, pair.0.message_id); + assert_eq!(found.0.content_hash, pair.0.content_hash); + + let exported = engine + .list_messages_for_export(None, None) + .await + .expect("export"); + let found = exported + .iter() + .find(|m| m.id == pair.0.id) + .expect("the saved message is exportable"); + assert_eq!(found.message_id, pair.0.message_id); + assert_eq!(found.content_hash, pair.0.content_hash); + + // A message with no Message-ID stays None rather than becoming "". + let plain = synced_email("acct-1", "INBOX", "42", "8"); + assert_eq!(plain.0.message_id, None); + save_synced(&engine, &plain).await; + let fetched = engine.get_message(&plain.0.id).await.expect("get_message"); + assert_eq!(fetched.message_id, None); + assert_eq!(fetched.content_hash, None); + } + + /// Save one exportable message into `folder_id` of `account_id`. + async fn save_export_message( + engine: &DatabaseEngine, + id: &str, + account_id: &str, + folder_id: &str, + ) { + let email = nuncio_core::model::Email { id: id.to_string(), account_id: account_id.to_string(), - folder_id: folder_id.to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), subject: format!("Subject {id}"), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some(format!("Body {id}")), body_html: None, attachments: Vec::new(), - } - } - - #[tokio::test] - async fn existing_message_ids_returns_only_the_ids_already_persisted() { - let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + message_id: None, + content_hash: None, + }; + let placement = nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + }; engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) + .save_email_at( + &email, + nuncio_core::model::IdentitySource::Surrogate, + &placement, + ) .await .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-a", "inbox")) - .await - .unwrap(); - - let found = engine - .existing_message_ids(&[ - "msg-1".to_string(), - "msg-2".to_string(), - "msg-3-never-saved".to_string(), - ]) - .await - .unwrap(); - - assert_eq!(found.len(), 2); - assert!(found.contains("msg-1")); - assert!(found.contains("msg-2")); - assert!(!found.contains("msg-3-never-saved")); } #[tokio::test] @@ -4186,14 +5566,8 @@ mod tests { #[tokio::test] async fn list_messages_for_export_with_no_filters_returns_every_message() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-b", "archive")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; + save_export_message(&engine, "msg-2", "acct-b", "archive").await; let all = engine.list_messages_for_export(None, None).await.unwrap(); assert_eq!(all.len(), 2); @@ -4204,14 +5578,8 @@ mod tests { #[tokio::test] async fn list_messages_for_export_filters_by_account_id() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-b", "inbox")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; + save_export_message(&engine, "msg-2", "acct-b", "inbox").await; let scoped = engine .list_messages_for_export(Some("acct-a"), None) @@ -4224,14 +5592,8 @@ mod tests { #[tokio::test] async fn list_messages_for_export_filters_by_folder_id() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-a", "archive")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; + save_export_message(&engine, "msg-2", "acct-a", "archive").await; let scoped = engine .list_messages_for_export(None, Some("archive")) @@ -4244,18 +5606,9 @@ mod tests { #[tokio::test] async fn list_messages_for_export_filters_by_account_and_folder_together() { let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-2", "acct-a", "archive")) - .await - .unwrap(); - engine - .save_email(&export_test_email("msg-3", "acct-b", "inbox")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; + save_export_message(&engine, "msg-2", "acct-a", "archive").await; + save_export_message(&engine, "msg-3", "acct-b", "inbox").await; let scoped = engine .list_messages_for_export(Some("acct-a"), Some("inbox")) @@ -4272,10 +5625,7 @@ mod tests { #[tokio::test] async fn export_messages_to_file_fails_when_worm_audit_write_fails() { let (engine, dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); - engine - .save_email(&export_test_email("msg-1", "acct-a", "inbox")) - .await - .unwrap(); + save_export_message(&engine, "msg-1", "acct-a", "inbox").await; let messages = engine.list_messages_for_export(None, None).await.unwrap(); engine.close().await; diff --git a/crates/nuncio-store/src/search.rs b/crates/nuncio-store/src/search.rs index 92d63e1..50ebf5f 100644 --- a/crates/nuncio-store/src/search.rs +++ b/crates/nuncio-store/src/search.rs @@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize}; /// Search hit result for full-text query matches. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SearchHit { - /// Entity ID matching the query. + /// Entity ID matching the query. For a message hit this is the + /// folder-independent message key: the index holds one row per message, so + /// a hit names a message and never one of its mailbox placements. Resolve + /// it through [`DatabaseEngine::get_message`]. pub id: String, /// Matching title or subject. pub title: String, @@ -50,7 +53,7 @@ impl<'a> SearchEngine<'a> { let rows: Vec<(String, String, String)> = sqlx::query_as( r#" - SELECT id, subject, snippet(messages_fts, 3, '', '', '...', 10) as snippet + SELECT message_key, subject, snippet(messages_fts, 3, '', '', '...', 10) as snippet FROM messages_fts WHERE messages_fts MATCH ? ORDER BY rank @@ -94,7 +97,7 @@ impl<'a> SearchEngine<'a> { let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( "SELECT id, title, snippet, rk FROM ( \ - SELECT id, subject AS title, \ + SELECT message_key AS id, subject AS title, \ snippet(messages_fts, 3, '', '', '...', 10) AS snippet, rank AS rk \ FROM messages_fts WHERE messages_fts MATCH ", ); @@ -255,21 +258,39 @@ mod tests { nuncio_core::model::Email { id: id.to_string(), account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "bob@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some(body_plain.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } - /// The crux test: proves body search works over the REAL `save_email` write path (no + fn sample_placement(uid: &str) -> nuncio_core::model::Placement { + nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: uid.to_string(), + read: false, + } + } + + async fn save(db: &DatabaseEngine, email: &nuncio_core::model::Email) { + db.save_email_at( + email, + nuncio_core::model::IdentitySource::Surrogate, + &sample_placement(&email.id), + ) + .await + .unwrap(); + } + + /// The crux test: proves body search works over the REAL `save_email_at` write path (no /// raw-SQL plaintext injection bypassing encryption), while the stored body column stays /// ciphertext. This is what the old `fts5_message_search_and_triggers` test failed to /// prove -- it inserted plaintext directly via raw SQL, which trivially "worked" but never @@ -288,7 +309,7 @@ mod tests { "Quarterly Financial Meeting", "Let's discuss the annual budget revenue forecast", ); - db.save_email(&email).await.unwrap(); + save(&db, &email).await; // Body search matches a plaintext body term -- proves the FTS index holds real, // decrypted-at-write-time plaintext trigrams, not the AES-256-GCM ciphertext that is @@ -305,39 +326,44 @@ mod tests { // The body column at rest must remain encrypted ciphertext -- never the plaintext // search term, and never equal to the original plaintext body. let (stored_body,): (String,) = - sqlx::query_as("SELECT body_plain FROM messages WHERE id = 'msg-1'") + sqlx::query_as("SELECT body_plain FROM messages WHERE message_key = 'msg-1'") .fetch_one(db.pool()) .await .unwrap(); assert_ne!(stored_body, email.body_plain.clone().unwrap()); assert!(!stored_body.to_lowercase().contains("revenue")); - // Re-saving via save_email (an update, not a fresh insert) replaces the FTS row rather - // than duplicating or leaving a stale entry: the old term no longer matches, the new - // one does. - let mut updated = email.clone(); - updated.body_plain = Some("Updated strategy review notes".to_string()); - db.save_email(&updated).await.unwrap(); + // Re-saving the same key with a different body must leave the index alone: the message + // row is first-write-wins, so a second write that cannot change the stored body must + // not change what search returns for it either. Anything else would have search + // disagree with the store -- either finding a body that was never persisted, or holding + // two rows for one message. + let mut rewritten = email.clone(); + rewritten.body_plain = Some("Updated strategy review notes".to_string()); + save(&db, &rewritten).await; - assert!(search.search_messages("revenue").await.unwrap().is_empty()); - let new_hits = search.search_messages("strategy").await.unwrap(); - assert_eq!(new_hits.len(), 1); - assert_eq!(new_hits[0].id, "msg-1"); + let unchanged = search.search_messages("revenue").await.unwrap(); + assert_eq!(unchanged.len(), 1, "the stored body is still searchable"); + assert_eq!(unchanged[0].id, "msg-1"); + assert!( + search.search_messages("strategy").await.unwrap().is_empty(), + "a body the store rejected must never become searchable" + ); - // Delete message trigger still cleans up the FTS row (subject/sender/id only -- no + // Delete message trigger still cleans up the FTS row (subject/sender/key only -- no // ciphertext involved). - sqlx::query("DELETE FROM messages WHERE id = 'msg-1'") + sqlx::query("DELETE FROM messages WHERE message_key = 'msg-1'") .execute(db.pool()) .await .unwrap(); - let deleted_hits = search.search_messages("strategy").await.unwrap(); + let deleted_hits = search.search_messages("revenue").await.unwrap(); assert!(deleted_hits.is_empty()); } /// Proves the backfill half of the fix: a message saved through the real write path, /// whose FTS entry is then lost (simulating either a row written before the FTS5 index - /// existed, or one written by a process that bypassed `save_email`'s explicit indexing), + /// existed, or one written by a process that bypassed `save_email_at`'s explicit indexing), /// is NOT searchable until a migration pass runs -- at which point `backfill_message_fts` /// decrypts the ciphertext body column with this engine's real storage key and repopulates /// the plaintext-derived trigram index, with no search call ever required to trigger it. @@ -349,11 +375,11 @@ mod tests { "Legacy Roadmap Notes", "Confidential quarterly roadmap details", ); - db.save_email(&email).await.unwrap(); + save(&db, &email).await; // Simulate pre-existing data whose FTS entry is missing (e.g. written before the FTS5 // index existed) by dropping its messages_fts row directly. - sqlx::query("DELETE FROM messages_fts WHERE id = 'msg-legacy-1'") + sqlx::query("DELETE FROM messages_fts WHERE message_key = 'msg-legacy-1'") .execute(db.pool()) .await .unwrap(); @@ -383,13 +409,15 @@ mod tests { async fn search_messages_page_returns_every_hit_once_no_gaps() { let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); for i in 0..5 { - db.save_email(&sample_email( - &format!("msg-{i}"), - "Budget Planning Meeting", - "annual budget revenue forecast discussion", - )) - .await - .unwrap(); + save( + &db, + &sample_email( + &format!("msg-{i}"), + "Budget Planning Meeting", + "annual budget revenue forecast discussion", + ), + ) + .await; } let search = SearchEngine::new(&db); diff --git a/crates/nunciod/src/grpc.rs b/crates/nunciod/src/grpc.rs index 8d387bf..1a9b8be 100644 --- a/crates/nunciod/src/grpc.rs +++ b/crates/nunciod/src/grpc.rs @@ -963,23 +963,28 @@ fn map_attachment_from_proto(attachment: AttachmentProto) -> nuncio_core::model: } } -/// Maps a `nuncio_core::model::Email` onto its wire-format `nuncio.v1.Message` -/// representation. The store (`DatabaseEngine::get_message` / -/// `list_messages`) already returns `body_plain`/`body_html` decrypted, so -/// there is no further decryption to do here -- just field-for-field -/// mapping. -fn map_email_to_proto(email: nuncio_core::model::Email) -> MessageProto { +/// Maps a message and the mailbox occupancy it was read through onto the +/// wire-format `nuncio.v1.Message` representation. The store +/// (`DatabaseEngine::get_message` / `list_messages`) already returns +/// `body_plain`/`body_html` decrypted, so there is no further decryption to do +/// here -- just field-for-field mapping. +/// +/// `folder_id` and `read` come from the placement: both are per-mailbox facts +/// now, and the wire message still carries exactly one of each, so it describes +/// one occupancy of the message rather than the message as a whole. +fn map_email_to_proto(placed: nuncio_store::db::MessageWithPlacement) -> MessageProto { + let (email, placement) = placed; MessageProto { id: email.id, account_id: email.account_id, - folder_id: email.folder_id, + folder_id: placement.folder_id, subject: email.subject, sender: email.sender, recipient: email.recipient, received_at: Some(nuncio_proto::time::timestamp_from_unix_secs( email.received_at, )), - read: email.read, + read: placement.read, body_plain: email.body_plain, body_html: email.body_html, attachments: email @@ -1633,8 +1638,163 @@ struct MailGrpcService { sync_dispatcher: Arc, } +/// Account-merging read helpers. +/// +/// The store's folder and message reads are account-scoped, because folder ids +/// are not globally unique -- every account has an `INBOX`, and grouping across +/// accounts would report one merged folder holding two accounts' mail. The +/// `nuncio.v1.Mail` requests carry no account field, so these reproduce the +/// pre-split cross-account behaviour by asking each account and merging. They +/// are a compatibility shim, not the destination: the honest fix is an account +/// on the wire, which lands with the placement-aware surface. +impl MailGrpcService { + /// One page of folders, merged across every configured account. + /// + /// Counts for a folder id that several accounts share are summed, matching + /// what the old account-blind `GROUP BY folder_id` returned. + async fn merged_folders_page( + &self, + after: Option, + page_size: usize, + ) -> Result<(Vec, Option), nuncio_store::db::DatabaseError> + { + let accounts = self.db.list_accounts().await?; + let mut merged: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + // One more than the page from each account, for the same reason the + // store itself over-fetches by one: the merge cannot tell "this is the + // last page" from "this account was truncated at exactly page_size" + // unless at least one extra row can show up. + let probe = page_size.saturating_add(1); + for account in &accounts { + // Each account is asked from the same cursor; the merge below + // re-truncates, so no account's page is starved by another's. + let (folders, _) = self + .db + .list_folders_page(&account.id, after.clone(), probe) + .await?; + for folder in folders { + merged + .entry(folder.id.clone()) + .and_modify(|existing| { + existing.total_messages += folder.total_messages; + existing.unread_messages += folder.unread_messages; + }) + .or_insert(folder); + } + } + + let mut folders: Vec = merged.into_values().collect(); + let has_more = folders.len() > page_size; + folders.truncate(page_size); + let next = has_more + .then(|| folders.last().map(|f| f.id.clone())) + .flatten(); + Ok((folders, next)) + } + + /// One page of a folder's messages, merged across every configured account, + /// in the store's own `received_at DESC, message_key DESC, uidvalidity + /// DESC, uid DESC` order. + async fn merged_messages_page( + &self, + folder_id: &str, + after: Option, + page_size: usize, + ) -> Result< + ( + Vec, + Option, + ), + nuncio_store::db::DatabaseError, + > { + let accounts = self.db.list_accounts().await?; + let mut rows: Vec = Vec::new(); + // One more than the page from each account -- see `merged_folders_page` + // for why the extra row is what makes "is there another page?" + // answerable after the merge. + let probe = page_size.saturating_add(1); + for account in &accounts { + let (page, _) = self + .db + .list_messages_page(&account.id, folder_id, after.clone(), probe) + .await?; + rows.extend(page); + } + + // Re-impose the store's ordering over the concatenated per-account + // pages, so the cursor this returns is a position in the merged + // sequence rather than in whichever account happened to be last. + rows.sort_by(|(a_mail, a_place), (b_mail, b_place)| { + b_mail + .received_at + .cmp(&a_mail.received_at) + .then_with(|| b_mail.id.cmp(&a_mail.id)) + .then_with(|| b_place.uid_validity.cmp(&a_place.uid_validity)) + .then_with(|| b_place.remote_id.cmp(&a_place.remote_id)) + }); + + let has_more = rows.len() > page_size; + rows.truncate(page_size); + let next = has_more + .then(|| { + rows.last().map(|(email, placement)| { + ( + email.received_at, + email.id.clone(), + placement.uid_validity.clone(), + placement.remote_id.clone(), + ) + }) + }) + .flatten(); + Ok((rows, next)) + } +} + +/// The mailbox occupancy a message-scoped RPC should speak for. +/// +/// `GetMessage`/`MarkRead`/`PreviewRule` name a message, but `folder_id` and the +/// read flag are per-mailbox now, so one occupancy has to be chosen. This takes +/// the store's first in its deterministic order, so every message-scoped RPC +/// picks the same one and they stay consistent with each other. +/// +/// The two failure modes are deliberately distinct and neither is papered over: +/// a read error surfaces as `internal`, and a message with no occupancy is +/// reported as not found. A message that occupies no mailbox is not "unread in +/// no folder" -- it is gone -- and inventing a location for it would let a +/// `WHERE FOLDER = …` preview match somewhere the message has never been. +/// +/// A free function rather than a method because `Mail` and `Filters` are +/// separate services that both need it. +async fn primary_placement( + db: &DatabaseEngine, + message_key: &str, +) -> Result { + let placements = db.placements_of(message_key).await.map_err(|e| { + Status::internal(format!( + "failed to read placements for '{message_key}': {e}" + )) + })?; + placements.into_iter().next().ok_or_else(|| { + errors::status_with_metadata( + ErrorReason::MessageNotFound, + format!("message '{message_key}' occupies no mailbox"), + [("message_id".to_string(), message_key.to_string())], + ) + }) +} + #[tonic::async_trait] impl Mail for MailGrpcService { + /// Lists folders across every **configured** account, merged by folder id. + /// + /// Mail whose owning account no longer has a configuration row is not + /// listed, even though its rows survive in the store. That is deliberate: + /// removing an account should stop its cached mail appearing, and the + /// account list is what defines "whose mail this daemon serves". The rows + /// are still there for an explicit `Export`, and are reachable again if the + /// account is re-added. async fn list_folders( &self, request: Request, @@ -1646,9 +1806,13 @@ impl Mail for MailGrpcService { _ => None, })?; + // The store scopes folders by account -- folder ids are not globally + // unique, every account has an INBOX -- while this RPC has no account + // field yet. Merging every account's page by folder id reproduces the + // pre-split behaviour exactly; narrowing it properly needs a wire + // change and belongs with the placement-aware surface. let (folders, next) = self - .db - .list_folders_page(after, page_size) + .merged_folders_page(after, page_size) .await .map_err(|e| Status::internal(format!("failed to list folders: {e}")))?; @@ -1663,6 +1827,13 @@ impl Mail for MailGrpcService { })) } + /// Lists one folder's messages across every **configured** account, merged + /// into a single newest-first sequence. + /// + /// As with [`Self::list_folders`], mail whose owning account no longer has + /// a configuration row is not listed: the account list defines whose mail + /// this daemon serves, so removing an account hides its cached mail rather + /// than leaving it visible under a folder nobody owns. async fn list_messages( &self, request: Request, @@ -1672,19 +1843,31 @@ impl Mail for MailGrpcService { return Err(Status::invalid_argument("folder_id is required")); } let page_size = pagination::clamp_page_size(req.page_size); + // Four cursor fields, not two: the store's keyset addresses a + // placement (`received_at, message_key, uidvalidity, uid`), because one + // message can occupy a single folder twice and a message-only cursor + // makes the second copy unreachable across a page boundary. let after = pagination::parse_token(&req.page_token, |f| match f { - [CursorField::I(ts), CursorField::S(id)] => Some((*ts, id.clone())), + [CursorField::I(ts), CursorField::S(key), CursorField::S(uidvalidity), CursorField::S(uid)] => { + Some((*ts, key.clone(), uidvalidity.clone(), uid.clone())) + } _ => None, })?; let (messages, next) = self - .db - .list_messages_page(&req.folder_id, after, page_size) + .merged_messages_page(&req.folder_id, after, page_size) .await .map_err(|e| Status::internal(format!("failed to list messages: {e}")))?; let next_page_token = next - .map(|(ts, id)| pagination::encode_cursor(&[CursorField::I(ts), CursorField::S(id)])) + .map(|(ts, key, uidvalidity, uid)| { + pagination::encode_cursor(&[ + CursorField::I(ts), + CursorField::S(key), + CursorField::S(uidvalidity), + CursorField::S(uid), + ]) + }) .unwrap_or_default(); let messages = messages.into_iter().map(map_email_to_proto).collect(); @@ -1711,8 +1894,10 @@ impl Mail for MailGrpcService { ) })?; + let placement = primary_placement(&self.db, &req.message_id).await?; + Ok(Response::new(GetMessageResponse { - message: Some(map_email_to_proto(email)), + message: Some(map_email_to_proto((email, placement))), })) } @@ -1733,8 +1918,13 @@ impl Mail for MailGrpcService { return Err(Status::invalid_argument("message_id is required")); } + // `\Seen` is per-mailbox, and this request names only a message, so it + // is applied to the occupancy `GetMessage` would report -- keeping the + // two RPCs consistent with each other. Naming a placement on the wire + // is what makes this well-defined, and lands with that surface. + let placement = primary_placement(&self.db, &req.message_id).await?; self.db - .set_message_read(&req.message_id, req.read) + .set_placement_read(&placement.key(), req.read) .await .map_err(|e| { errors::status_with_metadata( @@ -1977,18 +2167,28 @@ fn synthetic_preview_email(id: &str) -> nuncio_core::model::Email { nuncio_core::model::Email { id: id.to_string(), account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - // Synthetic preview only -- never persisted or mutated remotely. - remote_id: id.to_string(), - uid_validity: String::new(), subject: "Test Subject".to_string(), sender: "test@nuncio.mx".to_string(), recipient: "me@nuncio.mx".to_string(), received_at, - read: false, body_plain: Some("Sample body text".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, + } +} + +/// The occupancy a synthetic preview message is pretended to sit in, so +/// `WHERE FOLDER = ...` and `WHERE ACCOUNT = ...` have something to answer +/// against. Synthetic only -- never persisted, never mutated remotely. +fn synthetic_preview_placement(id: &str) -> nuncio_core::model::Placement { + nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, } } @@ -2173,17 +2373,41 @@ impl Filters for FiltersGrpcService { let preview_engine = FilterEngine::new(vec![rule]) .map_err(|e| Status::internal(format!("failed to build preview engine: {e}")))?; - let email = match &req.message_id { + // A preview needs a placement as well as a message, because the rule + // language can ask where the message sits. + // + // For a message that IS stored, the occupancy must be the real one. + // Substituting an invented `inbox` placement -- on a read error, or for + // a message with none -- would let `WHERE FOLDER = 'inbox'` report a + // match against a location the message does not have, which is the one + // thing a preview must never do. So a read error surfaces, and a stored + // message with no occupancy is reported as not found rather than + // previewed somewhere imaginary; `primary_placement` does both. + // + // A message id that resolves to nothing at all still falls back to the + // fully synthetic pair, exactly as before: that is the documented + // "preview against a sample" path, and it invents a message as well as + // a placement, so it claims nothing about stored mail. + let (email, placement) = match &req.message_id { Some(message_id) if !message_id.is_empty() => { match self.db.get_message(message_id).await { - Ok(email) => email, - Err(_) => synthetic_preview_email(message_id), + Ok(email) => (email, primary_placement(&self.db, message_id).await?), + Err(_) => ( + synthetic_preview_email(message_id), + synthetic_preview_placement(message_id), + ), } } - _ => synthetic_preview_email("msg-test"), + _ => ( + synthetic_preview_email("msg-test"), + synthetic_preview_placement("msg-test"), + ), }; - let preview = preview_engine.preview(&email); + let preview = preview_engine.preview(nuncio_filter::PlacedEmail { + email: &email, + placement: &placement, + }); Ok(Response::new(map_preview_result_to_proto(preview))) } @@ -2419,8 +2643,28 @@ impl Filters for FiltersGrpcService { let is_last_page = batch.len() < chunk_size; for email in &batch { - let actions_for_email = - crate::sync::apply_filter_actions(&db, &engine, email).await; + // Rules can ask where a message is, so a retroactive + // pass evaluates each mailbox the message occupies -- + // the same fan-out live sync does. The fire-once claim + // inside `apply_filter_actions` keeps a message in two + // folders from acting twice. + let placements = match db.placements_of(&email.id).await { + Ok(placements) => placements, + Err(e) => { + let _ = tx + .send(Err(Status::internal(format!( + "failed to read placements during triage: {e}" + )))) + .await; + return; + } + }; + let mut actions_for_email = 0usize; + for placement in &placements { + actions_for_email += + crate::sync::apply_filter_actions(&db, &engine, email, placement) + .await; + } scanned += 1; applied += actions_for_email as u64; if actions_for_email > 0 { @@ -3294,20 +3538,25 @@ mod tests { let email = nuncio_core::model::Email { id: "msg-1".to_string(), account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), subject: "s".to_string(), sender: "a@b.com".to_string(), recipient: "c@d.com".to_string(), received_at: original_secs, - read: false, body_plain: None, body_html: None, attachments: Vec::new(), - remote_id: "remote-1".to_string(), + message_id: None, + content_hash: None, + }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), uid_validity: "1".to_string(), + remote_id: "remote-1".to_string(), + read: false, }; - let proto = map_email_to_proto(email); + let proto = map_email_to_proto((email, placement)); let ts = proto.received_at.expect("received_at is always populated"); assert_eq!( nuncio_proto::time::timestamp_to_unix_secs(&ts), @@ -4478,35 +4727,80 @@ mod tests { SearchMessagesRequest, SendMessageRequest, SyncRequest, }; - fn sample_email( + /// A message and the one mailbox occupancy the tests seed it into. + fn sample_placed( id: &str, folder_id: &str, subject: &str, body: &str, - ) -> nuncio_core::model::Email { - nuncio_core::model::Email { - id: id.to_string(), - account_id: "acct-mail-1".to_string(), - folder_id: folder_id.to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), - subject: subject.to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some(body.to_string()), - body_html: None, - attachments: Vec::new(), + ) -> (nuncio_core::model::Email, nuncio_core::model::Placement) { + ( + nuncio_core::model::Email { + id: id.to_string(), + account_id: "acct-mail-1".to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some(body.to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::Placement { + account_id: "acct-mail-1".to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + }, + ) + } + + /// Persist a message into its occupancy the way a sync pass would, first + /// making sure the owning account row exists. + /// + /// The account row is not decoration: the `Mail` read RPCs carry no account + /// field and so merge across the configured accounts, which means mail + /// belonging to no account is unreachable through them. That is also true + /// in production -- mail only ever arrives for an account the daemon holds + /// a configuration for -- so seeding one keeps these tests on the real path. + async fn seed_message( + db: &DatabaseEngine, + email: &nuncio_core::model::Email, + placement: &nuncio_core::model::Placement, + ) { + let existing = db.list_accounts().await.expect("list accounts"); + if !existing.iter().any(|a| a.id == placement.account_id) { + db.save_account(&nuncio_core::AccountConfig { + id: placement.account_id.clone(), + name: placement.account_id.clone(), + email_address: format!("{}@nuncio.mx", placement.account_id), + keyring_secret_key: format!("nuncio/{}", placement.account_id), + sync_interval_secs: 60, + transport: nuncio_core::Transport::Jmap(nuncio_core::JmapTransport { + endpoint_host: "jmap.nuncio.mx".to_string(), + }), + }) + .await + .expect("save the owning account"); } + db.save_email_at( + email, + nuncio_core::model::IdentitySource::Surrogate, + placement, + ) + .await + .expect("seed message"); } /// Paging a folder larger than `page_size` across multiple pages via /// `next_page_token` must return every message EXACTLY ONCE, with no /// duplicates and no gaps, and in the stable newest-first keyset order - /// (`received_at DESC, id DESC`). Some messages deliberately share a - /// `received_at` so the `(received_at, id)` tiebreaker is exercised across - /// a page boundary. + /// (`received_at DESC, message_key DESC, uidvalidity DESC, uid DESC`). + /// Some messages deliberately share a `received_at` so the tiebreaker + /// fields are exercised across a page boundary. #[tokio::test] async fn list_messages_keyset_pagination_returns_every_message_once_no_gaps() { let (db, _dir) = DatabaseEngine::connect_ephemeral() @@ -4522,9 +4816,9 @@ mod tests { ("m7", 1_700_000_030), ]; for (id, ts) in seed { - let mut email = sample_email(id, "inbox", "subject", "body"); + let (mut email, placement) = sample_placed(id, "inbox", "subject", "body"); email.received_at = ts; - db.save_email(&email).await.expect("seed message"); + seed_message(&db, &email, &placement).await; } let event_bus = Arc::new(EventBus::new()); @@ -4578,6 +4872,253 @@ mod tests { assert!(pages >= 4, "a 7-item set at page_size 2 must span >1 page"); } + // ---- Cross-account merge shims ------------------------------------- + // + // `ListFolders`/`ListMessages` carry no account field, so the handlers ask + // every configured account and merge. These exercise that merge across two + // accounts, which nothing else does: the keyset test above seeds one + // account, and the export scoping test bypasses the Mail service entirely. + + /// Seed one message into `account_id`'s copy of `folder_id`, at + /// `received_at`, creating the account row if it is not there yet. + async fn seed_in_account( + db: &DatabaseEngine, + account_id: &str, + id: &str, + folder_id: &str, + received_at: i64, + ) { + let (mut email, mut placement) = sample_placed(id, folder_id, "subject", "body"); + email.account_id = account_id.to_string(); + email.received_at = received_at; + placement.account_id = account_id.to_string(); + seed_message(db, &email, &placement).await; + } + + /// Boot a Mail server over `db` and return a connected client. + async fn mail_client_over(db: DatabaseEngine) -> MailClient { + let (addr, _handle) = spawn_test_server_with( + Arc::new(EventBus::new()), + Arc::new(db), + Arc::new(FilterEngine::new(Vec::new()).expect("empty rule set")), + Arc::new(SecretManager::mock()), + "correct-token", + ) + .await; + // The handle is dropped: `spawn_test_server_with` detaches the server + // task, and every test here finishes its calls before returning. + MailClient::connect(format!("http://{addr}")) + .await + .expect("client connects") + } + + /// Two accounts both have an `INBOX`. Folder ids are not globally unique, + /// so the merge has to fold them into ONE wire folder whose counts are the + /// sum -- which is what the account-blind `GROUP BY folder_id` used to + /// return before the store became account-scoped. + #[tokio::test] + async fn list_folders_sums_counts_for_a_folder_id_two_accounts_share() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + seed_in_account(&db, "acct-a", "m-a1", "inbox", 1_700_000_001).await; + seed_in_account(&db, "acct-a", "m-a2", "inbox", 1_700_000_002).await; + seed_in_account(&db, "acct-b", "m-b1", "inbox", 1_700_000_003).await; + // A folder only one of the two accounts has must survive the merge + // intact rather than being lost or double-counted. + seed_in_account(&db, "acct-b", "m-b2", "archive", 1_700_000_004).await; + + let mut client = mail_client_over(db).await; + let resp = client + .list_folders(authed_bearer_request(ListFoldersRequest { + page_size: 0, + page_token: String::new(), + })) + .await + .expect("list_folders succeeds") + .into_inner(); + + let mut folders = resp.folders; + folders.sort_by(|a, b| a.id.cmp(&b.id)); + assert_eq!( + folders.iter().map(|f| f.id.as_str()).collect::>(), + vec!["archive", "inbox"], + "one wire folder per distinct id, however many accounts hold it" + ); + let inbox = &folders[1]; + assert_eq!(inbox.total_messages, 3, "2 from acct-a + 1 from acct-b"); + assert_eq!(inbox.unread_messages, 3); + assert_eq!(folders[0].total_messages, 1); + } + + /// Paging a folder whose messages come from two accounts must return every + /// message exactly once, newest-first, with pages that straddle the account + /// boundary. Timestamps interleave so no page can be satisfied from a + /// single account -- if the merge ever paged one account at a time, the + /// order would come out wrong even though the set did not. + #[tokio::test] + async fn list_messages_merges_two_accounts_across_page_boundaries_without_gaps() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + seed_in_account(&db, "acct-a", "m-a1", "inbox", 1_700_000_001).await; + seed_in_account(&db, "acct-b", "m-b2", "inbox", 1_700_000_002).await; + seed_in_account(&db, "acct-a", "m-a3", "inbox", 1_700_000_003).await; + seed_in_account(&db, "acct-b", "m-b4", "inbox", 1_700_000_004).await; + seed_in_account(&db, "acct-a", "m-a5", "inbox", 1_700_000_005).await; + + let mut client = mail_client_over(db).await; + + let mut seen = Vec::new(); + let mut page_sizes = Vec::new(); + let mut page_token = String::new(); + let mut pages = 0; + loop { + let resp = client + .list_messages(authed_bearer_request(ListMessagesRequest { + folder_id: "inbox".to_string(), + page_size: 2, + page_token: page_token.clone(), + })) + .await + .expect("list_messages page succeeds") + .into_inner(); + pages += 1; + assert!( + resp.messages.len() <= 2, + "a merged page must still respect page_size" + ); + page_sizes.push(resp.messages.len()); + seen.extend(resp.messages.iter().map(|m| m.id.clone())); + if resp.next_page_token.is_empty() { + break; + } + page_token = resp.next_page_token; + assert!(pages < 100, "pagination must terminate"); + } + + assert_eq!( + seen, + vec!["m-a5", "m-b4", "m-a3", "m-b2", "m-a1"], + "the merged listing must be newest-first across BOTH accounts, once each" + ); + assert_eq!( + page_sizes, + vec![2, 2, 1], + "and it must genuinely span pages that straddle the account boundary" + ); + } + + /// A configured account holding nothing in this folder must not truncate + /// the listing. + /// + /// This is the exact shape the merge gets wrong if it asks each account for + /// only `page_size` rows: one account returns a full page, the other + /// returns none, the merged total lands on `page_size` exactly, and "is + /// there more?" reads as no -- so pages 2 and 3 are silently unreachable. + /// The interleaved test above cannot see it, because there both accounts + /// contribute and the total always overshoots. + #[tokio::test] + async fn a_folder_owned_by_one_account_still_pages_past_the_first_page() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + for (index, id) in ["m-1", "m-2", "m-3", "m-4", "m-5"].iter().enumerate() { + seed_in_account(&db, "acct-a", id, "inbox", 1_700_000_000 + index as i64).await; + } + // Configured, but with nothing in `inbox` -- an ordinary second account + // whose mail lives elsewhere. + seed_in_account(&db, "acct-b", "m-elsewhere", "archive", 1_700_000_009).await; + + let mut client = mail_client_over(db).await; + + let mut seen = Vec::new(); + let mut page_token = String::new(); + let mut pages = 0; + loop { + let resp = client + .list_messages(authed_bearer_request(ListMessagesRequest { + folder_id: "inbox".to_string(), + page_size: 2, + page_token: page_token.clone(), + })) + .await + .expect("list_messages page succeeds") + .into_inner(); + pages += 1; + seen.extend(resp.messages.iter().map(|m| m.id.clone())); + if resp.next_page_token.is_empty() { + break; + } + page_token = resp.next_page_token; + assert!(pages < 100, "pagination must terminate"); + } + + assert_eq!( + seen, + vec!["m-5", "m-4", "m-3", "m-2", "m-1"], + "an empty second account must not cut the listing short" + ); + } + + /// Removing an account hides its cached mail, even though the rows survive + /// in the store for an explicit export. This is intended behaviour, not an + /// accident of the merge -- it is the rule both RPCs' doc comments state, + /// and it is why seeding a message in these tests also persists an account. + #[tokio::test] + async fn mail_for_a_deleted_account_is_no_longer_listed() { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("connect ephemeral test db"); + seed_in_account(&db, "acct-keep", "m-keep", "inbox", 1_700_000_001).await; + seed_in_account(&db, "acct-drop", "m-drop", "inbox", 1_700_000_002).await; + + db.delete_account("acct-drop") + .await + .expect("delete the account"); + // The mail itself is deliberately NOT deleted with the account, so the + // test is about visibility, not about a cascade. + assert!( + !db.placements_of("m-drop") + .await + .expect("read placements") + .is_empty(), + "the orphaned mail must still be in the store, just unlisted" + ); + + let mut client = mail_client_over(db).await; + let messages = client + .list_messages(authed_bearer_request(ListMessagesRequest { + folder_id: "inbox".to_string(), + page_size: 50, + page_token: String::new(), + })) + .await + .expect("list_messages succeeds") + .into_inner() + .messages; + assert_eq!( + messages.iter().map(|m| m.id.as_str()).collect::>(), + vec!["m-keep"], + "mail belonging to a removed account must not be listed" + ); + + let folders = client + .list_folders(authed_bearer_request(ListFoldersRequest { + page_size: 50, + page_token: String::new(), + })) + .await + .expect("list_folders succeeds") + .into_inner() + .folders; + assert_eq!(folders.len(), 1); + assert_eq!( + folders[0].total_messages, 1, + "the removed account's mail must not be counted either" + ); + } + /// A `page_token` the server cannot decode is rejected with a typed /// `VALIDATION_FAILED` `ErrorInfo` mapped to `InvalidArgument` -- never a /// panic and never a silently-empty first page. @@ -4799,22 +5340,16 @@ mod tests { let (db, _dir) = DatabaseEngine::connect_ephemeral() .await .expect("connect ephemeral test db"); - db.save_email(&sample_email( + let (email_1, placement_1) = sample_placed( "msg-mail-1", "inbox", "Quarterly Roadmap", "Let's discuss the annual revenue forecast", - )) - .await - .expect("seed message 1"); - db.save_email(&sample_email( - "msg-mail-2", - "inbox", - "Lunch Plans", - "Sandwiches at noon", - )) - .await - .expect("seed message 2"); + ); + seed_message(&db, &email_1, &placement_1).await; + let (email_2, placement_2) = + sample_placed("msg-mail-2", "inbox", "Lunch Plans", "Sandwiches at noon"); + seed_message(&db, &email_2, &placement_2).await; let event_bus = Arc::new(EventBus::new()); let mut events = event_bus.subscribe_events(); @@ -5163,12 +5698,17 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock_backend.add_message(sample_email( + let (injected_email, injected_placement) = sample_placed( "msg-sync-injected-1", "inbox", "Injected Sync Subject", "Injected sync body", - )); + ); + mock_backend.add_message(nuncio_mail::PlacedMessage { + email: injected_email, + source: nuncio_core::model::IdentitySource::Surrogate, + placement: injected_placement, + }); let overrides = MailEngineOverrides { mail_backend: Some(Arc::new(mock_backend)), @@ -5529,6 +6069,42 @@ mod tests { /// not just re-reading it back over `ListRules`); `ListRules` reflects /// the persisted rule; `DeleteRule` removes it AND reloads the engine /// back to empty. + /// Evaluate `engine` against a fixed sample message in a fixed INBOX + /// occupancy. + /// + /// Evaluation needs both halves now, because rules can ask where a message + /// sits; these tests only care whether the live rule set matched at all, so + /// the occupancy is a constant. + fn evaluate_sample_subject( + engine: &FilterEngine, + subject: &str, + ) -> Vec<(nuncio_filter::FilterRule, Vec)> { + let email = nuncio_core::model::Email { + id: "msg-filters-1".to_string(), + account_id: "acct-1".to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("body".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }; + let placement = nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: "1".to_string(), + read: false, + }; + engine.evaluate(nuncio_filter::PlacedEmail { + email: &email, + placement: &placement, + }) + } + #[tokio::test] async fn create_list_and_delete_rule_round_trip_and_keep_the_live_engine_in_sync() { let (addr, _handle, _db, filter_engine, _dir) = @@ -5538,26 +6114,8 @@ mod tests { .await .expect("client connects"); - let sample_email = |subject: &str| nuncio_core::model::Email { - id: "msg-filters-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: subject.to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some("body".to_string()), - body_html: None, - attachments: Vec::new(), - }; - // Before creating anything, the live engine matches nothing. - assert!(filter_engine - .evaluate(&sample_email("Urgent Meeting")) - .is_empty()); + assert!(evaluate_sample_subject(&filter_engine, "Urgent Meeting").is_empty()); let create_response = client .create_rule(authed_bearer_request(CreateRuleRequest { @@ -5579,7 +6137,7 @@ mod tests { // The live `FilterEngine`'s `ArcSwap` rule set was reloaded -- // proven by evaluating the SAME instance the server holds, not a // fresh one. - let matches = filter_engine.evaluate(&sample_email("Urgent Meeting")); + let matches = evaluate_sample_subject(&filter_engine, "Urgent Meeting"); assert_eq!(matches.len(), 1); assert_eq!(matches[0].0.id, rule_id); @@ -5612,9 +6170,7 @@ mod tests { .expect("list_rules succeeds") .into_inner(); assert!(list_after_delete.rules.is_empty()); - assert!(filter_engine - .evaluate(&sample_email("Urgent Meeting")) - .is_empty()); + assert!(evaluate_sample_subject(&filter_engine, "Urgent Meeting").is_empty()); } #[tokio::test] @@ -5648,23 +6204,7 @@ mod tests { list_response.rules.is_empty(), "an invalid rule must never be persisted" ); - assert!(filter_engine - .evaluate(&nuncio_core::model::Email { - id: "msg-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "anything".to_string(), - sender: "a@b.com".to_string(), - recipient: "c@d.com".to_string(), - received_at: 0, - read: false, - body_plain: None, - body_html: None, - attachments: Vec::new(), - }) - .is_empty()); + assert!(evaluate_sample_subject(&filter_engine, "anything").is_empty()); } /// Two `CreateRule` calls with the SAME name AND the SAME NSQL text @@ -5799,21 +6339,29 @@ mod tests { async fn preview_rule_dry_runs_against_a_stored_message_without_persisting() { let (addr, _handle, db, _engine, _dir) = spawn_filters_test_server("correct-token").await; - db.save_email(&nuncio_core::model::Email { - id: "msg-preview-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "preview-1".to_string(), - uid_validity: "1".to_string(), - subject: "Urgent Board Meeting".to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some("Please review the attached deck".to_string()), - body_html: None, - attachments: Vec::new(), - }) + db.save_email_at( + &nuncio_core::model::Email { + id: "msg-preview-1".to_string(), + account_id: "acct-1".to_string(), + subject: "Urgent Board Meeting".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("Please review the attached deck".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::IdentitySource::Surrogate, + &nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: "preview-1".to_string(), + read: false, + }, + ) .await .expect("seed message"); @@ -5863,6 +6411,73 @@ mod tests { ); } + /// A preview of a STORED message must never invent the occupancy it + /// evaluates against. + /// + /// `WHERE FOLDER = …` is answered from the placement, so substituting a + /// synthetic `inbox` one when the real placements cannot be read would make + /// the preview report a match against a folder the message is not in -- + /// stated with full confidence, about real stored mail. Preview's whole job + /// is to say truthfully what a rule would do, so an unreadable placement + /// has to surface as an error. + /// + /// Dropping the `placements` table makes `placements_of` fail for real + /// while `get_message` still succeeds, which is exactly the split that + /// produced the fabrication. + #[tokio::test] + async fn preview_rule_surfaces_a_placement_read_error_instead_of_inventing_a_folder() { + let (addr, _handle, db, _engine, _dir) = spawn_filters_test_server("correct-token").await; + + db.save_email_at( + &nuncio_core::model::Email { + id: "msg-preview-archive".to_string(), + account_id: "acct-1".to_string(), + subject: "Urgent Board Meeting".to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some("Please review the attached deck".to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::IdentitySource::Surrogate, + &nuncio_core::model::Placement { + account_id: "acct-1".to_string(), + folder_id: "archive".to_string(), + uid_validity: "1".to_string(), + remote_id: "preview-archive".to_string(), + read: false, + }, + ) + .await + .expect("seed message"); + + sqlx::query("DROP TABLE placements") + .execute(db.pool()) + .await + .expect("drop the placements table"); + + let mut client = FiltersClient::connect(format!("http://{addr}")) + .await + .expect("client connects"); + + let err = client + .preview_rule(authed_bearer_request(PreviewRuleRequest { + nsql: "WHERE folder = 'inbox' ACTION DELETE".to_string(), + message_id: Some("msg-preview-archive".to_string()), + })) + .await + .expect_err("an unreadable placement must not be papered over"); + assert_eq!(err.code(), Code::Internal); + assert!( + err.message().contains("placements"), + "the error must name what could not be read: {}", + err.message() + ); + } + /// When no `message_id` is given (or it does not resolve to a stored /// message), `PreviewRule` falls back to a fixed synthetic sample /// message rather than failing the RPC. @@ -5973,21 +6588,7 @@ mod tests { assert_eq!(list_response.rules[0].name, "Renamed"); // The live `FilterEngine` was reloaded with the updated rule. - let matches = filter_engine.evaluate(&nuncio_core::model::Email { - id: "msg-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "Urgent Meeting".to_string(), - sender: "a@b.com".to_string(), - recipient: "c@d.com".to_string(), - received_at: 0, - read: false, - body_plain: None, - body_html: None, - attachments: Vec::new(), - }); + let matches = evaluate_sample_subject(&filter_engine, "Urgent Meeting"); assert_eq!(matches.len(), 1); assert_eq!(matches[0].0.id, rule_id); } @@ -6154,21 +6755,7 @@ mod tests { assert_eq!(list_response.rules[0].nsql_text, SAMPLE_RULE_NSQL); // The live engine was reloaded to include the imported rule. - let matches = filter_engine.evaluate(&nuncio_core::model::Email { - id: "msg-1".to_string(), - account_id: "acct-1".to_string(), - folder_id: "inbox".to_string(), - remote_id: "1".to_string(), - uid_validity: "1".to_string(), - subject: "Urgent Meeting".to_string(), - sender: "a@b.com".to_string(), - recipient: "c@d.com".to_string(), - received_at: 0, - read: false, - body_plain: None, - body_html: None, - attachments: Vec::new(), - }); + let matches = evaluate_sample_subject(&filter_engine, "Urgent Meeting"); assert_eq!(matches.len(), 1); } @@ -6274,22 +6861,20 @@ mod tests { let (db, _dir) = DatabaseEngine::connect_ephemeral() .await .expect("connect ephemeral test db"); - db.save_email(&sample_email( + let (export_1, export_1_at) = sample_placed( "msg-export-1", "inbox", "Export Subject One", "Export body one", - )) - .await - .expect("seed message 1"); - db.save_email(&sample_email( + ); + seed_message(&db, &export_1, &export_1_at).await; + let (export_2, export_2_at) = sample_placed( "msg-export-2", "archive", "Export Subject Two", "Export body two", - )) - .await - .expect("seed message 2"); + ); + seed_message(&db, &export_2, &export_2_at).await; let event_bus = Arc::new(EventBus::new()); let secrets = Arc::new(SecretManager::mock()); @@ -6339,12 +6924,16 @@ mod tests { let (db, _dir) = DatabaseEngine::connect_ephemeral() .await .expect("connect ephemeral test db"); - let mut msg_a = sample_email("msg-acct-a", "inbox", "From Account A", "Body A"); + let (mut msg_a, mut at_a) = + sample_placed("msg-acct-a", "inbox", "From Account A", "Body A"); msg_a.account_id = "acct-a".to_string(); - db.save_email(&msg_a).await.expect("seed account a message"); - let mut msg_b = sample_email("msg-acct-b", "inbox", "From Account B", "Body B"); + at_a.account_id = "acct-a".to_string(); + seed_message(&db, &msg_a, &at_a).await; + let (mut msg_b, mut at_b) = + sample_placed("msg-acct-b", "inbox", "From Account B", "Body B"); msg_b.account_id = "acct-b".to_string(); - db.save_email(&msg_b).await.expect("seed account b message"); + at_b.account_id = "acct-b".to_string(); + seed_message(&db, &msg_b, &at_b).await; let event_bus = Arc::new(EventBus::new()); let secrets = Arc::new(SecretManager::mock()); diff --git a/crates/nunciod/src/outbox.rs b/crates/nunciod/src/outbox.rs index 01a5b7e..05ed581 100644 --- a/crates/nunciod/src/outbox.rs +++ b/crates/nunciod/src/outbox.rs @@ -289,7 +289,7 @@ async fn execute_one( match mutation.mutation_type.as_str() { "MOVE" | "COPY" | "FLAG" | "UNFLAG" | "DELETE" => { - dispatch_mailbox_mutation(env, mutation, &email, &payload).await + dispatch_mailbox_mutation(db, env, mutation, &email, &payload).await } "FORWARD" => dispatch_forward(db, env, &email, &payload).await, "WEBHOOK" => dispatch_webhook(env, mutation, &email, &payload).await, @@ -298,10 +298,30 @@ async fn execute_one( } /// Execute a mailbox-affecting mutation (move/copy/flag/unflag/delete) against -/// the account's mail backend, recovering the remote addressing (protocol id + -/// UIDVALIDITY scope) from the stored message row so the backend can enforce -/// its UIDVALIDITY guard. +/// the account's mail backend, recovering the remote addressing (folder, +/// protocol id, UIDVALIDITY scope) from the message's stored placement so the +/// backend can enforce its UIDVALIDITY guard. +/// +/// The addressing comes from a placement rather than the message row, because +/// those coordinates are properties of a mailbox occupancy and a message can +/// hold several. The outbox row records only the message key, so which occupancy +/// the rule matched is not recoverable here -- and when the message occupies +/// more than one mailbox, this **refuses** rather than picking one. +/// +/// Guessing is not the conservative option. A rule that matched the `INBOX` +/// copy would `MOVE` or `DELETE` whichever copy happened to sort first, against +/// mail the user never targeted and with no undo; Gmail labels make +/// multi-placement ordinary rather than exotic. Unresolvable *addressing* is no +/// more a licence to act than an unverifiable claim is in +/// [`crate::sync::apply_filter_actions`]: refusing is inert and visible, +/// guessing is destructive and irreversible. +/// +/// The refusal is permanent, not retryable: nothing about a later attempt makes +/// the outbox row remember a placement it never recorded. Carrying the +/// originating placement on the row is the real fix, and belongs with the +/// placement-aware wire surface. async fn dispatch_mailbox_mutation( + db: &DatabaseEngine, env: &dyn RemoteExecutionEnv, mutation: &PendingRemoteMutation, email: &nuncio_core::model::Email, @@ -330,20 +350,56 @@ async fn dispatch_mailbox_mutation( other => return Disposition::Permanent(format!("unexpected mailbox mutation '{other}'")), }; - let backend = match env.mail_backend(&email.account_id).await { + // Recover the remote addressing from a stored placement rather than + // parsing it out of the opaque message key. The backend still + // SELECT-verifies the UIDVALIDITY before mutating. + let placements = match db.placements_of(&email.id).await { + Ok(placements) => placements, + Err(e) => { + return Disposition::Retry(format!("failed to resolve the message's placements: {e}")) + } + }; + let placement = match placements.as_slice() { + [only] => only, + [] => { + // No occupancy means no way to address the message on any server, + // and no retry can conjure one back. + return Disposition::Permanent(format!( + "message '{}' occupies no mailbox to address", + email.id + )); + } + several => { + let folders: Vec<&str> = several.iter().map(|p| p.folder_id.as_str()).collect(); + tracing::warn!( + mutation_id = %mutation.id, + op = %mutation.mutation_type, + message_id = %email.id, + placement_count = several.len(), + "outbox: refusing an ambiguously-addressed mutation" + ); + return Disposition::Permanent(format!( + "message '{}' occupies {} mailboxes ({}) and the outbox row does not record \ + which one rule '{}' matched; refusing to guess a {} target", + email.id, + several.len(), + folders.join(", "), + mutation.rule_id, + mutation.mutation_type + )); + } + }; + + let backend = match env.mail_backend(&placement.account_id).await { Ok(backend) => backend, Err(e) => return Disposition::Retry(format!("failed to build mail backend: {e}")), }; - // Recover the remote addressing from the stored message row -- the - // protocol-native id and the UIDVALIDITY scope it was captured under -- - // rather than parsing it out of the opaque surrogate id. The backend still - // SELECT-verifies the UIDVALIDITY before mutating. let spec = RemoteMutationSpec { message_id: email.id.clone(), - remote_id: email.remote_id.clone(), - folder_id: email.folder_id.clone(), - uid_validity: email.uid_validity.clone(), + remote_id: placement.remote_id.clone(), + folder_id: placement.folder_id.clone(), + uid_validity: placement.uid_validity.clone(), kind, }; @@ -580,7 +636,7 @@ mod tests { use super::*; use crate::lifecycle::{ShutdownController, ShutdownSignal}; use crate::test_tracing::with_recorder; - use nuncio_core::model::Email; + use nuncio_core::model::{Email, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::MutationPayload; use nuncio_mail::MockMailBackend; @@ -634,17 +690,27 @@ mod tests { Email { id: "msg-outbox-log".to_string(), account_id: "acct-outbox-log".to_string(), - folder_id: "INBOX".to_string(), - remote_id: "77".to_string(), - uid_validity: "9".to_string(), subject: "Confidential outbox subject".to_string(), sender: "alice@example.com".to_string(), recipient: "owner@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some("secret outbox body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, + } + } + + /// The one mailbox occupancy `sample_email` is stored in -- the addressing + /// the outbox recovers to build its `RemoteMutationSpec`. + fn sample_placement() -> Placement { + Placement { + account_id: "acct-outbox-log".to_string(), + folder_id: "INBOX".to_string(), + uid_validity: "9".to_string(), + remote_id: "77".to_string(), + read: false, } } @@ -683,7 +749,13 @@ mod tests { db.save_folder_sync_state("acct-outbox-log", "INBOX", "9:100") .await .expect("save folder checkpoint"); - db.save_email(&sample_email()).await.expect("save email"); + db.save_email_at( + &sample_email(), + IdentitySource::Surrogate, + &sample_placement(), + ) + .await + .expect("save email"); db.save_pending_mutation(&flag_mutation()) .await .expect("save pending mutation"); @@ -727,4 +799,105 @@ mod tests { ); } } + + /// A second occupancy of `sample_email`, in a different mailbox -- an + /// ordinary Gmail label, or a `COPY` another client made. + fn second_placement() -> Placement { + Placement { + account_id: "acct-outbox-log".to_string(), + folder_id: "Archive".to_string(), + uid_validity: "9".to_string(), + remote_id: "78".to_string(), + read: false, + } + } + + /// Seed the message into every given occupancy and dispatch one FLAG + /// mutation against it. Returns the honest disposition and every spec the + /// backend was actually asked to apply. + /// + /// Calls [`dispatch_mailbox_mutation`] rather than draining through + /// [`execute_pending_mutations`] deliberately. The drain's own + /// "attempting"/"completed" `tracing` callsites are what + /// `drain_logs_attempt_and_completion_without_leaking_message_content` + /// asserts on, and `tracing` caches callsite interest globally: a + /// concurrent test hitting those callsites with no recorder installed + /// caches them as uninteresting and makes that test fail. Addressing is + /// decided here anyway, so the narrower call is also the tighter assertion. + async fn dispatch_with_placements( + placements: &[Placement], + ) -> (Disposition, Vec) { + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + for placement in placements { + db.save_email_at(&sample_email(), IdentitySource::Surrogate, placement) + .await + .expect("store the occupancy"); + } + + let backend = MockMailBackend::new(); + let env = SucceedingEnv { + backend: backend.clone(), + }; + let mutation = flag_mutation(); + let payload = + serde_json::from_str::(&mutation.payload).expect("parse payload"); + let disposition = + dispatch_mailbox_mutation(&db, &env, &mutation, &sample_email(), &payload).await; + (disposition, backend.applied_mutations()) + } + + /// The unambiguous case still works: exactly one occupancy, so the row's + /// target is fully determined and the mutation is addressed from it. + #[tokio::test] + async fn a_single_placement_mutation_is_addressed_from_that_placement() { + let (disposition, applied) = dispatch_with_placements(&[sample_placement()]).await; + + assert!( + matches!(disposition, Disposition::Completed), + "a fully-addressed mutation must go through" + ); + assert_eq!(applied.len(), 1, "the backend really was asked to act"); + assert_eq!(applied[0].folder_id, "INBOX"); + assert_eq!(applied[0].remote_id, "77"); + assert_eq!(applied[0].uid_validity, "9"); + assert_eq!(applied[0].message_id, "msg-outbox-log"); + } + + /// The ambiguous case refuses. The outbox row names only a message, so for + /// a message in two mailboxes there is no way to know which copy the rule + /// matched -- and a guessed MOVE or DELETE lands on mail the user never + /// targeted, with no undo. + #[tokio::test] + async fn a_multi_placement_mutation_is_refused_rather_than_guessed() { + let (disposition, applied) = + dispatch_with_placements(&[sample_placement(), second_placement()]).await; + + assert!( + applied.is_empty(), + "an ambiguously-addressed mutation must never reach the server" + ); + match disposition { + // Permanent, not Retry: no later attempt makes the outbox row + // remember a placement it never recorded. + Disposition::Permanent(reason) => { + assert!( + reason.contains("INBOX") && reason.contains("Archive"), + "the refusal must name the competing mailboxes: {reason}" + ); + assert!( + reason.contains("refusing"), + "and say plainly that it refused: {reason}" + ); + } + Disposition::Completed => panic!("expected a permanent refusal, got Completed"), + Disposition::Conflict(observed) => { + panic!("expected a permanent refusal, got Conflict({observed})") + } + Disposition::Retry(reason) => { + panic!("expected a permanent refusal, got Retry({reason})") + } + } + } } diff --git a/crates/nunciod/src/sync.rs b/crates/nunciod/src/sync.rs index aca69d5..13b1309 100644 --- a/crates/nunciod/src/sync.rs +++ b/crates/nunciod/src/sync.rs @@ -7,7 +7,8 @@ //! account it connects a real [`MailBackend`] (a real [`ImapEngine`] for an //! IMAP/SMTP transport, built from [`AccountConfig`] plus the //! account's keyring-stored password), fetches folders and messages, and -//! persists every message through [`DatabaseEngine::save_email`]. +//! persists every message, together with the mailbox occupancy it was found +//! in, through [`DatabaseEngine::save_email_at`]. //! //! # Testability //! @@ -20,6 +21,7 @@ //! all-accounts sync fans these per-account runs out through //! [`crate::sync_dispatcher::sync_all_configured`], so concurrency and the //! per-account timeout are bounded in one place rather than serialized here. +use nuncio_core::model::PlacementKey; use nuncio_core::{AccountConfig, CoreCommand, CoreEvent, EventBus, Transport}; use nuncio_filter::{FilterEngine, OutboxManager, RuleAction}; use nuncio_mail::{ImapEngine, JmapEngine, MailBackend, MailError}; @@ -68,24 +70,39 @@ fn remote_action_tag(action: &RuleAction) -> Option<(&'static str, Option Option<(&'static str, Option usize { let mut applied = 0usize; + let placed = nuncio_filter::PlacedEmail { email, placement }; - for (rule, actions) in filter_engine.evaluate(email) { + for (rule, actions) in filter_engine.evaluate(placed) { + // Claim before acting, never after. The claim is keyed on message + // identity, so the same mail reaching a second mailbox -- an ordinary + // MOVE, or a Gmail label -- evaluates again but acts only once. + match db.claim_filter_fire(&rule.id, &email.id).await { + Ok(true) => {} + Ok(false) => { + tracing::debug!( + rule_id = %rule.id, + message_key = %email.id, + folder_id = %placement.folder_id, + "rule already fired for this message; skipping actions for this placement" + ); + continue; + } + Err(e) => { + // Fail closed. An unverifiable claim must not become a licence + // to act: a lost claim check on a FORWARD is a duplicate mail + // the recipient cannot un-receive. + tracing::error!( + rule_id = %rule.id, + message_key = %email.id, + error = %e, + "could not claim the filter fire; skipping actions" + ); + continue; + } + } + + let placement_key = placement.key(); for action in actions { let immediate_ok = match &action { - RuleAction::MarkRead => match db.set_message_read(&email.id, true).await { - Ok(()) => true, - Err(e) => { - tracing::warn!( - "filter rule '{}' MARK READ failed for message '{}': {e}", - rule.id, - email.id - ); - false + // One write with a boolean rather than two near-identical + // arms. Both name the occupancy that was evaluated, because + // `\Seen` is per-mailbox: a message read in INBOX is not + // thereby read in Archive. + RuleAction::MarkRead | RuleAction::MarkUnread => { + let read = matches!(action, RuleAction::MarkRead); + let label = if read { "MARK READ" } else { "MARK UNREAD" }; + match db.set_placement_read(&placement_key, read).await { + Ok(()) => true, + Err(e) => { + tracing::warn!( + "filter rule '{}' {label} failed for message '{}' in folder '{}': {e}", + rule.id, + email.id, + placement.folder_id + ); + false + } } - }, - RuleAction::MarkUnread => match db.set_message_read(&email.id, false).await { - Ok(()) => true, - Err(e) => { - tracing::warn!( - "filter rule '{}' MARK UNREAD failed for message '{}': {e}", - rule.id, - email.id - ); - false - } - }, + } RuleAction::MoveTo(_) | RuleAction::CopyTo(_) | RuleAction::Flag @@ -197,31 +243,42 @@ pub async fn apply_filter_actions( applied } -/// Fetch every folder and every message in every folder from `backend`, -/// persisting each message via [`DatabaseEngine::save_email`] and, for -/// messages genuinely new to the store, evaluating it against -/// `filter_engine` via [`apply_filter_actions`]. +/// Fetch every folder from `backend`, persist each reported occupancy via +/// [`DatabaseEngine::save_email_at`], evaluate the genuinely-new occupancies +/// against `filter_engine` through [`apply_filter_actions`], and delete the +/// occupancies the server no longer reports. /// -/// Sync is not yet incremental (a re-sync re-fetches every message the -/// backend still reports, not just genuinely new ones), so new-vs-seen must -/// be determined BEFORE `save_email` persists (or re-persists) anything -- -/// `save_email` is `INSERT OR REPLACE` and would otherwise erase the -/// distinction between "arriving for the first time" and "seen again". Doing -/// that with one existence lookup per message (`DatabaseEngine::get_message`) -/// is an N+1 query pattern over a fetched chunk; instead, this classifies an -/// entire folder's chunk with a single batched -/// `DatabaseEngine::existing_message_ids` lookup up front, then folds in a -/// per-batch "already seen this pass" set while iterating so a chunk that -/// itself contains a repeated id (e.g. a backend surfacing the same message -/// twice in one fetch) still fires at most once -- exactly as if each -/// message's existence had been checked one at a time immediately before its -/// own `save_email`. +/// # Why classification is per placement, not per message /// -/// Returns the total number of messages processed (a message id "processed" -/// more than once, e.g. by a backend that returns the same id from multiple -/// folders, is counted once per occurrence -- `save_email` itself is -/// `INSERT OR REPLACE`, so storage stays deduplicated by message id -/// regardless). +/// Message identity is folder-independent: a message already stored from +/// `INBOX` has the same key when it turns up in `Archive`. Asking "have I seen +/// this *message*?" therefore answers `yes` for a mailbox this daemon has never +/// filtered, and the arrival silently fires nothing -- no error, no log. The +/// question that matches the intent is "have I seen this *occupancy*?", so this +/// classifies on [`PlacementKey`] via +/// [`DatabaseEngine::existing_placements`]. +/// +/// Classification happens BEFORE the write, because `save_email_at` upserts and +/// would otherwise erase the distinction between a first arrival and a +/// re-report. One batched lookup covers the whole chunk rather than an N+1 of +/// per-row existence checks, and a per-pass "already seen" set folds in on top +/// so a backend that surfaces the same occupancy twice in one fetch still +/// classifies it as new exactly once. +/// +/// Firing once per *arrival* is not the same as acting once per *message*; the +/// second guarantee lives in the claim inside [`apply_filter_actions`]. +/// +/// # Absence +/// +/// [`nuncio_mail::FolderChanges::present`] is honoured as written: `None` means +/// the pass was incremental and cannot speak to absence, and treating it as +/// "nothing is present" would delete the whole folder. Only `Some(list)` -- +/// which `nuncio-mail` withholds when a SELECT omitted UIDVALIDITY, since the +/// keys it could build would name no stored row -- licenses deleting the stored +/// placements missing from it. +/// +/// Returns the total number of occupancies processed: one message reported from +/// two folders counts twice, because two mailboxes really did report it. /// /// Contains no event-bus or credential logic -- pure fetch-and-persist, so /// it is the smallest unit tests can exercise directly with a @@ -248,25 +305,29 @@ async fn fetch_and_persist( let changes = backend .sync_changes(&folder.id, last_state.as_deref()) .await?; - let emails = changes.upserts; - let fetched = emails.len(); + let placed = changes.upserts; + let fetched = placed.len(); let new_state = changes.next_state; // One round trip classifies the whole chunk instead of one lookup per - // message. A lookup failure that is not "not found" would be a - // genuine DB error; `existing_message_ids` surfaces those as `Err` - // rather than silently treating the batch as "not new", so callers - // never persist and skip-filter on an unverified assumption. - let chunk_ids: Vec = emails.iter().map(|e| e.id.clone()).collect(); - let already_present = db.existing_message_ids(&chunk_ids).await?; + // occupancy. This asks about *placements*, not messages: identity is + // folder-independent, so a message already stored from another folder + // is still a first arrival here, and testing the message key instead + // would classify it as seen and skip filtering it entirely. A lookup + // failure that is not "not found" surfaces as `Err` rather than + // silently reading as "not new", so nothing is persisted and + // skip-filtered on an unverified assumption. + let chunk_keys: Vec = placed.iter().map(|p| p.placement.key()).collect(); + let already_present = db.existing_placements(&chunk_keys).await?; let mut seen_this_pass = std::collections::HashSet::new(); - for email in emails { - let is_new = - !already_present.contains(&email.id) && seen_this_pass.insert(email.id.clone()); - db.save_email(&email).await?; + for message in placed { + let key = message.placement.key(); + let is_new = !already_present.contains(&key) && seen_this_pass.insert(key); + db.save_email_at(&message.email, message.source, &message.placement) + .await?; if is_new { - apply_filter_actions(db, filter_engine, &email).await; + apply_filter_actions(db, filter_engine, &message.email, &message.placement).await; } synced += 1; } @@ -274,31 +335,39 @@ async fn fetch_and_persist( // could only ever add: a message moved or deleted by any other client // -- another daemon, a phone, webmail -- stayed in the local store // forever, and every account accumulated ghosts. - if let Some(acct) = account_id { - let mut gone: Vec = changes.removals; + // + // What a folder stops mentioning is an occupancy, never a message: the + // same mail may still sit in another mailbox, and it goes only when its + // last placement does -- which `delete_placements` decides, not this. + // + // A `PlacementKey` names its own account, so an explicitly-reported + // removal is actionable with no ambient account context at all. Only + // the `present` diff needs one, because its other half is a lookup + // keyed by (account, folder). + let mut gone: Vec = changes.removals; - // `present` is the folder's complete contents when the pass was - // able to enumerate them. `None` and `Some(vec![])` are NOT the - // same: `None` means the pass was incremental and cannot speak to - // absence, and treating it as "nothing is present" would delete - // the folder. - if let Some(present) = changes.present { - let present: std::collections::HashSet = present.into_iter().collect(); - let stored = db.message_ids_in_folder(acct, &folder.id).await?; - gone.extend(stored.into_iter().filter(|id| !present.contains(id))); - } + // `present` is the folder's complete contents when the pass was + // able to enumerate them. `None` and `Some(vec![])` are NOT the + // same: `None` means the pass was incremental and cannot speak to + // absence, and treating it as "nothing is present" would delete + // the folder. + if let (Some(acct), Some(present)) = (account_id, changes.present) { + let present: std::collections::HashSet = present.into_iter().collect(); + let stored = db.placements_in_folder(acct, &folder.id).await?; + gone.extend(stored.into_iter().filter(|key| !present.contains(key))); + } - gone.sort_unstable(); - gone.dedup(); - if !gone.is_empty() { - let deleted = db.delete_messages(&gone).await?; - tracing::info!( - account_id = %acct, - folder_id = %folder.id, - deleted, - "removed messages that are no longer on the server" - ); - } + gone.sort_unstable(); + gone.dedup(); + if !gone.is_empty() { + let outcome = db.delete_placements(&gone).await?; + tracing::info!( + account_id = ?account_id, + folder_id = %folder.id, + placements_removed = outcome.placements_removed, + messages_reaped = outcome.messages_reaped, + "removed placements that are no longer on the server" + ); } // Persist the returned checkpoint only AFTER this folder's messages @@ -446,9 +515,9 @@ pub async fn run_account_sync( #[cfg(test)] mod tests { use super::*; - use nuncio_core::model::{Email, Folder}; + use nuncio_core::model::{Email, Folder, IdentitySource, Placement}; use nuncio_core::{CoreEvent, EngineStatus, TlsMode}; - use nuncio_mail::MockMailBackend; + use nuncio_mail::{MockMailBackend, PlacedMessage}; use nuncio_store::db::DatabaseEngine; use serde_json::json; use wiremock::matchers::{body_partial_json, method, path}; @@ -568,24 +637,58 @@ mod tests { } } - fn mock_email(id: &str, folder_id: &str, subject: &str) -> Email { - Email { - id: id.to_string(), - account_id: "acct-mock-1".to_string(), - folder_id: folder_id.to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), - subject: subject.to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: "bob@nuncio.mx".to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some(format!("body for {id}")), - body_html: None, - attachments: Vec::new(), + /// One mock message as a backend would surface it: identity plus the + /// occupancy it was found in. `id` doubles as the folder-scoped UID, which + /// keeps distinct ids in one folder distinct occupancies. + fn mock_placed(id: &str, folder_id: &str, subject: &str) -> PlacedMessage { + PlacedMessage { + email: Email { + id: id.to_string(), + account_id: "acct-mock-1".to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: "bob@nuncio.mx".to_string(), + received_at: 1_700_000_000, + body_plain: Some(format!("body for {id}")), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + source: IdentitySource::Surrogate, + placement: Placement { + account_id: "acct-mock-1".to_string(), + folder_id: folder_id.to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + }, } } + /// The read flag of a message's one occupancy, for the single-placement + /// mock fixtures. Fails loudly rather than defaulting if the message turns + /// out to sit in no mailbox at all -- "unread" and "not there" are + /// different answers and must not be conflated by a test helper. + async fn read_flag_of(db: &DatabaseEngine, message_key: &str) -> bool { + let placements = db + .placements_of(message_key) + .await + .expect("read the message's placements"); + assert_eq!( + placements.len(), + 1, + "fixture expects exactly one occupancy of '{message_key}'" + ); + placements[0].read + } + + /// The message key every [`MockMailBackend::with_same_message_in`] + /// occupancy carries. + fn shared_key() -> &'static str { + MockMailBackend::SHARED_MESSAGE_KEY + } + #[tokio::test] async fn sync_with_backend_persists_mock_messages_and_emits_events() { let (db, _dir) = DatabaseEngine::connect_ephemeral() @@ -601,8 +704,8 @@ mod tests { total_messages: 2, unread_messages: 2, }); - mock.add_message(mock_email("m1", "inbox", "Hello")); - mock.add_message(mock_email("m2", "inbox", "World")); + mock.add_message(mock_placed("m1", "inbox", "Hello")); + mock.add_message(mock_placed("m2", "inbox", "World")); let filter_engine = empty_filter_engine(); let synced = sync_with_backend( @@ -617,11 +720,14 @@ mod tests { assert_eq!(synced, 2); let persisted = db - .list_messages("inbox", 10) + .list_messages("acct-mock-1", "inbox", 10) .await .expect("list persisted messages"); assert_eq!(persisted.len(), 2); - let subjects: Vec = persisted.iter().map(|e| e.subject.clone()).collect(); + let subjects: Vec = persisted + .iter() + .map(|(email, _placement)| email.subject.clone()) + .collect(); assert!(subjects.contains(&"Hello".to_string())); assert!(subjects.contains(&"World".to_string())); @@ -815,12 +921,13 @@ mod tests { // folder, and its protocol-native JMAP object id round-trips in // `remote_id`. let persisted = db - .list_messages("inbox", 10) + .list_messages("acct-jmap-real-1", "inbox", 10) .await .expect("jmap message persisted"); assert_eq!(persisted.len(), 1); - assert_eq!(persisted[0].subject, "Welcome to JMAP Sync"); - assert_eq!(persisted[0].remote_id, "jmap-msg-1"); + let (email, placement) = &persisted[0]; + assert_eq!(email.subject, "Welcome to JMAP Sync"); + assert_eq!(placement.remote_id, "jmap-msg-1"); assert_eq!( events.recv().await.expect("start event"), @@ -927,17 +1034,16 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-urgent", "inbox", "Urgent: server down")); + mock.add_message(mock_placed("m-urgent", "inbox", "Urgent: server down")); let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) .await .expect("sync succeeds"); assert_eq!(synced, 1); - let stored = db.get_message("m-urgent").await.expect("message persisted"); assert!( - stored.read, - "MARK READ action must genuinely flip the stored read flag" + read_flag_of(&db, "m-urgent").await, + "MARK READ action must genuinely flip the occupancy's read flag" ); let logs = db @@ -979,7 +1085,7 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-archive", "inbox", "Please Archive Me")); + mock.add_message(mock_placed("m-archive", "inbox", "Please Archive Me")); let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) .await @@ -1004,11 +1110,7 @@ mod tests { assert_eq!(logs[0].message_id, "m-archive"); // The remote action must not have applied a local read-flag change. - let stored = db - .get_message("m-archive") - .await - .expect("message persisted"); - assert!(!stored.read); + assert!(!read_flag_of(&db, "m-archive").await); } #[tokio::test] @@ -1033,15 +1135,17 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-plain", "inbox", "Just a normal update")); + mock.add_message(mock_placed("m-plain", "inbox", "Just a normal update")); let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) .await .expect("sync succeeds"); assert_eq!(synced, 1); - let stored = db.get_message("m-plain").await.expect("message persisted"); - assert!(!stored.read, "non-matching message must stay untouched"); + assert!( + !read_flag_of(&db, "m-plain").await, + "non-matching message must stay untouched" + ); let logs = db .list_filter_execution_logs(10) @@ -1092,7 +1196,9 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-repeat", "inbox", "Urgent: server down")); + let repeat = mock_placed("m-repeat", "inbox", "Urgent: server down"); + let repeat_key = repeat.placement.key(); + mock.add_message(repeat); // First sync: the message is genuinely new, so both rules must fire. let synced_first = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) @@ -1100,11 +1206,7 @@ mod tests { .expect("first sync succeeds"); assert_eq!(synced_first, 1); - let stored_after_first = db - .get_message("m-repeat") - .await - .expect("message persisted after first sync"); - assert!(stored_after_first.read); + assert!(read_flag_of(&db, "m-repeat").await); let logs_after_first = db .list_filter_execution_logs(10) @@ -1121,7 +1223,7 @@ mod tests { // Manually flip the message back to unread, exactly as a user // reading and then un-reading it would -- proves the second sync // doesn't re-flip it back to read via a repeat MARK READ fire. - db.set_message_read("m-repeat", false) + db.set_placement_read(&repeat_key, false) .await .expect("manually mark unread"); @@ -1132,12 +1234,8 @@ mod tests { .expect("second sync succeeds"); assert_eq!(synced_second, 1); - let stored_after_second = db - .get_message("m-repeat") - .await - .expect("message persisted after second sync"); assert!( - !stored_after_second.read, + !read_flag_of(&db, "m-repeat").await, "a re-sync of an already-seen message must not re-fire MARK READ" ); @@ -1164,10 +1262,10 @@ mod tests { #[tokio::test] async fn sync_with_backend_fires_filters_only_for_new_messages_in_a_mixed_batch() { - // A single fetched chunk containing a mix of an already-seen message - // (persisted by a prior sync) and genuinely new ones must fire filter - // actions ONLY for the new ones, via the batched - // `existing_message_ids` classification -- not a per-message lookup. + // A single fetched chunk containing a mix of an already-stored + // occupancy (persisted by a prior sync) and genuinely new ones must + // fire filter actions ONLY for the new ones, via the batched + // `existing_placements` classification -- not a per-row lookup. let (db, _dir) = DatabaseEngine::connect_ephemeral() .await .expect("ephemeral db"); @@ -1181,16 +1279,13 @@ mod tests { .expect("parse rule"); let filter_engine = FilterEngine::new(vec![mark_read_rule]).expect("compile rule"); - // Pre-seed one message directly, simulating it having landed in a + // Pre-seed one occupancy directly, simulating it having landed in a // prior sync pass. It must NOT re-fire even though this pass's // backend still reports it (sync is not yet incremental). - db.save_email(&mock_email( - "m-already-seen", - "inbox", - "Urgent: known issue", - )) - .await - .expect("pre-seed existing message"); + let seeded = mock_placed("m-already-seen", "inbox", "Urgent: known issue"); + db.save_email_at(&seeded.email, seeded.source, &seeded.placement) + .await + .expect("pre-seed existing occupancy"); let mock = MockMailBackend::new(); mock.add_folder(Folder { @@ -1200,9 +1295,13 @@ mod tests { unread_messages: 3, }); // Mixed chunk: one already-present id, two genuinely new ids. - mock.add_message(mock_email("m-already-seen", "inbox", "Urgent: known issue")); - mock.add_message(mock_email("m-new-1", "inbox", "Urgent: brand new")); - mock.add_message(mock_email("m-new-2", "inbox", "Urgent: also new")); + mock.add_message(mock_placed( + "m-already-seen", + "inbox", + "Urgent: known issue", + )); + mock.add_message(mock_placed("m-new-1", "inbox", "Urgent: brand new")); + mock.add_message(mock_placed("m-new-2", "inbox", "Urgent: also new")); let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None) .await @@ -1224,16 +1323,12 @@ mod tests { assert!(fired_ids.contains("m-new-2")); assert!(!fired_ids.contains("m-already-seen")); - // The pre-seeded message must still be persisted (INSERT OR REPLACE - // semantics are unaffected by the batched new/seen classification) - // but its read flag must be untouched by this pass's MARK READ rule. - let already_seen = db - .get_message("m-already-seen") - .await - .expect("still persisted"); + // The pre-seeded occupancy must still be persisted (the upsert is + // unaffected by the batched new/seen classification) but its read flag + // must be untouched by this pass's MARK READ rule. assert!( - !already_seen.read, - "a message that already existed before this sync must not re-fire MARK READ" + !read_flag_of(&db, "m-already-seen").await, + "an occupancy that already existed before this sync must not re-fire MARK READ" ); } @@ -1256,7 +1351,7 @@ mod tests { total_messages: 1, unread_messages: 1, }); - mock.add_message(mock_email("m-inc-1", "inbox", "First")); + mock.add_message(mock_placed("m-inc-1", "inbox", "First")); let filter_engine = empty_filter_engine(); @@ -1301,4 +1396,224 @@ mod tests { vec![None, Some("uidnext-4242".to_string())], ); } + + /// A rule that matches the one message the shared-identity mock stages. + fn rule_matching_the_shared_message() -> nuncio_filter::FilterRule { + nuncio_filter::NsqlParser::parse_rule( + "Archive Shared", + 1, + "WHERE subject CONTAINS 'Shared' ACTION MOVE TO 'Archive'", + ) + .expect("parse rule") + } + + #[tokio::test] + async fn a_message_arriving_in_a_second_folder_still_gets_evaluated() { + // The regression this guards: with identity now folder-independent, a + // message already stored from INBOX has a known message_key, so a + // message-scoped "already present" test would classify its arrival in + // Archive as old and skip filtering it entirely -- silently, with no + // error and nothing in the log. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let event_bus = EventBus::new(); + + let rule = rule_matching_the_shared_message(); + let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule"); + + // Already stored, from INBOX. The message key is therefore known + // before this sync begins -- but nothing has ever filtered Archive. + let in_inbox = MockMailBackend::shared_message_in("INBOX", "1"); + db.save_email_at(&in_inbox.email, in_inbox.source, &in_inbox.placement) + .await + .expect("pre-seed the INBOX occupancy"); + + let backend = MockMailBackend::with_same_message_in(&["Archive"]); + let processed = fetch_and_persist( + &db, + &event_bus, + &backend, + &filter_engine, + Some(MockMailBackend::SHARED_ACCOUNT_ID), + ) + .await + .expect("sync succeeds"); + + assert_eq!(processed, 1, "the Archive occupancy is processed"); + let placements = db + .placements_of(shared_key()) + .await + .expect("read the message's placements"); + assert_eq!( + placements.len(), + 2, + "one message, now occupying both mailboxes" + ); + + assert!( + db.has_filter_fired(&rule.id, shared_key()) + .await + .expect("read the claim ledger"), + "an arrival in a second folder must be evaluated, not skipped as already-seen" + ); + let pending = db + .list_pending_mutations(10) + .await + .expect("list pending mutations"); + assert_eq!( + pending.len(), + 1, + "with a real outbox mutation behind the fire, not just a ledger row" + ); + assert_eq!(pending[0].message_id, shared_key()); + } + + #[tokio::test] + async fn a_rule_fires_once_for_a_message_that_lands_in_two_folders() { + // MOVE and FLAG converge under repetition; FORWARD and CALL WEBHOOK do + // not. Two occupancies of one message must not page an on-call + // engineer twice. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let event_bus = EventBus::new(); + + let rule = rule_matching_the_shared_message(); + let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule"); + + let backend = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]); + let processed = fetch_and_persist( + &db, + &event_bus, + &backend, + &filter_engine, + Some(MockMailBackend::SHARED_ACCOUNT_ID), + ) + .await + .expect("sync succeeds"); + + assert_eq!(processed, 2, "both occupancies are processed"); + let placements = db + .placements_of(shared_key()) + .await + .expect("read the message's placements"); + assert_eq!(placements.len(), 2); + + assert_eq!( + db.filter_fire_count().await.expect("read the claim ledger"), + 1, + "one claim, therefore one set of actions" + ); + assert!(db + .has_filter_fired(&rule.id, shared_key()) + .await + .expect("read the claim ledger")); + + let pending = db + .list_pending_mutations(10) + .await + .expect("list pending mutations"); + assert_eq!( + pending.len(), + 1, + "one outbox mutation for the message, not one per folder" + ); + let logs = db + .list_filter_execution_logs(10) + .await + .expect("list execution logs"); + assert_eq!(logs.len(), 1, "and one audit entry, not one per folder"); + } + + #[tokio::test] + async fn an_unverifiable_claim_skips_the_actions_rather_than_licensing_them() { + // Fail closed. If the claim itself errors, the engine cannot know + // whether this rule has already acted on this message -- and a lost + // claim check on a FORWARD is a duplicate mail the recipient cannot + // un-receive. Dropping the ledger table makes the claim error for real + // while every other write still works, so a mutation appearing here + // would be a genuine "acted without a claim", not a dead store. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + + let rule = rule_matching_the_shared_message(); + let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule"); + + let placed = MockMailBackend::shared_message_in("INBOX", "1"); + db.save_email_at(&placed.email, placed.source, &placed.placement) + .await + .expect("store the occupancy"); + + sqlx::query("DROP TABLE filter_fired") + .execute(db.pool()) + .await + .expect("drop the claim ledger"); + + let applied = + apply_filter_actions(&db, &filter_engine, &placed.email, &placed.placement).await; + assert_eq!(applied, 0, "an unclaimed rule must apply nothing"); + + let pending = db + .list_pending_mutations(10) + .await + .expect("list pending mutations"); + assert!( + pending.is_empty(), + "no outbox mutation may be enqueued off an unverifiable claim" + ); + let logs = db + .list_filter_execution_logs(10) + .await + .expect("list execution logs"); + assert!( + logs.is_empty(), + "and nothing may be recorded as having fired" + ); + } + + #[tokio::test] + async fn an_account_less_pass_still_applies_server_confirmed_removals() { + // A `PlacementKey` names its own account, so an explicitly-reported + // removal needs no ambient account context. Gating the whole removals + // block on `account_id.is_some()` would silently discard a VANISHED + // report on a mock-driven `SyncAll`, leaving a ghost the server has + // already told us is gone. + let (db, _dir) = DatabaseEngine::connect_ephemeral() + .await + .expect("ephemeral db"); + let event_bus = EventBus::new(); + let filter_engine = empty_filter_engine(); + + let staged = mock_placed("m-vanished", "inbox", "Deleted elsewhere"); + db.save_email_at(&staged.email, staged.source, &staged.placement) + .await + .expect("store the occupancy a prior pass left behind"); + + let mock = MockMailBackend::new(); + mock.add_folder(Folder { + id: "inbox".to_string(), + name: "Inbox".to_string(), + total_messages: 0, + unread_messages: 0, + }); + mock.set_removals(vec![staged.placement.key()]); + + // `None` account: the pass has nowhere to key a checkpoint and cannot + // run the `present` diff, but the removal is still actionable. + let synced = fetch_and_persist(&db, &event_bus, &mock, &filter_engine, None) + .await + .expect("sync succeeds"); + assert_eq!(synced, 0, "nothing new arrived"); + + let placements = db + .placements_of("m-vanished") + .await + .expect("read the message's placements"); + assert!( + placements.is_empty(), + "the server-confirmed removal must be applied without an account id" + ); + } } diff --git a/crates/nunciod/tests/filters_triage_e2e_test.rs b/crates/nunciod/tests/filters_triage_e2e_test.rs index 60689d3..57680d0 100644 --- a/crates/nunciod/tests/filters_triage_e2e_test.rs +++ b/crates/nunciod/tests/filters_triage_e2e_test.rs @@ -3,7 +3,7 @@ //! Boots a real `nunciod` daemon gRPC server (`nunciod::grpc:: //! serve_on_listener`) on an ephemeral loopback port, backed by a real //! temp file-backed [`DatabaseEngine`] seeded directly via -//! [`DatabaseEngine::save_email`] (standing in for messages that were synced +//! [`DatabaseEngine::save_email_at`] (standing in for messages that were synced //! before the active rule set existed) and a live [`FilterEngine`] carrying //! a real persisted rule. //! @@ -25,7 +25,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use nuncio_core::model::Email; +use nuncio_core::model::{Email, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::{FilterEngine, NsqlParser}; use nuncio_proto::v1::filters_client::FiltersClient; @@ -42,20 +42,52 @@ fn sample_email(id: &str, subject: &str) -> Email { Email { id: id.to_string(), account_id: ACCOUNT_ID.to_string(), - folder_id: "inbox".to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "triage-e2e@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some("body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, } } +/// The INBOX occupancy a seeded message sits in. `id` doubles as the UID, so +/// distinct messages occupy distinct rows. +fn sample_placement(id: &str) -> Placement { + Placement { + account_id: ACCOUNT_ID.to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + } +} + +/// Seed a message and its occupancy the way a sync pass would. +async fn seed(db: &DatabaseEngine, id: &str, subject: &str) { + db.save_email_at( + &sample_email(id, subject), + IdentitySource::Surrogate, + &sample_placement(id), + ) + .await + .unwrap_or_else(|e| panic!("seed message '{id}': {e}")); +} + +/// The read flag of a seeded message's single occupancy. +async fn read_flag_of(db: &DatabaseEngine, id: &str) -> bool { + let placements = db.placements_of(id).await.expect("read placements"); + assert_eq!( + placements.len(), + 1, + "'{id}' should occupy exactly one folder" + ); + placements[0].read +} + /// Boots the real daemon gRPC server (bearer-token authenticated) on an /// ephemeral loopback port, backed by a real temp file-backed /// `DatabaseEngine` and the given live `FilterEngine`. Returns the server @@ -143,15 +175,9 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun // Seed messages directly into the store, standing in for mail that was // synced BEFORE these rules existed -- exactly the scenario `Triage` is // for. Two match (one per rule), one does not. - db.save_email(&sample_email("m-urgent", "Urgent: server down")) - .await - .expect("seed urgent message"); - db.save_email(&sample_email("m-archive", "Please Archive Me")) - .await - .expect("seed archive message"); - db.save_email(&sample_email("m-plain", "Just saying hi")) - .await - .expect("seed non-matching message"); + seed(&db, "m-urgent", "Urgent: server down").await; + seed(&db, "m-archive", "Please Archive Me").await; + seed(&db, "m-plain", "Just saying hi").await; let mut client = FiltersClient::connect(format!("http://{addr}")) .await @@ -188,8 +214,10 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun // Real side effects, not a fabricated count: the MARK READ action // actually flipped the stored read flag... - let urgent = db.get_message("m-urgent").await.expect("get urgent"); - assert!(urgent.read, "MARK READ must genuinely flip the read flag"); + assert!( + read_flag_of(&db, "m-urgent").await, + "MARK READ must genuinely flip the occupancy's read flag" + ); // ...and the MOVE TO action actually queued a real outbox mutation. let pending = db @@ -203,8 +231,10 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun ); assert_eq!(pending[0].message_id, "m-archive"); - let plain = db.get_message("m-plain").await.expect("get plain"); - assert!(!plain.read, "a non-matching message must be left untouched"); + assert!( + !read_flag_of(&db, "m-plain").await, + "a non-matching message must be left untouched" + ); } #[tokio::test] @@ -219,12 +249,7 @@ async fn triage_streams_multiple_chunks_for_a_store_bigger_than_one_chunk() { // updates before the final `done: true` -- proof this is a real // multi-chunk stream, not one batch disguised as one. for i in 0..5 { - db.save_email(&sample_email( - &format!("m-{i:02}"), - &format!("Message number {i}"), - )) - .await - .expect("seed message"); + seed(&db, &format!("m-{i:02}"), &format!("Message number {i}")).await; } let mut client = FiltersClient::connect(format!("http://{addr}")) diff --git a/crates/nunciod/tests/no_secrets_canary_test.rs b/crates/nunciod/tests/no_secrets_canary_test.rs index 3113464..413a78c 100644 --- a/crates/nunciod/tests/no_secrets_canary_test.rs +++ b/crates/nunciod/tests/no_secrets_canary_test.rs @@ -27,10 +27,10 @@ use std::sync::{Arc, Mutex}; use nuncio_cal::{CalDavAccountConfig, CalDavClient, CalendarBackend}; use nuncio_contacts::{CardDavAccountConfig, CardDavClient, ContactsBackend}; -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher}; -use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender}; +use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender, PlacedMessage}; use nuncio_proto::v1::{ AccountConfig, AddAccountRequest, GetStatusRequest, MarkReadRequest, SendMessageRequest, SyncRequest, TlsMode, @@ -139,17 +139,35 @@ fn sentinel_inbound_email() -> Email { Email { id: "msg-canary-1".to_string(), account_id: ACCOUNT_ID.to_string(), - folder_id: "inbox".to_string(), - remote_id: "msg-canary-1".to_string(), - uid_validity: "1".to_string(), subject: SENTINEL_SUBJECT.to_string(), sender: SENTINEL_SENDER.to_string(), recipient: SENTINEL_RECIPIENT.to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some(SENTINEL_BODY.to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, + } +} + +/// The INBOX occupancy the sentinel message is found in. +fn sentinel_placement() -> Placement { + Placement { + account_id: ACCOUNT_ID.to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: "msg-canary-1".to_string(), + read: false, + } +} + +/// The sentinel message as a backend would surface it. +fn sentinel_placed_message() -> PlacedMessage { + PlacedMessage { + email: sentinel_inbound_email(), + source: IdentitySource::Surrogate, + placement: sentinel_placement(), } } @@ -212,7 +230,7 @@ async fn drive_daemon() { total_messages: 1, unread_messages: 1, }); - mock_backend.add_message(sentinel_inbound_email()); + mock_backend.add_message(sentinel_placed_message()); let mock_sender = MockMessageSender::new(); let overrides = MailEngineOverrides { mail_backend: Some(Arc::new(mock_backend)), @@ -307,7 +325,12 @@ async fn drive_filter_and_webhook() { ) .expect("parse rule"); let engine = FilterEngine::new(vec![rule]).expect("compile rule"); - let matches = engine.evaluate(&sentinel_inbound_email()); + let sentinel = sentinel_inbound_email(); + let placement = sentinel_placement(); + let matches = engine.evaluate(nuncio_filter::PlacedEmail { + email: &sentinel, + placement: &placement, + }); assert_eq!(matches.len(), 1, "rule must fire on the sentinel message"); let mock_server = MockServer::start().await; diff --git a/crates/nunciod/tests/outbox_executor_e2e_test.rs b/crates/nunciod/tests/outbox_executor_e2e_test.rs index 6a9c42e..f16b4a9 100644 --- a/crates/nunciod/tests/outbox_executor_e2e_test.rs +++ b/crates/nunciod/tests/outbox_executor_e2e_test.rs @@ -12,7 +12,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use async_trait::async_trait; -use nuncio_core::model::Email; +use nuncio_core::model::{Email, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher, WebhookError}; use nuncio_mail::{ @@ -116,17 +116,28 @@ fn sample_email(subject: &str) -> Email { Email { id: "surrogate-inbox-42".to_string(), account_id: ACCOUNT_ID.to_string(), - folder_id: FOLDER_ID.to_string(), - remote_id: "42".to_string(), - uid_validity: "1".to_string(), subject: subject.to_string(), sender: "alice@nuncio.mx".to_string(), recipient: "owner@nuncio.mx".to_string(), received_at: 1_700_000_000, - read: false, body_plain: Some("original body".to_string()), body_html: None, attachments: Vec::new(), + message_id: None, + content_hash: None, + } +} + +/// The mailbox occupancy the seeded message sits in. The outbox recovers its +/// remote addressing from here now -- the UID and the UIDVALIDITY scope it was +/// captured under are properties of the occupancy, not of the message. +fn sample_placement(remote_id: &str) -> Placement { + Placement { + account_id: ACCOUNT_ID.to_string(), + folder_id: FOLDER_ID.to_string(), + uid_validity: "1".to_string(), + remote_id: remote_id.to_string(), + read: false, } } @@ -142,11 +153,14 @@ async fn seed_and_enqueue(db: &DatabaseEngine, rule_nsql: &str, subject: &str) { .expect("save folder checkpoint"); let email = sample_email(subject); - db.save_email(&email).await.expect("save email"); + let placement = sample_placement("42"); + db.save_email_at(&email, IdentitySource::Surrogate, &placement) + .await + .expect("save email"); let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule"); let engine = FilterEngine::new(vec![rule]).expect("compile rule"); - apply_filter_actions(db, &engine, &email).await; + apply_filter_actions(db, &engine, &email, &placement).await; let pending = db.list_pending_mutations(10).await.expect("list pending"); assert_eq!( @@ -173,14 +187,16 @@ async fn seed_and_enqueue_distinct( .await .expect("save folder checkpoint"); - // `remote_id`/`uid_validity` (not just `id`) are part of the store's - // `messages` identity UNIQUE index -- leaving them at `sample_email`'s - // shared default would make this INSERT OR REPLACE the OTHER seeded - // message sharing that same remote identity out from under it. + // The occupancy needs its own UID as well as its own message key: the + // placements primary key is (account, folder, uidvalidity, uid), so + // leaving `remote_id` at the shared default would repoint the OTHER + // seeded message's occupancy at this message. let mut email = sample_email(subject); email.id = message_id.to_string(); - email.remote_id = message_id.to_string(); - db.save_email(&email).await.expect("save email"); + let placement = sample_placement(message_id); + db.save_email_at(&email, IdentitySource::Surrogate, &placement) + .await + .expect("save email"); let before = db .list_pending_mutations(100) @@ -190,7 +206,7 @@ async fn seed_and_enqueue_distinct( let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule"); let engine = FilterEngine::new(vec![rule]).expect("compile rule"); - apply_filter_actions(db, &engine, &email).await; + apply_filter_actions(db, &engine, &email, &placement).await; let after = db .list_pending_mutations(100) diff --git a/crates/nunciod/tests/spine_e2e_test.rs b/crates/nunciod/tests/spine_e2e_test.rs index b15c2ae..81f798e 100644 --- a/crates/nunciod/tests/spine_e2e_test.rs +++ b/crates/nunciod/tests/spine_e2e_test.rs @@ -31,10 +31,10 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use nuncio_core::model::{Email, Folder}; +use nuncio_core::model::{Email, Folder, IdentitySource, Placement}; use nuncio_core::EventBus; use nuncio_filter::FilterEngine; -use nuncio_mail::{MockMailBackend, MockMessageSender}; +use nuncio_mail::{MockMailBackend, MockMessageSender, PlacedMessage}; use nuncio_proto::v1::event::Kind; use nuncio_proto::v1::{ AccountConfig, AddAccountRequest, GetMessageRequest, ListAccountsRequest, ListMessagesRequest, @@ -73,21 +73,31 @@ fn sample_account_config() -> AccountConfig { } } -fn mock_inbound_email(id: &str, subject: &str, body: &str) -> Email { - Email { - id: id.to_string(), - account_id: ACCOUNT_ID.to_string(), - folder_id: "inbox".to_string(), - remote_id: id.to_string(), - uid_validity: "1".to_string(), - subject: subject.to_string(), - sender: "alice@nuncio.mx".to_string(), - recipient: ACCOUNT_EMAIL.to_string(), - received_at: 1_700_000_000, - read: false, - body_plain: Some(body.to_string()), - body_html: None, - attachments: Vec::new(), +/// One inbound message as a backend surfaces it: identity, the tier the key +/// came from, and the INBOX occupancy this pass found it in. +fn mock_inbound_email(id: &str, subject: &str, body: &str) -> PlacedMessage { + PlacedMessage { + email: Email { + id: id.to_string(), + account_id: ACCOUNT_ID.to_string(), + subject: subject.to_string(), + sender: "alice@nuncio.mx".to_string(), + recipient: ACCOUNT_EMAIL.to_string(), + received_at: 1_700_000_000, + body_plain: Some(body.to_string()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + source: IdentitySource::Surrogate, + placement: Placement { + account_id: ACCOUNT_ID.to_string(), + folder_id: "inbox".to_string(), + uid_validity: "1".to_string(), + remote_id: id.to_string(), + read: false, + }, } } diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index c564ae6..c23b358 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -104,7 +104,7 @@ part of the original M1–M7 numbering: `tls_mode_from_db` silent `ImplicitTls` fallback (#271), the `NaturalLanguageScheduler` that returns wrong times (#258), plus CLI and audit cleanups. -- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 44 open). Six tracks +- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 56 open). Six tracks that must land before `nuncio.v1` can be frozen: **WS-A** contract hardening (typed errors, `Timestamp`/`Duration` unification, keyset pagination, id conventions, enum hygiene, account transport `oneof`); **WS-B** mail model and @@ -228,19 +228,29 @@ The engine becomes production-solid. ### M6.5, WS and OBS — the work between M6 and M7 Opened after M5, when the reshape needed before an honest `v1` freeze proved -larger than M6. Status as of 2026-08-07: +larger than M6. Status as of 2026-08-12: | Track | Closed / Open | Scope | | :--- | :---: | :--- | | **M6.5** Feature completeness & de-fabrication | 3 / 8 | Remove the remaining fabricated or silently-wrong paths (Contacts sync wiring #255, DAV XML namespaces #257, TLS-mode fallback #271, NL scheduler #258) | | **WS-A** Contract hardening (pre-freeze reshape) | 6 / 3 | Typed errors, `Timestamp`/`Duration`, keyset pagination, id conventions, enum hygiene, account transport `oneof` — **gates M7** | -| **WS-B** Mail model & mutations | 1 / 9 | Message identity, drafts, folder management, attachments on receive | -| **WS-C** Sync, push & lifecycle | 4 / 4 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown | +| **WS-B** Mail model & mutations | 1 / 14 | Message identity, drafts, folder management, attachments on receive | +| **WS-C** Sync, push & lifecycle | 4 / 9 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown | | **WS-D** Security to 9/10 | 4 / 8 | Whole-DB SQLCipher (#299), UDS / named-pipe IPC (#305), signed updates (#300), HTML sanitization (#302) | -| **WS-E** Calendar/Contacts write & Search | 2 / 10 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service | +| **WS-E** Calendar/Contacts write & Search | 2 / 12 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service | | **WS-F** Ops, usability & maintainability | 5 / 10 | CLI polish, `System.Backup`, and the `grpc.rs` (#333) / `DatabaseEngine` (#334) decompositions | | **OBS-1 … OBS-9** Observability | 9 / 0 | **COMPLETE.** Tracing subscriber, RPC spans + request-id, domain/lifecycle coverage, protocol instrumentation, `GetStatus`, CLI verbosity, redaction canary | +WS-B and WS-C grew when **ADR 0002** decided +the convergent multi-engine sync model and broke it into #386–#397. Those are +in flight, not closed: the message/placement identity split, Message-ID and +content-hash capture, `user_version` gating, the raw IMAP command layer, the +four-rung change enumeration ladder, and three-state verified mutations are all +implemented and green on a branch, and the counts above move when it merges. +Still open behind them: the mutation RPCs and durable conflict surface (#395), +DAV href/ETag identity and `sync-collection` (#392, #393), the scheduler and +rate limiting (#396), and the ≥3-engine convergence harness (#397). + The OBS stories (#357–#365) were merged without a milestone assignment; the epic is complete regardless, and `docs/LOGGING.md` is its reference. diff --git a/docs/roadmap/index.html b/docs/roadmap/index.html index ac57958..2aa1164 100644 --- a/docs/roadmap/index.html +++ b/docs/roadmap/index.html @@ -280,11 +280,13 @@

Opened after M5, when the reshape needed before an honest v1 freeze proved larger than M6 alone. M6.5 de-fabrication (3/11) removes the remaining silently-wrong paths: Contacts sync wiring (#255), namespace-aware DAV XML (#257), the TLS-mode fallback (#271), the NL scheduler (#258). - WS-A … WS-F (22/66) is the pre-freeze reshape: contract hardening (WS-A, gates M7), mail model + WS-A … WS-F (22/78) is the pre-freeze reshape: contract hardening (WS-A, gates M7), mail model & mutations (WS-B), sync/push/lifecycle (WS-C), security to 9/10 (WS-D — SQLCipher #299, UDS & named-pipe IPC #305, signed updates #300), calendar/contacts write-back & search (WS-E), and ops, usability & maintainability (WS-F — including the grpc.rs #333 and DatabaseEngine - #334 decompositions). OBS-1 … OBS-9 is complete (9/9, #357–#365): layered tracing subscriber, per-RPC + #334 decompositions). WS-B and WS-C grew when ADR 0002 decided the convergent multi-engine sync + model and broke it into #386–#397; the message/placement identity split, change enumeration and verified + mutations are implemented and green on a branch, not yet merged. OBS-1 … OBS-9 is complete (9/9, #357–#365): layered tracing subscriber, per-RPC spans with request-id correlation, domain and lifecycle coverage, protocol instrumentation, a real System.GetStatus, CLI -v/-vv with typed errors, and a Redacted<T> policy with a no-secrets-in-logs canary.

diff --git a/docs/superpowers/plans/2026-08-11-message-placement-split.md b/docs/superpowers/plans/2026-08-11-message-placement-split.md new file mode 100644 index 0000000..f859f0c --- /dev/null +++ b/docs/superpowers/plans/2026-08-11-message-placement-split.md @@ -0,0 +1,2377 @@ +# Message/Placement Identity Split — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Separate a mail message's *identity* (content-addressed, folder-independent) from its *placements* (one row per folder the message occupies), so a message moved between folders stays one message. + +**Architecture:** `messages` becomes identity + immutable content, keyed by a content-addressed `message_key` derived through a four-tier precedence (`EMAILID` → `X-GM-MSGID` → `(account, Message-ID, content_hash)` → legacy surrogate). A new `placements` table holds `(account_id, folder_id, uidvalidity, uid)` as its primary key plus the per-mailbox read flag. The Rust `Email` struct **loses** its `folder_id` / `remote_id` / `uid_validity` / `read` scalars so the compiler flags every site that assumed a message lives in exactly one folder. Filter evaluation becomes per-placement, and fire-once becomes an atomic `(rule_id, message_key)` claim. + +**Tech Stack:** Rust 2021 (toolchain 1.97.1), `sqlx` + SQLite/WAL, `sha2`, `tonic`/`prost`, `tokio`. + +## Global Constraints + +- **Branch:** `story/message-placement-split`, already created off `dev` with `story/message-id-capture` (#386) and `story/schema-version-and-token-scheme` (#387) merged in. Baseline gate is green on it (627+ tests). Do not rebase onto the #401–#403 chain. +- **Gate.** Removing `Email`'s folder scalars breaks four crates until Task 9 lands, so the gate is staged: + - **Tasks 1–8:** all three of `cargo fmt --all -- --check`, `cargo clippy -p --all-targets -- -D warnings`, and `cargo test -p ` must pass. The workspace will not compile end-to-end during these tasks; that is expected and is not a reason to stop or to widen the task. + **Clippy is not optional at the per-crate stage.** `cargo check-all` is the workspace-wide clippy gate and cannot run mid-refactor, so a per-crate clippy run is the only thing standing between a lint failure and the end of the plan. Omitting it once already let `should_implement_trait` reach the branch undetected. + - **Task 9 onward, and the PR head:** the full gate — `cargo fmt --all -- --check`, then `cargo check-all`, then `cargo test-all`. + - Warnings are hard errors throughout; `unwrap_used` / `expect_used` / `panic` / `todo` are `deny` outside tests. Never silence a lint to get a commit through. + - `cargo test-all` can exceed a 10-minute tool timeout on this workspace; allow ~540s. +- **Commits:** Conventional Commits, imperative, <72-char subject, **no AI attribution**. `Refs #394` in the message body is fine. +- **Comments:** explain intent, constraints, rationale. **No issue numbers, GH refs, or `Phase N` breadcrumbs in `.rs` / `.proto` comments.** +- **No live network in tests.** IMAP/JMAP/SMTP/CalDAV/CardDAV all mocked; keyring via `MockKeyring`. Ephemeral SQLite (`tempfile` / `:memory:`) per test. +- **Never key identity on a Message-ID alone.** Tier 3 requires `Message-ID` **and** `content_hash` both present; if either is missing, fall through to the surrogate tier. First-write-wins on the body. +- **`uidvalidity` is in the placement primary key.** UIDs restart at 1 after a UIDVALIDITY bump and would otherwise collide with old rows. +- **FTS plaintext-orphan rule.** Dropping *a* placement must NOT delete the `messages_fts` row; dropping the *last* placement must. Inverted, this leaks plaintext bodies in orphaned FTS rows. +- **No `.proto` schema changes in this plan.** Only a doc-comment correction on `Message.id`. Placement fields on the wire are #395's scope, and the repo allows one proto-touching story in flight at a time. + +--- + +## File Structure + +| File | Responsibility | Change | +| --- | --- | --- | +| `crates/nuncio-core/src/model.rs` | `Email` (identity + content), new `Placement`, new `MessageIdentity` / `IdentitySource`, `derive_message_key` | Modify | +| `crates/nuncio-store/src/db.rs` | `placements` + `filter_fired` schema, `IDENTITY_SCHEMA_VERSION` bump, all message reads/writes/deletes | Modify | +| `crates/nuncio-store/src/search.rs` | FTS hit mapping onto `message_key` | Modify | +| `crates/nuncio-filter/src/engine.rs` | per-placement condition evaluation | Modify | +| `crates/nuncio-filter/src/validator.rs` | `Folder` / `Account` field validation unchanged, comments corrected | Modify | +| `crates/nuncio-mail/src/imap.rs` | emit identity + placement instead of a surrogate-keyed `Email` | Modify | +| `crates/nuncio-mail/src/jmap.rs` | same | Modify | +| `crates/nuncio-mail/src/mock.rs` | mock backend emits placements | Modify | +| `crates/nunciod/src/sync.rs` | placement-aware new-vs-seen, per-placement filter fan-out, placement removals | Modify | +| `crates/nunciod/src/grpc.rs` | present a placement-in-context on the wire | Modify | +| `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto` | `Message.id` doc comment only | Modify | + +--- + +## Task 1: Core identity and placement types + +**Files:** +- Modify: `crates/nuncio-core/src/model.rs` +- Test: `crates/nuncio-core/src/model.rs` (inline `#[cfg(test)] mod tests`) + +**Interfaces:** +- Consumes: `Email::normalize_message_id`, `Email::content_hash_of` (both already exist from #386). +- Produces: + - `pub enum IdentitySource { EmailId, GmailMsgId, MessageIdContent, Surrogate }` with `as_str(&self) -> &'static str` and `pub fn from_str(s: &str) -> Option` + - `pub struct MessageIdentity { pub key: String, pub source: IdentitySource }` + - `pub struct RemoteIdentity<'a> { pub email_id: Option<&'a str>, pub gm_msgid: Option<&'a str>, pub message_id: Option<&'a str>, pub content_hash: Option<&'a str> }` + - `pub struct Placement { pub account_id: String, pub folder_id: String, pub uid_validity: String, pub remote_id: String, pub read: bool }` + - `Email::derive_message_key(account_id: &str, remote: RemoteIdentity<'_>, folder_id: &str, uid_validity: &str, remote_id: &str) -> MessageIdentity` + - `Email` keeps `id`, `account_id`, `subject`, `sender`, `recipient`, `received_at`, `body_plain`, `body_html`, `attachments`, `message_id`, `content_hash`; **loses** `folder_id`, `remote_id`, `uid_validity`, `read`. + +- [ ] **Step 1: Write the failing tests** + +Add to the existing `mod tests` in `crates/nuncio-core/src/model.rs`: + +```rust +#[test] +fn message_key_precedence_prefers_emailid_over_everything_below_it() { + let remote = RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: Some("1234567890"), + message_id: Some("a@b.example"), + content_hash: Some("deadbeef"), + }; + let id = Email::derive_message_key("acct-1", remote, "INBOX", "42", "5"); + assert_eq!(id.source, IdentitySource::EmailId); + + // Same EMAILID in a different folder is the SAME message. + let elsewhere = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: Some("M00000001"), gm_msgid: None, message_id: None, content_hash: None }, + "Archive", + "99", + "7", + ); + assert_eq!(id.key, elsewhere.key); +} + +#[test] +fn message_key_is_account_scoped_at_every_tier() { + let remote = || RemoteIdentity { + email_id: Some("M00000001"), + gm_msgid: None, + message_id: None, + content_hash: None, + }; + let a = Email::derive_message_key("acct-1", remote(), "INBOX", "42", "5"); + let b = Email::derive_message_key("acct-2", remote(), "INBOX", "42", "5"); + assert_ne!(a.key, b.key, "the same EMAILID in two accounts must not collide"); +} + +#[test] +fn a_message_id_without_a_content_hash_never_reaches_the_content_tier() { + // The non-negotiable: a Message-ID is public and forgeable, so it must + // never key storage on its own. Missing content hash falls through to the + // folder-scoped surrogate rather than trusting the header alone. + let id = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: None, message_id: Some("a@b.example"), content_hash: None }, + "INBOX", + "42", + "5", + ); + assert_eq!(id.source, IdentitySource::Surrogate); + + let surrogate_only = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: None, message_id: None, content_hash: None }, + "INBOX", + "42", + "5", + ); + assert_eq!(id.key, surrogate_only.key); +} + +#[test] +fn the_same_message_id_with_different_content_is_two_messages() { + let mk = |hash: &str| { + Email::derive_message_key( + "acct-1", + RemoteIdentity { + email_id: None, + gm_msgid: None, + message_id: Some("a@b.example"), + content_hash: Some(hash), + }, + "INBOX", + "42", + "5", + ) + }; + let list_copy = mk("1111"); + let direct_copy = mk("2222"); + assert_eq!(list_copy.source, IdentitySource::MessageIdContent); + assert_ne!(list_copy.key, direct_copy.key); +} + +#[test] +fn tiers_are_domain_separated_so_a_shared_value_cannot_collide_across_them() { + let as_emailid = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: Some("X"), gm_msgid: None, message_id: None, content_hash: None }, + "INBOX", "42", "5", + ); + let as_gmsgid = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: Some("X"), message_id: None, content_hash: None }, + "INBOX", "42", "5", + ); + assert_ne!(as_emailid.key, as_gmsgid.key); +} + +#[test] +fn identity_source_round_trips_through_its_stored_string() { + for source in [ + IdentitySource::EmailId, + IdentitySource::GmailMsgId, + IdentitySource::MessageIdContent, + IdentitySource::Surrogate, + ] { + assert_eq!(IdentitySource::from_str(source.as_str()), Some(source)); + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-core model::tests` +Expected: FAIL — `cannot find type RemoteIdentity`, `no function derive_message_key`. + +- [ ] **Step 3: Implement the identity types** + +In `crates/nuncio-core/src/model.rs`, add above `impl Email`: + +```rust +/// Which tier of the identity precedence produced a [`MessageIdentity`]. +/// +/// Persisted alongside the key so a later pass can tell a server-assigned +/// identity from a locally-derived fallback without recomputing it, and so a +/// store can be audited for how much of it rests on the weakest tier. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum IdentitySource { + /// RFC 8474 OBJECTID `EMAILID` -- server-assigned and stable across folders. + EmailId, + /// Gmail's `X-GM-MSGID`, the same guarantee by a vendor extension. + GmailMsgId, + /// `Message-ID` **and** a content hash together. Never the header alone. + MessageIdContent, + /// Folder-scoped last resort: the message is only identified by where it + /// currently sits, so the same mail in another folder is a second message. + Surrogate, +} + +impl IdentitySource { + /// Stable token persisted in `messages.identity_source`. + pub fn as_str(&self) -> &'static str { + match self { + Self::EmailId => "emailid", + Self::GmailMsgId => "gmsgid", + Self::MessageIdContent => "msgid_content", + Self::Surrogate => "surrogate", + } + } + + /// Parse a token written by [`Self::as_str`]. Returns `None` for anything + /// else so an unrecognised value fails loudly rather than defaulting to a + /// tier that would overstate how trustworthy the key is. + pub fn from_str(s: &str) -> Option { + match s { + "emailid" => Some(Self::EmailId), + "gmsgid" => Some(Self::GmailMsgId), + "msgid_content" => Some(Self::MessageIdContent), + "surrogate" => Some(Self::Surrogate), + _ => None, + } + } +} + +/// A content-addressed message key together with the tier that produced it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MessageIdentity { + /// Hex SHA-256 key identifying the message within its account. + pub key: String, + /// Which precedence tier produced `key`. + pub source: IdentitySource, +} + +/// Server-supplied identity hints for one message, in precedence order. +/// +/// Borrowed rather than owned because every caller already holds these as +/// fields on a parsed response and only needs them for the duration of the +/// key derivation. +#[derive(Debug, Clone, Copy, Default)] +pub struct RemoteIdentity<'a> { + /// RFC 8474 `EMAILID` from a `FETCH ... (EMAILID)`, if the server has OBJECTID. + pub email_id: Option<&'a str>, + /// Gmail `X-GM-MSGID`, if the server advertises `X-GM-EXT-1`. + pub gm_msgid: Option<&'a str>, + /// Normalized `Message-ID` (see [`Email::normalize_message_id`]). + pub message_id: Option<&'a str>, + /// Hex SHA-256 over the full RFC822 octets (see [`Email::content_hash_of`]). + pub content_hash: Option<&'a str>, +} + +/// One occupancy of a message in one mailbox. +/// +/// A message can sit in several folders at once (an IMAP `COPY`, a Gmail label, +/// a message that is both in a thread's folder and in `\All`). The addressing +/// coordinates and the read flag are properties of the *occupancy*, not of the +/// message: IMAP `\Seen` is per-mailbox, and a UID is only meaningful inside one +/// `uid_validity` scope of one folder. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Placement { + /// Account owning the mailbox. + pub account_id: String, + /// Mailbox folder identifier (e.g. "inbox"). + pub folder_id: String, + /// The IMAP UIDVALIDITY the `remote_id` was captured under, as a decimal + /// string; a stable sentinel for protocols without one (JMAP). In the + /// primary key because UIDs restart at 1 after a bump and would otherwise + /// collide with rows written before it. + pub uid_validity: String, + /// Protocol-native id addressing the message in this mailbox: the IMAP UID + /// as a decimal string, or the JMAP Email object id. + pub remote_id: String, + /// Read/unread state **in this mailbox**. + pub read: bool, +} +``` + +- [ ] **Step 4: Implement `derive_message_key` and reshape `Email`** + +Add to `impl Email`: + +```rust + /// Domain-separated, length-prefixed digest over an identity tier's inputs. + /// + /// The tier label participates in the hash so that a value appearing in two + /// different tiers (a server whose `EMAILID` happens to equal another's + /// `X-GM-MSGID`) cannot produce one key. Length prefixes stop two distinct + /// field tuples serializing to the same byte stream, exactly as in + /// [`Email::surrogate_id`]. + fn identity_digest(tier: &str, account_id: &str, parts: &[&str]) -> String { + let mut hasher = Sha256::new(); + for field in [tier, account_id].into_iter().chain(parts.iter().copied()) { + hasher.update((field.len() as u64).to_le_bytes()); + hasher.update(field.as_bytes()); + } + hex::encode(hasher.finalize()) + } + + /// Derive the account-scoped message key by identity precedence. + /// + /// `EMAILID` → `X-GM-MSGID` → (`Message-ID` + content hash) → surrogate. + /// Every tier is account-scoped: identity is only ever claimed within the + /// account that fetched it, so one account's server cannot mint a key that + /// addresses another account's mail. + /// + /// The third tier requires **both** a `Message-ID` and a content hash. A + /// `Message-ID` is public -- it is echoed in the `References` of every reply + /// -- and forgeable, so keying on it alone would let a crafted message + /// address, and with an upsert overwrite, a message the user already trusts, + /// and would evade filtering by never being classified as new. Pairing it + /// with the content hash also gets the ordinary cases right: a mailing-list + /// copy and a direct copy share a `Message-ID` but differ in content, and so + /// are correctly two messages. + /// + /// The surrogate tier is folder-scoped and therefore *not* stable across a + /// move; it is the honest answer when the server offered nothing better. + pub fn derive_message_key( + account_id: &str, + remote: RemoteIdentity<'_>, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + ) -> MessageIdentity { + if let Some(email_id) = remote.email_id.filter(|v| !v.is_empty()) { + return MessageIdentity { + key: Self::identity_digest("emailid", account_id, &[email_id]), + source: IdentitySource::EmailId, + }; + } + if let Some(gm_msgid) = remote.gm_msgid.filter(|v| !v.is_empty()) { + return MessageIdentity { + key: Self::identity_digest("gmsgid", account_id, &[gm_msgid]), + source: IdentitySource::GmailMsgId, + }; + } + if let (Some(message_id), Some(content_hash)) = ( + remote.message_id.filter(|v| !v.is_empty()), + remote.content_hash.filter(|v| !v.is_empty()), + ) { + return MessageIdentity { + key: Self::identity_digest("msgid_content", account_id, &[message_id, content_hash]), + source: IdentitySource::MessageIdContent, + }; + } + MessageIdentity { + key: Self::identity_digest("surrogate", account_id, &[folder_id, uid_validity, remote_id]), + source: IdentitySource::Surrogate, + } + } +``` + +Then delete the `folder_id`, `remote_id`, `uid_validity`, and `read` fields from `struct Email`, and replace the doc comment on `Email::id` with: + +```rust + /// Opaque, account-scoped identity for the message itself. + /// + /// Derived by [`Email::derive_message_key`], so it is independent of which + /// folder the message currently occupies: moving a message between folders + /// does not change it. Where the message actually sits, and its read state + /// there, live in [`Placement`] rows instead. Callers must treat this as an + /// opaque token -- it carries no decodable structure. + pub id: String, +``` + +Keep `Email::surrogate_id` in place — the surrogate tier still uses the same +construction and existing tests cover it. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p nuncio-core model::tests` +Expected: PASS. Other crates will not compile yet; that is expected and is fixed in later tasks. + +- [ ] **Step 6: Commit** + +```bash +git add crates/nuncio-core/src/model.rs +git commit -m "feat(core): separate message identity from mailbox placement" +``` + +--- + +## Task 2: Store schema — placements, fire-once claims, and the version bump + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (`migrate`, `reset_cached_message_store`, `IDENTITY_SCHEMA_VERSION`) +- Test: `crates/nuncio-store/src/db.rs` inline tests + +**Interfaces:** +- Consumes: `DatabaseEngine::reconcile_schema_version` and `reset_cached_message_store` (both from #387). +- Produces: `placements` and `filter_fired` tables; `messages` reshaped to `message_key` + identity columns; `IDENTITY_SCHEMA_VERSION == 2`; and three store methods — + - `claim_filter_fire(&self, rule_id: &str, message_key: &str) -> Result` + - `has_filter_fired(&self, rule_id: &str, message_key: &str) -> Result` + - `filter_fire_count(&self) -> Result` + + The last two are `pub` (not test-only) because `nunciod` asserts on the ledger from another crate and cannot reach the pool. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn placements_primary_key_includes_uidvalidity() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + + // Same folder, same UID, different UIDVALIDITY: two distinct placements. + // After a UIDVALIDITY bump the server restarts UIDs at 1, so without + // uidvalidity in the key the new mail would overwrite the old. + for validity in ["42", "43"] { + sqlx::query( + "INSERT INTO placements (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES (?, ?, ?, ?, ?, 0)", + ) + .bind("acct-1") + .bind("INBOX") + .bind(validity) + .bind("1") + .bind(format!("key-{validity}")) + .execute(engine.pool()) + .await + .expect("both placements must insert"); + } + + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(count, 2); +} + +#[tokio::test] +async fn a_filter_fired_claim_is_won_exactly_once() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + assert!(engine.claim_filter_fire("rule-1", "key-1").await.unwrap()); + assert!( + !engine.claim_filter_fire("rule-1", "key-1").await.unwrap(), + "a second claim on the same (rule, message) must lose" + ); + assert!( + engine.claim_filter_fire("rule-2", "key-1").await.unwrap(), + "a different rule still gets its own claim" + ); +} + +#[tokio::test] +async fn bumping_the_identity_schema_rebuilds_placements_and_claims_too() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nuncio.db"); + let secrets = crate::vault::MockKeyring::new(); + + { + let engine = DatabaseEngine::connect_file(&db_path, &secrets).await.unwrap(); + sqlx::query( + "INSERT INTO placements (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES ('a', 'INBOX', '1', '1', 'k', 0)", + ) + .execute(engine.pool()) + .await + .unwrap(); + engine.claim_filter_fire("rule-1", "k").await.unwrap(); + // Pretend this file was written by the previous identity generation. + sqlx::query("PRAGMA user_version = 1").execute(engine.pool()).await.unwrap(); + engine.close().await; + } + + let engine = DatabaseEngine::connect_file(&db_path, &secrets).await.unwrap(); + let (placements,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM placements") + .fetch_one(engine.pool()) + .await + .unwrap(); + let (claims,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(placements, 0, "placements are derived from the server and must be rebuilt"); + assert_eq!(claims, 0, "fire-once claims name message keys that no longer resolve"); + engine.close().await; +} +``` + +If `DatabaseEngine` has no `pool()` accessor, add one gated to the crate: + +```rust + /// Test-only access to the underlying pool for schema-level assertions. + #[cfg(test)] + pub(crate) fn pool(&self) -> &sqlx::SqlitePool { + &self.pool + } +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-store db::tests::placements_primary_key_includes_uidvalidity db::tests::a_filter_fired_claim_is_won_exactly_once` +Expected: FAIL — `no such table: placements`, `no method claim_filter_fire`. + +- [ ] **Step 3: Add the schema** + +In `DatabaseEngine::migrate`, replace the `CREATE TABLE IF NOT EXISTS messages (...)` block with: + +```sql + -- Identity and immutable content, one row per message. Where the + -- message sits lives in `placements`; nothing here is folder-scoped. + CREATE TABLE IF NOT EXISTS messages ( + message_key TEXT PRIMARY KEY NOT NULL, + account_id TEXT NOT NULL, + subject TEXT NOT NULL, + sender TEXT NOT NULL, + recipient TEXT NOT NULL, + received_at INTEGER NOT NULL, + body_plain TEXT, + body_html TEXT, + message_id TEXT, + content_hash TEXT, + identity_source TEXT NOT NULL DEFAULT 'surrogate' + ); + + -- One row per (mailbox, message) occupancy. `uidvalidity` is in the + -- primary key because UIDs restart at 1 after a bump: without it the + -- first message of the new generation would overwrite the row of + -- whichever old message shared its UID. + CREATE TABLE IF NOT EXISTS placements ( + account_id TEXT NOT NULL, + folder_id TEXT NOT NULL, + uidvalidity TEXT NOT NULL, + uid TEXT NOT NULL, + message_key TEXT NOT NULL, + read_flag INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (account_id, folder_id, uidvalidity, uid) + ); + + -- Fire-once ledger for filter actions. Keyed on the message + -- *identity*, not a placement, so a message arriving in a second + -- folder does not re-fire a rule that already acted on it -- which + -- matters most for FORWARD and CALL WEBHOOK, whose effects land on + -- third parties and cannot be undone by convergence. + CREATE TABLE IF NOT EXISTS filter_fired ( + rule_id TEXT NOT NULL, + message_key TEXT NOT NULL, + fired_at INTEGER NOT NULL, + PRIMARY KEY (rule_id, message_key) + ); +``` + +Add alongside the existing indexes: + +```sql + CREATE INDEX IF NOT EXISTS idx_placements_message ON placements(message_key); + CREATE INDEX IF NOT EXISTS idx_placements_folder ON placements(account_id, folder_id); +``` + +The `messages_ad` trigger stays exactly as it is. Update only its comment to +record why it must remain on `messages`: + +```sql + -- Subject/sender are never encrypted in `messages`, so a delete-only trigger is + -- sufficient here: it just keeps the FTS index free of orphaned rows. Insertion and + -- update of `messages_fts` content happens explicitly in `upsert_message`, never via an + -- AFTER INSERT/UPDATE trigger, because such a trigger would only ever see the + -- ciphertext body column. + -- + -- This trigger MUST stay on `messages` and never move to `placements`. Dropping one + -- placement of a message that still sits in another folder must leave the body + -- searchable; only losing the last placement removes the `messages` row, and that is + -- what reaps the FTS row here. Firing on `placements` instead would delete the index + -- entry while the body remained -- and, inverted, leaving it off `messages` entirely + -- would strand plaintext bodies in `messages_fts` after the message itself was gone. + CREATE TRIGGER IF NOT EXISTS messages_ad AFTER DELETE ON messages BEGIN + DELETE FROM messages_fts WHERE message_key = old.message_key; + END; +``` + +Change the FTS virtual table's key column to match: + +```sql + CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts USING fts5( + message_key UNINDEXED, + subject, + sender, + body_plain, + tokenize = 'trigram' + ); +``` + +- [ ] **Step 4: Bump the version and extend the reset** + +```rust + pub const IDENTITY_SCHEMA_VERSION: i64 = 2; +``` + +Extend the statement list inside `reset_cached_message_store` to: + +```rust + for statement in [ + "DELETE FROM messages", + "DELETE FROM messages_fts", + "DELETE FROM placements", + "DELETE FROM filter_fired", + "DELETE FROM folder_sync_state", + "DELETE FROM filter_execution_logs", + ] { +``` + +and add to its doc comment, in the "Cleared" sentence, `mailbox placements` and +`the filter fire-once ledger`. + +Apply the same additions to the `for statement in [...]` list in the +existing `clear_cached_message_data`-style helper at `db.rs:596` if it is a +separate function. + +- [ ] **Step 5: Add `claim_filter_fire`** + +```rust + /// Atomically claim the right to run `rule_id`'s actions against + /// `message_key`, returning `true` only for the claim that won. + /// + /// The claim is the fire-once guard for side-effecting actions. It is keyed + /// on message identity rather than on a placement because the same message + /// legitimately arrives in several folders -- and, under the old + /// folder-scoped identity, each arrival looked like a different message and + /// fired the rule again. `INSERT ... ON CONFLICT DO NOTHING` makes winning + /// the claim a single atomic statement, so two passes racing over the same + /// message cannot both act. + pub async fn claim_filter_fire( + &self, + rule_id: &str, + message_key: &str, + ) -> Result { + let result = sqlx::query( + "INSERT INTO filter_fired (rule_id, message_key, fired_at) VALUES (?, ?, ?) + ON CONFLICT (rule_id, message_key) DO NOTHING", + ) + .bind(rule_id) + .bind(message_key) + .bind(chrono::Utc::now().timestamp()) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(result.rows_affected() == 1) + } + + /// Whether `rule_id` has already acted on `message_key`. + /// + /// Public rather than test-only: the claim ledger answers a question + /// operators and the outbox both have a real reason to ask -- "did this rule + /// already act on this message" -- and callers in other crates cannot reach + /// the pool directly. + pub async fn has_filter_fired( + &self, + rule_id: &str, + message_key: &str, + ) -> Result { + let found: Option<(i64,)> = sqlx::query_as( + "SELECT 1 FROM filter_fired WHERE rule_id = ? AND message_key = ?", + ) + .bind(rule_id) + .bind(message_key) + .fetch_optional(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(found.is_some()) + } + + /// How many distinct (rule, message) fires the ledger records. + /// + /// Counterpart to [`Self::has_filter_fired`] for callers that need to assert + /// on the ledger as a whole rather than one entry. + pub async fn filter_fire_count(&self) -> Result { + let (count,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM filter_fired") + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + Ok(count) + } +``` + +Use whichever clock helper `db.rs` already uses for `created_at`/`matched_at` +rather than introducing a second one. + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cargo test -p nuncio-store db::tests::placements_primary_key_includes_uidvalidity db::tests::a_filter_fired_claim_is_won_exactly_once db::tests::bumping_the_identity_schema_rebuilds` +Expected: PASS. The rest of `nuncio-store` will not compile until Task 3. + +- [ ] **Step 7: Commit** + +```bash +git add crates/nuncio-store/src/db.rs +git commit -m "feat(store): add placements, fire-once claims, and bump identity schema" +``` + +--- + +## Task 3: Store writes — upsert message, upsert placement, first-write-wins + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (`save_email` → `upsert_message` + `upsert_placement`, `backfill_message_fts`) +- Test: `crates/nuncio-store/src/db.rs` inline tests + +**Interfaces:** +- Consumes: `Email` and `Placement` from Task 1; `placements` schema from Task 2. +- Produces: + - `DatabaseEngine::upsert_message(&self, email: &Email, source: IdentitySource) -> Result` — `true` when the row was newly inserted + - `DatabaseEngine::upsert_placement(&self, message_key: &str, placement: &Placement) -> Result` — `true` when the placement was newly inserted + - `DatabaseEngine::save_email_at(&self, email: &Email, source: IdentitySource, placement: &Placement) -> Result` where `pub struct SaveOutcome { pub message_is_new: bool, pub placement_is_new: bool }` + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn a_second_placement_does_not_overwrite_the_stored_body() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let email = nuncio_core::model::Email { + id: "key-1".into(), + account_id: "acct-1".into(), + subject: "Original".into(), + sender: "a@nuncio.mx".into(), + recipient: "b@nuncio.mx".into(), + received_at: 1_000, + body_plain: Some("the trusted body".into()), + body_html: None, + attachments: Vec::new(), + message_id: Some("a@b.example".into()), + content_hash: Some("1111".into()), + }; + let inbox = nuncio_core::model::Placement { + account_id: "acct-1".into(), + folder_id: "INBOX".into(), + uid_validity: "42".into(), + remote_id: "5".into(), + read: false, + }; + let outcome = engine + .save_email_at(&email, nuncio_core::model::IdentitySource::MessageIdContent, &inbox) + .await + .unwrap(); + assert!(outcome.message_is_new && outcome.placement_is_new); + + // The same key arriving from another folder, carrying a different body. + let impostor = nuncio_core::model::Email { subject: "Replaced".into(), body_plain: Some("attacker body".into()), ..email.clone() }; + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + let outcome = engine + .save_email_at(&impostor, nuncio_core::model::IdentitySource::MessageIdContent, &archive) + .await + .unwrap(); + assert!(!outcome.message_is_new, "identity already existed"); + assert!(outcome.placement_is_new, "but the Archive placement is new"); + + let stored = engine.get_message("key-1").await.unwrap(); + assert_eq!(stored.subject, "Original", "first write must win"); + assert_eq!(stored.body_plain.as_deref(), Some("the trusted body")); +} + +#[tokio::test] +async fn re_placing_the_same_message_leaves_exactly_one_fts_row() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let (rows,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(rows, 1, "the FTS index holds one row per message, not per placement"); +} + +#[tokio::test] +async fn read_state_is_tracked_per_placement() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), read: true, ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let placements = engine.placements_of("key-1").await.unwrap(); + let inbox_read = placements.iter().find(|p| p.folder_id == "INBOX").unwrap().read; + let archive_read = placements.iter().find(|p| p.folder_id == "Archive").unwrap().read; + assert!(!inbox_read); + assert!(archive_read, "IMAP \\Seen is per-mailbox, so the flags differ"); +} +``` + +Add this test helper next to the tests: + +```rust + fn sample_message_and_placement( + key: &str, + folder: &str, + uid: &str, + ) -> (nuncio_core::model::Email, nuncio_core::model::Placement) { + ( + nuncio_core::model::Email { + id: key.into(), + account_id: "acct-1".into(), + subject: "Subject".into(), + sender: "a@nuncio.mx".into(), + recipient: "b@nuncio.mx".into(), + received_at: 1_000, + body_plain: Some("body".into()), + body_html: None, + attachments: Vec::new(), + message_id: None, + content_hash: None, + }, + nuncio_core::model::Placement { + account_id: "acct-1".into(), + folder_id: folder.into(), + uid_validity: "42".into(), + remote_id: uid.into(), + read: false, + }, + ) + } +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-store db::tests::a_second_placement_does_not_overwrite` +Expected: FAIL — `no method save_email_at`. + +- [ ] **Step 3: Implement the write path** + +Replace `save_email` with: + +```rust +/// What a [`DatabaseEngine::save_email_at`] call actually changed. +/// +/// The two flags answer different questions and a caller usually needs both: +/// `message_is_new` gates work that should happen once per message (indexing, +/// side-effecting filter actions), `placement_is_new` gates work that should +/// happen once per mailbox occupancy (per-folder counters, per-placement +/// evaluation). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SaveOutcome { + /// The message identity had not been stored before this call. + pub message_is_new: bool, + /// This mailbox occupancy had not been stored before this call. + pub placement_is_new: bool, +} + +impl DatabaseEngine { + /// Persist a message's identity and content, and record that it occupies + /// `placement`, in one transaction. + /// + /// The message row is **first-write-wins**: a second sighting of the same + /// key -- which is the normal case for a message that also exists in another + /// folder -- leaves the stored subject and body untouched. That is what + /// keeps a message whose identity rests on `Message-ID` + content hash from + /// being rewritten by a later fetch claiming the same key, and it is why the + /// FTS row is written only on the insert that actually created the message. + /// + /// The placement row is upserted rather than ignored, because its read flag + /// is genuinely mutable: `\Seen` changes in the mailbox and the store must + /// follow it. + pub async fn save_email_at( + &self, + email: &nuncio_core::model::Email, + source: nuncio_core::model::IdentitySource, + placement: &nuncio_core::model::Placement, + ) -> Result { + // Encryption failures MUST surface before any SQL runs: a swallowed error here would + // otherwise leave a row with an empty/placeholder body column, indistinguishable from + // a genuinely empty body on read-back. + let enc_plain = email + .body_plain + .as_ref() + .map(|p| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, p)) + .transpose()?; + let enc_html = email + .body_html + .as_ref() + .map(|h| crate::cipher::PayloadCipher::encrypt_text_at_rest(&self.storage_key, h)) + .transpose()?; + + let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + + // Ask before writing. SQLite reports one row affected for both arms of + // an upsert, so after the fact there is no way to tell an inserted + // placement from an updated one -- and the caller needs that distinction + // to decide whether this is a first arrival worth filtering. + let (already_placed,): (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&placement.account_id) + .bind(&placement.folder_id) + .bind(&placement.uid_validity) + .bind(&placement.remote_id) + .fetch_one(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + let placement_is_new = already_placed == 0; + + let inserted = sqlx::query( + r#" + INSERT INTO messages + (message_key, account_id, subject, sender, recipient, received_at, + body_plain, body_html, message_id, content_hash, identity_source) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (message_key) DO NOTHING + "#, + ) + .bind(&email.id) + .bind(&email.account_id) + .bind(&email.subject) + .bind(&email.sender) + .bind(&email.recipient) + .bind(email.received_at) + .bind(&enc_plain) + .bind(&enc_html) + .bind(&email.message_id) + .bind(&email.content_hash) + .bind(source.as_str()) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + let message_is_new = inserted.rows_affected() == 1; + + // Index only on the insert that created the message. Re-indexing on + // every placement would either duplicate the row (FTS5 standalone tables + // have no upsert) or rewrite it with a body the first-write-wins rule + // just rejected. + if message_is_new { + sqlx::query( + "INSERT INTO messages_fts (message_key, subject, sender, body_plain) VALUES (?, ?, ?, ?)", + ) + .bind(&email.id) + .bind(&email.subject) + .bind(&email.sender) + .bind(email.body_plain.as_deref().unwrap_or("")) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + } + + sqlx::query( + r#" + INSERT INTO placements + (account_id, folder_id, uidvalidity, uid, message_key, read_flag) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (account_id, folder_id, uidvalidity, uid) + DO UPDATE SET message_key = excluded.message_key, read_flag = excluded.read_flag + "#, + ) + .bind(&placement.account_id) + .bind(&placement.folder_id) + .bind(&placement.uid_validity) + .bind(&placement.remote_id) + .bind(&email.id) + .bind(if placement.read { 1i64 } else { 0i64 }) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + tx.commit().await.map_err(DatabaseError::Query)?; + + Ok(SaveOutcome { message_is_new, placement_is_new }) + } +} +``` + +Add the placement reader used by the tests: + +```rust + /// Every mailbox this message currently occupies. + pub async fn placements_of( + &self, + message_key: &str, + ) -> Result, DatabaseError> { + let rows: Vec<(String, String, String, String, i64)> = sqlx::query_as( + "SELECT account_id, folder_id, uidvalidity, uid, read_flag + FROM placements WHERE message_key = ? ORDER BY folder_id ASC", + ) + .bind(message_key) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(account_id, folder_id, uid_validity, remote_id, read_flag)| { + nuncio_core::model::Placement { + account_id, + folder_id, + uid_validity, + remote_id, + read: read_flag != 0, + } + }) + .collect()) + } +``` + +Update `backfill_message_fts` to select `message_key` instead of `id` and to +join nothing — it stays a `messages`-only backfill. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p nuncio-store db::tests` +Expected: the three new tests PASS. Pre-existing `save_email` tests fail to compile — fix them in Task 4 as part of the read-path migration. + +- [ ] **Step 5: Commit** + +```bash +git add crates/nuncio-store/src/db.rs +git commit -m "feat(store): write message identity and placement separately" +``` + +--- + +## Task 4: Store reads — placement-aware queries and folder derivation + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (`get_message`, `list_messages`, `list_messages_page`, `message_ids_in_folder`, `set_message_read`, `list_folders`, `list_folders_page`, unread count at `db.rs:471`) +- Modify: `crates/nuncio-store/src/search.rs` +- Test: `crates/nuncio-store/src/db.rs` inline tests + +**Interfaces:** +- Consumes: `placements_of`, `SaveOutcome` from Task 3. +- Produces: + - `get_message(&self, message_key: &str) -> Result` (identity only, no folder scalars) + - `list_messages(&self, account_id: &str, folder_id: &str, limit: usize) -> Result, DatabaseError>` + - `existing_placements(&self, keys: &[PlacementKey]) -> Result, DatabaseError>` with: + + ```rust + /// The four coordinates that address one mailbox occupancy. + /// + /// `Ord` is derived because the sync path sorts and dedups batches of these + /// before deleting; `Hash`/`Eq` because it is also the key of the + /// already-present set used to classify a fetched chunk. + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] + pub struct PlacementKey { + pub account_id: String, + pub folder_id: String, + pub uid_validity: String, + pub remote_id: String, + } + ``` +- Test helper: the tests below reuse `sample_message_and_placement`, added to `db.rs`'s test module in Task 3 Step 1. It returns an `(Email, Placement)` pair for `account_id = "acct-1"`, `uid_validity = "42"`. + - `set_placement_read(&self, key: &PlacementKey, read: bool) -> Result<(), DatabaseError>` + - `list_folders(&self, account_id: &str) -> Result, DatabaseError>` — now account-scoped + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn folders_are_derived_from_placements_and_scoped_per_account() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + for (account, folder, uid) in [("acct-1", "INBOX", "1"), ("acct-1", "Archive", "2"), ("acct-2", "INBOX", "1")] { + let (mut email, mut placement) = sample_message_and_placement(&format!("key-{account}-{folder}"), folder, uid); + email.account_id = account.into(); + placement.account_id = account.into(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &placement).await.unwrap(); + } + + let folders = engine.list_folders("acct-1").await.unwrap(); + let names: Vec<&str> = folders.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(names, vec!["Archive", "INBOX"]); + assert!( + !folders.iter().any(|f| f.total_messages > 1), + "acct-2's INBOX must not be folded into acct-1's" + ); +} + +#[tokio::test] +async fn one_message_in_two_folders_counts_once_in_each() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let folders = engine.list_folders("acct-1").await.unwrap(); + for folder in &folders { + assert_eq!(folder.total_messages, 1, "{} should hold the message once", folder.id); + } + let (messages,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages").fetch_one(engine.pool()).await.unwrap(); + assert_eq!(messages, 1, "but it is still one message"); +} + +#[tokio::test] +async fn existing_placements_reports_the_occupancy_not_the_message() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + + let in_inbox = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), uid_validity: "42".into(), remote_id: "5".into() }; + let in_archive = PlacementKey { folder_id: "Archive".into(), remote_id: "9".into(), ..in_inbox.clone() }; + + let found = engine.existing_placements(&[in_inbox.clone(), in_archive.clone()]).await.unwrap(); + assert!(found.contains(&in_inbox)); + assert!( + !found.contains(&in_archive), + "the same message arriving in a new folder is a new placement and must be seen as such" + ); +} + +#[tokio::test] +async fn marking_read_in_one_folder_leaves_the_other_unread() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let inbox_key = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), uid_validity: "42".into(), remote_id: "5".into() }; + engine.set_placement_read(&inbox_key, true).await.unwrap(); + + let placements = engine.placements_of("key-1").await.unwrap(); + assert!(placements.iter().find(|p| p.folder_id == "INBOX").unwrap().read); + assert!(!placements.iter().find(|p| p.folder_id == "Archive").unwrap().read); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-store db::tests::folders_are_derived_from_placements` +Expected: FAIL — `list_folders` takes no arguments; `PlacementKey` undefined. + +- [ ] **Step 3: Implement the read path** + +`get_message` — drop the folder scalars from both the `SELECT` and the +constructed `Email`: + +```rust + /// Retrieve a message's identity and content by its key. + /// + /// Returns the message alone. Where it sits, and whether it is read there, + /// are per-mailbox facts -- ask [`Self::placements_of`] for those. + pub async fn get_message( + &self, + message_key: &str, + ) -> Result { + let row: (String, String, String, String, String, i64, Option, Option, Option, Option) = + sqlx::query_as( + r#" + SELECT message_key, account_id, subject, sender, recipient, received_at, + body_plain, body_html, message_id, content_hash + FROM messages + WHERE message_key = ? + "#, + ) + .bind(message_key) + .fetch_one(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + + Ok(nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, + }) + } +``` + +`list_messages` and `list_messages_page` join through `placements` and return +pairs. Both gain an `account_id` parameter for the same reason +`message_ids_in_folder` already has one — folder ids are not globally unique: + +```rust + /// Messages occupying one folder of one account, newest first, paired with + /// the placement they were found through. + /// + /// The placement travels with the message because the caller almost always + /// needs the per-mailbox facts -- the read flag and the addressing UID -- + /// and re-deriving them would mean a second query per row. + #[allow(clippy::type_complexity)] + pub async fn list_messages( + &self, + account_id: &str, + folder_id: &str, + limit: usize, + ) -> Result, DatabaseError> { + let rows: Vec<( + String, + String, + String, + String, + String, + i64, + Option, + Option, + Option, + Option, + String, + String, + i64, + )> = sqlx::query_as( + r#" + SELECT m.message_key, m.account_id, m.subject, m.sender, m.recipient, + m.received_at, m.body_plain, m.body_html, m.message_id, m.content_hash, + p.uidvalidity, p.uid, p.read_flag + FROM messages m + JOIN placements p ON p.message_key = m.message_key + WHERE p.account_id = ? AND p.folder_id = ? + ORDER BY m.received_at DESC + LIMIT ? + "#, + ) + .bind(account_id) + .bind(folder_id) + .bind(limit as i64) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let dec_plain = row + .6 + .map(|p| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &p)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + let dec_html = row + .7 + .map(|h| crate::cipher::PayloadCipher::decrypt_text_at_rest(&self.storage_key, &h)) + .transpose() + .map_err(|e| DatabaseError::Decryption(e.to_string()))?; + + out.push(( + nuncio_core::model::Email { + id: row.0, + account_id: row.1, + subject: row.2, + sender: row.3, + recipient: row.4, + received_at: row.5, + body_plain: dec_plain, + body_html: dec_html, + attachments: Vec::new(), + message_id: row.8, + content_hash: row.9, + }, + nuncio_core::model::Placement { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity: row.10, + remote_id: row.11, + read: row.12 != 0, + }, + )); + } + Ok(out) + } +``` + +`list_messages_page` takes the same join and adds the existing keyset cursor +over `m.received_at`, fetching `page_size + 1` rows to detect a following page +exactly as it does today. + +`message_ids_in_folder` selects from `placements` and returns placement keys, since +that is what a UID-set diff subtracts: + +```rust + /// Every placement currently stored for one folder of one account. + /// + /// The local half of a UID-set diff. This returns placements rather than + /// message keys because what a server stops reporting is an occupancy, not a + /// message: the same mail may still exist in another folder and must survive + /// there. + pub async fn placements_in_folder( + &self, + account_id: &str, + folder_id: &str, + ) -> Result, DatabaseError> { + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT uidvalidity, uid FROM placements WHERE account_id = ? AND folder_id = ?", + ) + .bind(account_id) + .bind(folder_id) + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(uid_validity, remote_id)| PlacementKey { + account_id: account_id.to_string(), + folder_id: folder_id.to_string(), + uid_validity, + remote_id, + }) + .collect()) + } +``` + +`existing_placements` mirrors `existing_message_ids`, batching with a tuple +`IN` clause: + +```rust + /// The subset of `keys` already stored, in one round trip. + /// + /// SQLite supports row-value `IN` lists, so the whole batch is one query + /// rather than one existence check per placement. An empty slice + /// short-circuits, since `IN ()` is not valid SQL. + pub async fn existing_placements( + &self, + keys: &[PlacementKey], + ) -> Result, DatabaseError> { + if keys.is_empty() { + return Ok(std::collections::HashSet::new()); + } + + let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new( + "SELECT account_id, folder_id, uidvalidity, uid FROM placements + WHERE (account_id, folder_id, uidvalidity, uid) IN (", + ); + let mut separated = builder.separated(", "); + for key in keys { + separated.push("("); + separated.push_bind_unseparated(key.account_id.clone()); + separated.push_unseparated(", "); + separated.push_bind_unseparated(key.folder_id.clone()); + separated.push_unseparated(", "); + separated.push_bind_unseparated(key.uid_validity.clone()); + separated.push_unseparated(", "); + separated.push_bind_unseparated(key.remote_id.clone()); + separated.push_unseparated(")"); + } + builder.push(")"); + + let rows: Vec<(String, String, String, String)> = builder + .build_query_as() + .fetch_all(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + Ok(rows + .into_iter() + .map(|(account_id, folder_id, uid_validity, remote_id)| PlacementKey { + account_id, + folder_id, + uid_validity, + remote_id, + }) + .collect()) + } +``` + +> **Implementer note:** if `QueryBuilder`'s separated API makes the row-value +> list awkward, build the `IN` list with a plain loop over +> `builder.push("(").push_bind(..)` calls instead — the shape of the SQL is what +> matters, not the builder idiom. Verify the generated SQL against the test +> before moving on. + +`set_message_read` becomes `set_placement_read`, updating `placements` by its +full primary key and keeping the `RowNotFound` contract: + +```rust + /// Update one placement's read flag in place. + /// + /// Scoped to a single mailbox occupancy because IMAP `\Seen` is per-mailbox: + /// a message read in `INBOX` is not thereby read in `Archive`. Returns + /// `DatabaseError::Query(sqlx::Error::RowNotFound)` when no such placement + /// exists, so callers can still distinguish "flag flipped" from "never + /// there" rather than silently succeeding on a no-op update. + pub async fn set_placement_read( + &self, + key: &PlacementKey, + read: bool, + ) -> Result<(), DatabaseError> { + let result = sqlx::query( + "UPDATE placements SET read_flag = ? + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(if read { 1i64 } else { 0i64 }) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .execute(&self.pool) + .await + .map_err(DatabaseError::Query)?; + + if result.rows_affected() == 0 { + return Err(DatabaseError::Query(sqlx::Error::RowNotFound)); + } + Ok(()) + } +``` + +`list_folders` / `list_folders_page` group over `placements` and take an +`account_id`: + +```sql + SELECT folder_id, COUNT(*) as total, + SUM(CASE WHEN read_flag = 0 THEN 1 ELSE 0 END) as unread + FROM placements + WHERE account_id = ? + GROUP BY folder_id + ORDER BY folder_id ASC +``` + +Add to `list_folders`'s doc comment: + +```rust + /// Folders are derived from the placements that reference them rather than + /// stored in their own table, so a folder exists exactly as long as it holds + /// mail. Scoping by account is required, not cosmetic: folder ids are not + /// globally unique -- every account has an `INBOX` -- and grouping across + /// accounts would report one merged folder holding both accounts' mail. +``` + +Fix the unread count at `db.rs:471` to read +`SELECT COUNT(*) FROM placements WHERE read_flag = 0`. + +In `search.rs`, change the FTS hit column from `id` to `message_key` and have +hits resolve through `get_message`; a hit names a message, not a placement. + +- [ ] **Step 4: Migrate the existing tests in `db.rs`** + +Every pre-existing test constructing an `Email` with `folder_id` / `remote_id` / +`uid_validity` / `read` must move those into a `Placement` and call +`save_email_at`. Work through the compiler errors; there is no behaviour +decision to make in this step. Two that need real thought rather than +mechanical translation: + +- the corrupted-ciphertext test around `db.rs:3276` — it must still fail closed + on `get_message`, unchanged in intent +- `existing_message_ids_returns_only_the_ids_already_persisted` around + `db.rs:4649` — rewrite as the `existing_placements` test already added above, + and delete the original + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p nuncio-store` +Expected: PASS, whole crate. + +- [ ] **Step 6: Commit** + +```bash +git add crates/nuncio-store/src/db.rs crates/nuncio-store/src/search.rs +git commit -m "feat(store): read messages through their placements" +``` + +--- + +## Task 5: Store deletes — the FTS plaintext-orphan rule + +**Files:** +- Modify: `crates/nuncio-store/src/db.rs` (`delete_messages` → `delete_placements`) +- Test: `crates/nuncio-store/src/db.rs` inline tests + +**Interfaces:** +- Consumes: `PlacementKey` from Task 4; `save_email_at` / `placements_of` from Task 3. +- Produces: `delete_placements(&self, keys: &[PlacementKey]) -> Result` with `pub struct DeleteOutcome { pub placements_removed: u64, pub messages_reaped: u64 }` +- Test helper: `sample_message_and_placement`, added to `db.rs`'s test module in Task 3 Step 1 (returns an `(Email, Placement)` pair for `account_id = "acct-1"`, `uid_validity = "42"`). + +This task carries the non-negotiable that is easiest to get backwards, so its +tests are written to fail loudly in **both** directions. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn dropping_one_placement_keeps_the_body_searchable_from_the_other() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + let archive = nuncio_core::model::Placement { folder_id: "Archive".into(), remote_id: "9".into(), ..inbox.clone() }; + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &archive).await.unwrap(); + + let inbox_key = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), uid_validity: "42".into(), remote_id: "5".into() }; + let outcome = engine.delete_placements(&[inbox_key]).await.unwrap(); + assert_eq!(outcome.placements_removed, 1); + assert_eq!(outcome.messages_reaped, 0, "the message still lives in Archive"); + + engine.get_message("key-1").await.expect("the message must survive"); + let (fts,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!(fts, 1, "search must still find a message that still exists"); +} + +#[tokio::test] +async fn dropping_the_last_placement_reaps_the_message_and_its_plaintext_index() { + let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let (email, inbox) = sample_message_and_placement("key-1", "INBOX", "5"); + engine.save_email_at(&email, nuncio_core::model::IdentitySource::EmailId, &inbox).await.unwrap(); + + let inbox_key = PlacementKey { account_id: "acct-1".into(), folder_id: "INBOX".into(), uid_validity: "42".into(), remote_id: "5".into() }; + let outcome = engine.delete_placements(&[inbox_key]).await.unwrap(); + assert_eq!(outcome.messages_reaped, 1); + + engine.get_message("key-1").await.expect_err("the message is gone"); + let (fts,): (i64,) = sqlx::query_as("SELECT COUNT(*) FROM messages_fts WHERE message_key = 'key-1'") + .fetch_one(engine.pool()) + .await + .unwrap(); + assert_eq!( + fts, 0, + "an orphaned FTS row would leave the plaintext body readable to anyone \ + with filesystem access after the message itself was deleted" + ); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-store db::tests::dropping_one_placement db::tests::dropping_the_last_placement` +Expected: FAIL — `no method delete_placements`. + +- [ ] **Step 3: Implement the delete path** + +```rust +/// What a [`DatabaseEngine::delete_placements`] call removed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DeleteOutcome { + /// Mailbox occupancies removed. + pub placements_removed: u64, + /// Messages that lost their last placement and were removed with it. + pub messages_reaped: u64, +} + +impl DatabaseEngine { + /// Remove mailbox occupancies, reaping any message left with none. + /// + /// A server that stops reporting a UID has told us the message left *that + /// mailbox*, not that it ceased to exist -- a move reports exactly this in + /// the source folder while the message continues in the destination. So the + /// placement goes unconditionally and the message goes only when its last + /// placement does. + /// + /// The reap is what keeps the plaintext FTS index honest. `messages_fts` + /// holds decrypted bodies keyed on `message_key`, reaped by the + /// `messages_ad` trigger on `messages`. Deleting the message row is + /// therefore the only thing that clears the index: skip the reap and every + /// fully-deleted message leaves its body readable to anyone with filesystem + /// access to the database. + pub async fn delete_placements( + &self, + keys: &[PlacementKey], + ) -> Result { + if keys.is_empty() { + return Ok(DeleteOutcome { placements_removed: 0, messages_reaped: 0 }); + } + + let mut tx = self.pool.begin().await.map_err(DatabaseError::Query)?; + let mut placements_removed = 0u64; + let mut touched: Vec = Vec::with_capacity(keys.len()); + + for key in keys { + // Remember which message each placement pointed at before removing + // it -- afterwards there is nothing left to join through. + let owner: Option<(String,)> = sqlx::query_as( + "SELECT message_key FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .fetch_optional(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + let Some((message_key,)) = owner else { continue }; + + let result = sqlx::query( + "DELETE FROM placements + WHERE account_id = ? AND folder_id = ? AND uidvalidity = ? AND uid = ?", + ) + .bind(&key.account_id) + .bind(&key.folder_id) + .bind(&key.uid_validity) + .bind(&key.remote_id) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + + placements_removed += result.rows_affected(); + touched.push(message_key); + } + + touched.sort_unstable(); + touched.dedup(); + + let mut messages_reaped = 0u64; + for message_key in touched { + // Reap only where nothing is left pointing at the message. The + // `messages_ad` trigger clears the FTS row as a consequence. + let result = sqlx::query( + "DELETE FROM messages WHERE message_key = ? + AND NOT EXISTS (SELECT 1 FROM placements WHERE message_key = ?)", + ) + .bind(&message_key) + .bind(&message_key) + .execute(&mut *tx) + .await + .map_err(DatabaseError::Query)?; + messages_reaped += result.rows_affected(); + } + + tx.commit().await.map_err(DatabaseError::Query)?; + Ok(DeleteOutcome { placements_removed, messages_reaped }) + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p nuncio-store` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/nuncio-store/src/db.rs +git commit -m "feat(store): reap a message only when its last placement goes" +``` + +--- + +## Task 6: Filter evaluation becomes per-placement + +**Files:** +- Modify: `crates/nuncio-filter/src/engine.rs` +- Modify: `crates/nuncio-filter/src/validator.rs` (comments only) +- Test: `crates/nuncio-filter/src/engine.rs` inline tests + +**Interfaces:** +- Consumes: `Email`, `Placement` from Task 1. +- Produces: + - `pub struct PlacedEmail<'a> { pub email: &'a Email, pub placement: &'a Placement }` + - `CompiledFilter::evaluate_condition(&self, placed: PlacedEmail<'_>) -> bool` + - `FilterEngine::evaluate(&self, placed: PlacedEmail<'_>) -> Vec<(FilterRule, Vec)>` + - `FilterEngine::evaluate_with_timeout(&self, placed: PlacedEmail<'_>, timeout: Duration) -> Vec<(FilterRule, Vec)>` + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn a_folder_condition_matches_only_the_placement_it_is_evaluated_against() { + let engine = FilterEngine::new(vec![rule_where_folder_is("INBOX")]).unwrap(); + let email = sample_email("key-1"); + let inbox = sample_placement("INBOX", "5"); + let archive = sample_placement("Archive", "9"); + + assert_eq!( + engine.evaluate(PlacedEmail { email: &email, placement: &inbox }).len(), + 1, + "the INBOX placement matches" + ); + assert_eq!( + engine.evaluate(PlacedEmail { email: &email, placement: &archive }).len(), + 0, + "the same message's Archive placement does not" + ); +} + +#[test] +fn an_account_condition_reads_the_placement_account() { + let engine = FilterEngine::new(vec![rule_where_account_is("acct-1")]).unwrap(); + let email = sample_email("key-1"); + let mine = sample_placement("INBOX", "5"); + let theirs = Placement { account_id: "acct-2".into(), ..sample_placement("INBOX", "5") }; + + assert_eq!(engine.evaluate(PlacedEmail { email: &email, placement: &mine }).len(), 1); + assert_eq!(engine.evaluate(PlacedEmail { email: &email, placement: &theirs }).len(), 0); +} +``` + +Add the test helpers `sample_email`, `sample_placement`, +`rule_where_folder_is`, `rule_where_account_is` next to the existing engine +test helpers, mirroring the rule-construction style already used at +`engine.rs:650`. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nuncio-filter engine::tests::a_folder_condition_matches_only` +Expected: FAIL — `cannot find type PlacedEmail`. + +- [ ] **Step 3: Implement per-placement evaluation** + +Add near the top of `engine.rs`: + +```rust +/// A message together with the one mailbox occupancy being evaluated. +/// +/// `WHERE FOLDER = 'INBOX'` has no answer for a message that sits in three +/// folders at once, so evaluation is defined against a single placement and the +/// caller fans out over however many the message has. Borrowing both halves +/// keeps that fan-out free of clones. +#[derive(Debug, Clone, Copy)] +pub struct PlacedEmail<'a> { + /// Identity and content -- the same for every placement. + pub email: &'a Email, + /// The occupancy this evaluation is about. + pub placement: &'a Placement, +} +``` + +Thread `PlacedEmail` through `evaluate_condition`, `eval_node`, and +`eval_leaf`, replacing the `email: &Email` parameter. In `eval_leaf`, the +content fields read `placed.email`, and the two placement-scoped fields become: + +```rust + FilterField::Folder => Self::eval_string_op( + &placed.placement.folder_id, + &leaf.operator, + &leaf.value, + regexes, + ), + FilterField::Account => Self::eval_string_op( + &placed.placement.account_id, + &leaf.operator, + &leaf.value, + regexes, + ), +``` + +In `evaluate` and `evaluate_with_timeout`, the account-scope gate reads the +placement: + +```rust + if filter.rule.matches_account(&placed.placement.account_id) && filter.evaluate_condition(placed) { +``` + +`evaluate_with_timeout` moves owned values into `spawn_blocking`, so it needs +owned clones inside the loop: + +```rust + let filter_for_task = filter.clone(); + let email_for_task = placed.email.clone(); + let placement_for_task = placed.placement.clone(); + let eval_task = tokio::task::spawn_blocking(move || { + filter_for_task.evaluate_condition(PlacedEmail { + email: &email_for_task, + placement: &placement_for_task, + }) + }); +``` + +Update `evaluate`'s doc comment to record the new contract: + +```rust + /// Evaluate one placement of a message, returning the matching rules' actions. + /// + /// Defined per placement, not per message: `FOLDER` and `ACCOUNT` are + /// properties of where the message sits, and a message can sit in several + /// places at once. Callers holding a multi-placement message evaluate each + /// placement and are responsible for not acting twice on one message -- see + /// the fire-once claim in the store. + /// + /// A rule only ever fires for the account it was created against + /// (`FilterRule::target_account`, set via `ON ACCOUNT` or `*` for all + /// accounts) — condition matching alone is not account-scoped, so this + /// check must happen before `evaluate_condition` runs. +``` + +In `validator.rs`, correct the comment above the `Folder | Account` arm at +line ~99 to say these validate against a placement's coordinates. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p nuncio-filter` +Expected: PASS. Existing engine tests need their `evaluate(&email)` calls +rewritten as `evaluate(PlacedEmail { .. })`; that is mechanical. + +- [ ] **Step 5: Commit** + +```bash +git add crates/nuncio-filter/src/engine.rs crates/nuncio-filter/src/validator.rs +git commit -m "feat(filter): evaluate rules against a placement, not a message" +``` + +--- + +## Task 7: Mail backends emit identity and placement + +**Files:** +- Modify: `crates/nuncio-mail/src/imap.rs` (~lines 1186, 1208, 1244–1310) +- Modify: `crates/nuncio-mail/src/jmap.rs` (~line 401) +- Modify: `crates/nuncio-mail/src/mock.rs` +- Modify: `crates/nuncio-mail/src/backend.rs` (`FolderChanges`) +- Test: `crates/nuncio-mail/tests/wiremock_jmap_test.rs`, inline `imap.rs` tests + +**Interfaces:** +- Consumes: `derive_message_key`, `RemoteIdentity`, `Placement`, `MessageIdentity`. +- Produces: `pub struct PlacedMessage { pub email: Email, pub source: IdentitySource, pub placement: Placement }`; `FolderChanges.upserts: Vec`; `FolderChanges.removals: Vec`; `FolderChanges.present: Option>`. + +- [ ] **Step 1: Write the failing test** + +```rust +#[tokio::test] +async fn a_message_fetched_from_two_folders_keeps_one_identity() { + // Both folders return the same Message-ID and the same octets, so the + // content tier must produce one key from two different UIDs. + let raw = b"Message-ID: \r\nSubject: Hi\r\n\r\nbody"; + let hash = Email::content_hash_of(raw); + let inbox = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: None, message_id: Some("a@b.example"), content_hash: Some(&hash) }, + "INBOX", "42", "5", + ); + let archive = Email::derive_message_key( + "acct-1", + RemoteIdentity { email_id: None, gm_msgid: None, message_id: Some("a@b.example"), content_hash: Some(&hash) }, + "Archive", "77", "9", + ); + assert_eq!(inbox.key, archive.key); + assert_eq!(inbox.source, IdentitySource::MessageIdContent); +} +``` + +Place this in `crates/nuncio-mail/src/imap.rs`'s test module so it guards the +backend's own use of the precedence, not just the core function. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p nuncio-mail a_message_fetched_from_two_folders` +Expected: FAIL to compile until `nuncio-mail` is migrated. + +- [ ] **Step 3: Migrate the IMAP backend** + +At each of `imap.rs:1186`, `1208`, and `1278`, the call currently reads +`Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id)`. +Replace with a helper on the backend: + +```rust + /// Build the identity and placement for one fetched message. + /// + /// `EMAILID` and `X-GM-MSGID` are only present when the server advertised + /// `OBJECTID` / `X-GM-EXT-1` and the FETCH asked for them; when neither is + /// available the pair of `Message-ID` and content hash carries the identity, + /// and only if both are missing does this fall back to the folder-scoped + /// surrogate -- which is the one tier that does not survive a move. + fn place( + &self, + folder_id: &str, + uid_validity: &str, + remote_id: &str, + email_id: Option<&str>, + gm_msgid: Option<&str>, + message_id: Option<&str>, + content_hash: Option<&str>, + read: bool, + ) -> (MessageIdentity, Placement) { + let identity = Email::derive_message_key( + &self.account_id, + RemoteIdentity { email_id, gm_msgid, message_id, content_hash }, + folder_id, + uid_validity, + remote_id, + ); + let placement = Placement { + account_id: self.account_id.clone(), + folder_id: folder_id.to_string(), + uid_validity: uid_validity.to_string(), + remote_id: remote_id.to_string(), + read, + }; + (identity, placement) + } +``` + +`EMAILID` and `X-GM-MSGID` are not fetched today. Pass `None` for both and +leave a comment saying so — wiring the FETCH items is #395/#396 territory and +adding it here would widen this change without a test server that serves them: + +```rust + // EMAILID (RFC 8474) and X-GM-MSGID are not requested by the current + // FETCH, so identity rests on the Message-ID/content pair where the + // message carried a Message-ID and on the folder-scoped surrogate + // otherwise. Adding those FETCH items lifts existing rows to a stronger + // tier on the next resync at no cost to this code path. +``` + +Change `FolderChanges` in `backend.rs`: + +```rust +/// One message as a backend surfaced it: identity, content, and the mailbox +/// occupancy it was found in. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PlacedMessage { + /// Identity and content. `email.id` is the derived message key. + pub email: Email, + /// Which precedence tier produced `email.id`. + pub source: IdentitySource, + /// The occupancy this fetch found it in. + pub placement: Placement, +} +``` + +and retype `upserts`, `removals`, and `present` as noted in the Interfaces +block above. `removals` and `present` become placement keys because what a +folder stops reporting is an occupancy. + +- [ ] **Step 4: Migrate the JMAP backend and the mock** + +At `jmap.rs:401`, JMAP has no UIDVALIDITY, so it keeps its existing sentinel as +`uid_validity` and uses the JMAP object id as `remote_id`. JMAP's object id is +server-assigned and account-stable, so pass it as `email_id` — that is exactly +what the top tier is for: + +```rust + // A JMAP Email id is server-assigned and stable across mailboxes within + // the account, which is the same guarantee RFC 8474 EMAILID gives, so it + // enters at the top tier rather than through the surrogate. +``` + +In `mock.rs`, have `MockMailBackend` emit `PlacedMessage` values so daemon +tests can construct multi-placement scenarios. + +Leave the JMAP `present: None` gap exactly as it is — JMAP still accumulates +ghosts until `Email/changes` lands, and pretending otherwise here would delete +mail. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p nuncio-mail` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/nuncio-mail/ +git commit -m "feat(mail): derive message identity and emit placements" +``` + +--- + +## Task 8: Sync wiring — placement-aware fan-out and the fire-once claim + +**Files:** +- Modify: `crates/nunciod/src/sync.rs` (`fetch_and_persist`, `apply_filter_actions`) +- Test: `crates/nunciod/src/sync.rs` inline tests, `crates/nunciod/tests/filters_triage_e2e_test.rs` + +**Interfaces:** +- Consumes: `save_email_at`, `existing_placements`, `delete_placements`, `claim_filter_fire`, `PlacedEmail`, `PlacedMessage`. +- Produces: no new public API; `fetch_and_persist` keeps its signature and return type. + +This task carries the bug the issue calls out by name: under a message-scoped +"is new" test, a message arriving in a second folder looks like it was already +seen and **silently never fires any filter**. + +- [ ] **Step 1: Write the failing tests** + +```rust +#[tokio::test] +async fn a_message_arriving_in_a_second_folder_still_gets_evaluated() { + // The regression this guards: with identity now folder-independent, a + // message already stored from INBOX has a known message_key, so a + // message-scoped "already present" test would classify its arrival in + // Archive as old and skip filtering it entirely. + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let backend = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]); + let engine = FilterEngine::new(vec![rule_matching_everything()]).unwrap(); + + let processed = fetch_and_persist(&db, &EventBus::new(), &backend, &engine, Some("acct-1")) + .await + .unwrap(); + + assert_eq!(processed, 2, "both placements are processed"); + let placements = db.placements_of(&expected_key()).await.unwrap(); + assert_eq!(placements.len(), 2); +} + +#[tokio::test] +async fn a_rule_fires_once_for_a_message_that_lands_in_two_folders() { + // MOVE and FLAG converge under repetition; FORWARD and CALL WEBHOOK do not. + // Two placements of one message must not page an on-call engineer twice. + let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap(); + let backend = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]); + let engine = FilterEngine::new(vec![rule_matching_everything()]).unwrap(); + + fetch_and_persist(&db, &EventBus::new(), &backend, &engine, Some("acct-1")) + .await + .unwrap(); + + assert_eq!( + db.filter_fire_count().await.unwrap(), + 1, + "one claim, therefore one set of actions" + ); + assert!(db.has_filter_fired(&rule_matching_everything().id, &expected_key()).await.unwrap()); +} +``` + +Add `MockMailBackend::with_same_message_in(folders: &[&str])` to `mock.rs`, +returning the same `email` (same derived key) under a different +`folder_id`/`remote_id` per folder. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p nunciod sync::tests::a_message_arriving_in_a_second_folder` +Expected: FAIL to compile until the sync path is migrated. + +- [ ] **Step 3: Rewrite the classification and fan-out** + +In `fetch_and_persist`, replace the chunk-classification block: + +```rust + // One round trip classifies the whole chunk. This asks about + // *placements*, not messages: identity is folder-independent now, so a + // message already stored from another folder is still a first arrival + // here, and testing the message key instead would classify it as seen + // and skip filtering it entirely. + let chunk_keys: Vec = placed + .iter() + .map(|p| PlacementKey { + account_id: p.placement.account_id.clone(), + folder_id: p.placement.folder_id.clone(), + uid_validity: p.placement.uid_validity.clone(), + remote_id: p.placement.remote_id.clone(), + }) + .collect(); + let already_present = db.existing_placements(&chunk_keys).await?; + let mut seen_this_pass = std::collections::HashSet::new(); + + for message in placed { + let key = PlacementKey { + account_id: message.placement.account_id.clone(), + folder_id: message.placement.folder_id.clone(), + uid_validity: message.placement.uid_validity.clone(), + remote_id: message.placement.remote_id.clone(), + }; + let is_new = !already_present.contains(&key) && seen_this_pass.insert(key); + db.save_email_at(&message.email, message.source, &message.placement).await?; + if is_new { + apply_filter_actions(db, filter_engine, &message.email, &message.placement).await; + } + synced += 1; + } +``` + +Change `apply_filter_actions` to take the placement, evaluate against it, and +gate every action behind the claim: + +```rust +async fn apply_filter_actions( + db: &nuncio_store::db::DatabaseEngine, + filter_engine: &FilterEngine, + email: &nuncio_core::model::Email, + placement: &nuncio_core::model::Placement, +) { + let placed = nuncio_filter::PlacedEmail { email, placement }; + for (rule, actions) in filter_engine.evaluate(placed) { + // Claim before acting. The claim is keyed on message identity, so the + // same mail arriving in a second folder -- an ordinary MOVE, or a Gmail + // label -- does not run the actions again. That is load-bearing for + // FORWARD and CALL WEBHOOK, which land on third parties and have no + // shared state to converge against; MOVE and FLAG would survive the + // repetition, but there is no reason to pay for it. + match db.claim_filter_fire(&rule.id, &email.id).await { + Ok(true) => {} + Ok(false) => { + tracing::debug!( + rule_id = %rule.id, + message_key = %email.id, + folder_id = %placement.folder_id, + "rule already fired for this message; skipping actions for this placement" + ); + continue; + } + Err(e) => { + // Fail closed. An unverifiable claim must not become a licence + // to act: a lost claim check on a FORWARD is a duplicate mail + // the recipient cannot un-receive. + tracing::error!( + rule_id = %rule.id, + message_key = %email.id, + error = %e, + "could not claim the filter fire; skipping actions" + ); + continue; + } + } + + // ... existing per-action dispatch, unchanged apart from taking + // `placement` where it previously read `email.folder_id` / + // `email.remote_id` / `email.uid_validity` + } +} +``` + +Every `RemoteMutationSpec` built here now takes its `folder_id`, `remote_id`, +and `uid_validity` from `placement`, and its `message_id` from `email.id`. +`MARK READ` names the placement it was evaluated against — which is what makes +it well-defined at all, since `\Seen` is per-mailbox. + +Update the removals block to work in placements: + +```rust + if let Some(present) = changes.present { + let present: std::collections::HashSet = present.into_iter().collect(); + let stored = db.placements_in_folder(acct, &folder.id).await?; + gone.extend(stored.into_iter().filter(|key| !present.contains(key))); + } + + gone.sort_unstable(); + gone.dedup(); + if !gone.is_empty() { + let outcome = db.delete_placements(&gone).await?; + tracing::info!( + account_id = %acct, + folder_id = %folder.id, + placements_removed = outcome.placements_removed, + messages_reaped = outcome.messages_reaped, + "removed placements that are no longer on the server" + ); + } +``` + +`PlacementKey` needs `Ord` for the `sort_unstable`/`dedup` pair; derive +`PartialOrd, Ord` on it in Task 4. + +Rewrite the doc comment on `fetch_and_persist` so it describes placement +classification rather than message classification — the current text at +`sync.rs:200–228` explains the old model in detail and would be actively +misleading left as-is. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p nunciod` +Expected: PASS, including `filters_triage_e2e_test`. + +- [ ] **Step 5: Commit** + +```bash +git add crates/nunciod/src/sync.rs crates/nuncio-mail/src/mock.rs +git commit -m "feat(sync): classify arrivals per placement and claim fires once" +``` + +--- + +## Task 9: Daemon surface — gRPC, CLI, and the proto comment + +**Files:** +- Modify: `crates/nunciod/src/grpc.rs` +- Modify: `crates/nuncio-proto/proto/nuncio/v1/nuncio.proto` (comment only) +- Modify: `crates/nuncio-cli/src/runner.rs` (only where it fails to compile) +- Test: `crates/nunciod/tests/spine_e2e_test.rs`, `crates/nunciod/tests/typed_errors_e2e_test.rs` + +**Interfaces:** +- Consumes: everything above. +- Produces: no wire-schema change. `Message.folder_id` and `Message.read` are populated from the placement the message was reached through. + +- [ ] **Step 1: Write the failing test** + +```rust +#[tokio::test] +async fn list_messages_reports_the_folder_it_was_asked_about() { + // A message in two folders appears in both listings, each time carrying the + // placement that listing is about -- not an arbitrary one of the two. + let harness = SpineHarness::start().await; + harness.seed_message_in(&["INBOX", "Archive"]).await; + + for folder in ["INBOX", "Archive"] { + let listed = harness + .client() + .list_messages(ListMessagesRequest { folder_id: folder.into(), ..Default::default() }) + .await + .unwrap() + .into_inner(); + assert_eq!(listed.messages.len(), 1); + assert_eq!(listed.messages[0].folder_id, folder); + } +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test -p nunciod --test spine_e2e_test list_messages_reports_the_folder` +Expected: FAIL to compile until `grpc.rs` is migrated. + +- [ ] **Step 3: Migrate the gRPC layer** + +**Account scoping at the boundary.** The store methods now take an +`account_id`, but `ListFoldersRequest` and `ListMessagesRequest` carry none and +this plan does not change the wire schema. Bridge it by iterating the accounts +and merging, which reproduces exactly today's behaviour — no wire change, no +regression: + +```rust + // The request carries no account, so this reproduces the pre-existing + // account-blind view: every account's folders, merged by folder id. The + // store below it is account-scoped and correct; only this presentation + // is ambiguous, and it is ambiguous in precisely the way it already was. + // An account-scoped request field is the real fix and belongs with the + // rest of the placement surface. + let mut merged: std::collections::BTreeMap = + std::collections::BTreeMap::new(); + for account in db.list_accounts().await? { + for folder in db.list_folders(&account.id).await? { + let entry = merged.entry(folder.id.clone()).or_insert_with(|| { + nuncio_core::model::Folder { + id: folder.id.clone(), + name: folder.name.clone(), + total_messages: 0, + unread_messages: 0, + } + }); + entry.total_messages += folder.total_messages; + entry.unread_messages += folder.unread_messages; + } + } +``` + +Apply the same account loop to `ListMessages`, concatenating each account's +matches for the requested folder and then applying the existing ordering and +page size to the merged result. Use whatever account-listing method `db` +already exposes rather than adding one. + +`ListMessages` already carries a folder, so map each `(Email, Placement)` pair +straight onto the proto `Message`, taking `folder_id` and `read` from the +placement. + +`GetMessage` receives only an id, so it must choose. Resolve through +`placements_of` and take the first — `placements_of` orders by `folder_id ASC`, +so the choice is deterministic rather than arbitrary: + +```rust + // A message key can name several placements and this RPC's request + // carries no folder to disambiguate them. Report the first by folder id + // so the answer is at least stable between calls, and mark the response + // honestly: the full placement list needs a wire field this contract + // does not have yet. + let placements = db.placements_of(&request.message_id).await?; + let Some(primary) = placements.first() else { + return Err(tonic::Status::not_found( + "message has no placement; it is no longer in any folder", + )); + }; +``` + +Add the same note to the `.proto` beside `Message.folder_id`: + +```proto + // Mailbox folder identifier (e.g. "inbox"). + // + // A message can occupy several folders at once. This names the one the + // message was reached through: the requested folder for a listing, and the + // lowest-sorting folder id for a lookup by id. A field carrying the complete + // placement set is not yet part of this contract. + string folder_id = 3; +``` + +and correct the `Message.id` comment, whose current text describes the old +folder-scoped derivation: + +```proto + // Unique message identifier. OPAQUE and server-assigned: content-addressed + // from the message's strongest available identity, so it is independent of + // which folder the message currently occupies and survives a move. Never a + // raw upstream protocol id (e.g. an IMAP UID, which is only stable within a + // single IMAP session and is NOT a durable cross-session identifier). + // Clients MUST treat this as an opaque token -- pass it back verbatim in + // later requests, but never parse, construct, or assume any structure in + // its value. + string id = 1; +``` + +- [ ] **Step 4: Reconcile the descriptor golden** + +Run: `cargo test -p nuncio-proto descriptor_matches_committed_golden` + +If it passes, comments are not in the descriptor and nothing more is needed. +If it fails, the test prints the exact path of the freshly generated +descriptor; copy it over `crates/nuncio-proto/proto/nuncio/v1/descriptor.bin` +exactly as the failure message instructs, then re-run to confirm. + +- [ ] **Step 5: Fix the CLI** + +Work through `cargo check-all` errors in `crates/nuncio-cli/src/runner.rs`. +The CLI consumes the proto types, which have not changed shape, so the fixes +should be confined to test fixtures constructing `Message` values. + +- [ ] **Step 6: Run the full gate** + +```bash +cargo fmt --all -- --check +cargo check-all +cargo test-all +``` + +Expected: all three pass. Give the test command ~540s; `cargo test-all` can +exceed a 10-minute tool timeout on this workspace. + +- [ ] **Step 7: Commit** + +```bash +git add crates/nunciod/src/grpc.rs crates/nuncio-cli/src/runner.rs crates/nuncio-proto/ +git commit -m "feat(daemon): present the placement a message was reached through" +``` + +--- + +## Task 10: Documentation and PR + +**Files:** +- Modify: `docs/ROADMAP.md` and `docs/roadmap/index.html` (both, in one commit) +- Modify: `docs/HANDOFF.md` + +- [ ] **Step 1: Update the roadmap in both formats** + +Mark the message/placement split done in `docs/ROADMAP.md`, and make the +matching edit to `docs/roadmap/index.html`. The rendered page is published at + and must never drift from the +markdown; the repo requires both to change in the same commit. + +- [ ] **Step 2: Commit** + +```bash +git add docs/ +git commit -m "docs(roadmap): mark the message/placement split complete" +``` + +- [ ] **Step 3: Push and open the PR** + +SSH has no key in this environment; push over HTTPS with the `gh` credential +helper and leave the remote as SSH: + +```bash +git -c credential.helper='!gh auth git-credential' \ + push https://github.com/KofTwentyTwo/nuncio.git story/message-placement-split +``` + +Open the PR against `dev`. The body must carry the reasoning, not just the +what, and must state explicitly: + +- the branch merges #398 (#386) and #400 (#387) as its declared ADR + dependencies, so **those two PRs must land first** +- `IDENTITY_SCHEMA_VERSION` goes 1 → 2, so **every existing local store resets + its cached mail on next open**; accounts, credentials, and rules survive +- `EMAILID` / `X-GM-MSGID` are not yet fetched, so identity currently rests on + the Message-ID/content tier and the surrogate fallback; adding those FETCH + items lifts rows to a stronger tier on the next resync +- JMAP still reports `present: None` and therefore still accumulates ghost + messages — unchanged by this PR, and needs `Email/changes` +- the wire contract is unchanged; `Message.folder_id` now names the placement + the message was reached through, and the full placement set awaits #395 +- expect conflicts in `db.rs`, `imap.rs`, and `sync.rs` against the + #401–#403 chain + +--- + +## Deferred, deliberately + +Named here so they are not mistaken for oversights. + +- **`EMAILID` / `X-GM-MSGID` are not requested.** Wiring the FETCH items needs a + test server that serves them; the precedence handles their absence correctly + and picks them up for free once they arrive. +- **The full placement set is not on the wire.** `Message` gains it in #395, + which owns the proto surface. +- **`GetMessage` picks one placement.** Deterministic (lowest folder id) and + documented, but genuinely lossy until the contract can express the set. +- **JMAP ghost messages persist.** `present: None` is unchanged; the fix is + `Email/changes` and is not filed as an issue yet.