feat(store): split message identity from mailbox placements #405

Merged
KofTwentyTwo merged 24 commits from story/message-placement-split into dev 2026-08-14 20:20:55 +00:00
26 changed files with 7917 additions and 1387 deletions

No files matched your search

+10 -11
View File
@@ -87,10 +87,13 @@ impl ExportEngine {
writer.write_all(from_line.as_bytes())?;
total_bytes += from_line.len() as u64;
// Headers
// Headers. No X-Nuncio-Folder header: the folder is a property of
// a placement now, not of the message, so `Email` has none to
// name here. Restoring it would mean threading the relevant
// `Placement` into this function's signature.
let headers = format!(
"Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nX-Nuncio-Folder: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n",
email.id, email.sender, email.recipient, email.subject, date_str, email.account_id, email.folder_id
"Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n",
email.id, email.sender, email.recipient, email.subject, date_str, email.account_id
);
writer.write_all(headers.as_bytes())?;
total_bytes += headers.len() as u64;
@@ -184,32 +187,28 @@ mod tests {
Email {
id: "msg-100".to_string(),
account_id: "acct-1".to_string(),
folder_id: "INBOX".to_string(),
remote_id: "100".to_string(),
uid_validity: "1".to_string(),
subject: "Test Subject 1".to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1700000000,
read: false,
body_plain: Some("Hello world body text".to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
},
Email {
id: "msg-101".to_string(),
account_id: "acct-1".to_string(),
folder_id: "INBOX".to_string(),
remote_id: "101".to_string(),
uid_validity: "1".to_string(),
subject: "Test Subject 2".to_string(),
sender: "charlie@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1700000100,
read: true,
body_plain: Some("Second body text".to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
},
]
}
+621 -24
View File
@@ -61,33 +61,153 @@ pub struct Attachment {
pub content: Bytes,
}
/// Email message domain entity owned by Nuncio core.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Email {
/// Opaque, deterministic surrogate identifier for the message.
/// Which tier of the identity precedence produced a [`MessageIdentity`].
///
/// Persisted alongside the key so a later pass can tell a server-assigned
/// identity from a locally-derived fallback without recomputing it, and so a
/// store can be audited for how much of it rests on the weakest tier.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum IdentitySource {
/// RFC 8474 OBJECTID `EMAILID` -- server-assigned and stable across folders.
EmailId,
/// Gmail's `X-GM-MSGID`, the same guarantee by a vendor extension.
GmailMsgId,
/// `Message-ID` **and** a content hash together. Never the header alone.
MessageIdContent,
/// Folder-scoped last resort: the message is only identified by where it
/// currently sits, so the same mail in another folder is a second message.
Surrogate,
}
impl IdentitySource {
/// Stable token persisted in `messages.identity_source`.
pub fn as_str(&self) -> &'static str {
match self {
Self::EmailId => "emailid",
Self::GmailMsgId => "gmsgid",
Self::MessageIdContent => "msgid_content",
Self::Surrogate => "surrogate",
}
}
/// Parse a token written by [`Self::as_str`]. Returns `None` for anything
/// else so an unrecognised value fails loudly rather than defaulting to a
/// tier that would overstate how trustworthy the key is.
///
/// Derived from [`Email::surrogate_id`] over the account, folder,
/// UIDVALIDITY, and protocol-native id. It carries no decodable internal
/// encoding (callers must treat it as opaque) and is stable across
/// re-syncs of the same message, so persisting it upserts the same row
/// rather than duplicating. Distinct messages that happen to share a
/// protocol id across folders or accounts hash to distinct surrogates, so
/// one can never silently overwrite another.
pub id: String,
/// Account identifier owning the message.
/// Deliberately not `std::str::FromStr`: that trait forces a `Result` and
/// therefore an error type, when the only failure here is "not one of four
/// known tokens" and `None` already says exactly that.
pub fn from_token(s: &str) -> Option<Self> {
match s {
"emailid" => Some(Self::EmailId),
"gmsgid" => Some(Self::GmailMsgId),
"msgid_content" => Some(Self::MessageIdContent),
"surrogate" => Some(Self::Surrogate),
_ => None,
}
}
}
/// A content-addressed message key together with the tier that produced it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MessageIdentity {
/// Hex SHA-256 key identifying the message within its account.
pub key: String,
/// Which precedence tier produced `key`.
pub source: IdentitySource,
}
/// Server-supplied identity hints for one message, in precedence order.
///
/// Borrowed rather than owned because every caller already holds these as
/// fields on a parsed response and only needs them for the duration of the
/// key derivation.
#[derive(Debug, Clone, Copy, Default)]
pub struct RemoteIdentity<'a> {
/// RFC 8474 `EMAILID` from a `FETCH ... (EMAILID)`, if the server has OBJECTID.
pub email_id: Option<&'a str>,
/// Gmail `X-GM-MSGID`, if the server advertises `X-GM-EXT-1`.
pub gm_msgid: Option<&'a str>,
/// Normalized `Message-ID` (see [`Email::normalize_message_id`]).
pub message_id: Option<&'a str>,
/// Hex SHA-256 over the full RFC822 octets (see [`Email::content_hash_of`]).
pub content_hash: Option<&'a str>,
}
/// One occupancy of a message in one mailbox.
///
/// A message can sit in several folders at once (an IMAP `COPY`, a Gmail label,
/// a message that is both in a thread's folder and in `\All`). The addressing
/// coordinates and the read flag are properties of the *occupancy*, not of the
/// message: IMAP `\Seen` is per-mailbox, and a UID is only meaningful inside one
/// `uid_validity` scope of one folder.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Placement {
/// Account owning the mailbox.
pub account_id: String,
/// Mailbox folder identifier (e.g. "inbox").
pub folder_id: String,
/// Protocol-native message id used to address the message on its server:
/// the IMAP UID as a decimal string, or the JMAP Email object id. This is
/// what a remote mutation must use to identify the message on the wire --
/// never the opaque [`Email::id`].
pub remote_id: String,
/// The addressing scope the `remote_id` is only meaningful within: the IMAP
/// folder UIDVALIDITY as a decimal string. Protocols without a UIDVALIDITY
/// (JMAP) carry a stable sentinel instead, so the surrogate id stays
/// deterministic.
/// The IMAP UIDVALIDITY the `remote_id` was captured under, as a decimal
/// string; a stable sentinel for protocols without one (JMAP). In the
/// primary key because UIDs restart at 1 after a bump and would otherwise
/// collide with rows written before it.
pub uid_validity: String,
/// Protocol-native id addressing the message in this mailbox: the IMAP UID
/// as a decimal string, or the JMAP Email object id.
pub remote_id: String,
/// Read/unread state **in this mailbox**.
pub read: bool,
}
/// The four coordinates that address one mailbox occupancy.
///
/// The addressing half of a [`Placement`], without the read flag: enough to
/// name an occupancy, not enough to describe it. A protocol backend reports
/// these when a folder stops mentioning a message, and the store deletes
/// exactly the rows they name -- which is why the type lives here in `core`
/// rather than in either crate. What a folder stops reporting is an occupancy,
/// never a message; a message goes only when its last placement does.
///
/// `Ord` is derived because the sync path sorts and dedups batches of these
/// before deleting; `Hash`/`Eq` because it is also the key of the
/// already-present set used to classify a fetched chunk.
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
pub struct PlacementKey {
/// Account owning the mailbox.
pub account_id: String,
/// Mailbox folder identifier.
pub folder_id: String,
/// The UIDVALIDITY generation `remote_id` was captured under.
pub uid_validity: String,
/// Protocol-native id addressing the message in this mailbox.
pub remote_id: String,
}
impl Placement {
/// The addressing coordinates of this occupancy, dropping the read flag.
pub fn key(&self) -> PlacementKey {
PlacementKey {
account_id: self.account_id.clone(),
folder_id: self.folder_id.clone(),
uid_validity: self.uid_validity.clone(),
remote_id: self.remote_id.clone(),
}
}
}
/// Email message domain entity owned by Nuncio core.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Email {
/// Opaque, account-scoped identity for the message itself.
///
/// Derived by [`Email::derive_message_key`], so it is independent of which
/// folder the message currently occupies: moving a message between folders
/// does not change it. Where the message actually sits, and its read state
/// there, live in [`Placement`] rows instead. Callers must treat this as an
/// opaque token -- it carries no decodable structure.
pub id: String,
/// Account identifier owning the message.
pub account_id: String,
/// Subject line.
pub subject: String,
/// Sender address (e.g. "alice@nuncio.mx").
@@ -96,14 +216,32 @@ pub struct Email {
pub recipient: String,
/// Unix timestamp of message arrival.
pub received_at: i64,
/// Read/unread flag status.
pub read: bool,
/// Plaintext message body.
pub body_plain: Option<String>,
/// HTML message body.
pub body_html: Option<String>,
/// List of attached files.
pub attachments: Vec<Attachment>,
/// The message's RFC 5322 `Message-ID`, normalized by
/// [`Email::normalize_message_id`], or `None` when the message carried
/// none (the header is `SHOULD`, not `MUST`) or the transport did not
/// surface it.
///
/// Captured but not yet used for identity. It is a *hint* toward a
/// folder-independent identity, never an identity on its own: the value is
/// public (it is echoed in the `References` of every reply) and forgeable,
/// so keying storage on it alone would let a crafted message address, and
/// overwrite, one the user already trusts.
pub message_id: Option<String>,
/// Hex SHA-256 over the full RFC822 octets the message was parsed from, or
/// `None` when only headers or an envelope were available.
///
/// Pairs with [`Email::message_id`]: the same `Message-ID` carrying
/// different content is two messages, not one. That case is ordinary --
/// a mailing-list copy and a direct copy of the same mail share a
/// `Message-ID` while differing in headers and footer, and a draft keeps
/// one `Message-ID` across every save.
pub content_hash: Option<String>,
}
impl Email {
@@ -131,6 +269,139 @@ impl Email {
}
hex::encode(hasher.finalize())
}
/// Canonicalize a raw `Message-ID` header value.
///
/// Every engine syncing an account must derive byte-identical values from
/// the same header, or two caches that agree in substance will disagree in
/// bytes and no comparison between them can be trusted. The rules are
/// therefore fixed and deliberately small:
///
/// 1. Remove **all** ASCII whitespace. A `msg-id` cannot legally contain
/// any, so this unfolds a wrapped header without needing to know how the
/// transport folded it -- and transports differ.
/// 2. Strip one surrounding pair of angle brackets, which servers include
/// or omit inconsistently between `ENVELOPE` and a raw header.
/// 3. Preserve case. Only the domain half is case-insensitive per RFC 5322
/// §3.6.4; lowercasing the whole value would merge ids that the
/// originating server considers distinct, and the same server returns
/// the same bytes to every engine anyway.
///
/// Returns `None` for an absent or empty value so a missing header stays
/// distinguishable from a present-but-empty one.
pub fn normalize_message_id(raw: &str) -> Option<String> {
let stripped: String = raw.chars().filter(|c| !c.is_ascii_whitespace()).collect();
let stripped = stripped
.strip_prefix('<')
.and_then(|s| s.strip_suffix('>'))
.unwrap_or(&stripped);
if stripped.is_empty() {
None
} else {
Some(stripped.to_string())
}
}
/// Hex SHA-256 over a message's full RFC822 octets, for the
/// `content_hash` field.
///
/// Deterministic for identical bytes, which is what makes it usable as the
/// tiebreaker against a duplicated `Message-ID`. It is *not* stable across
/// servers that re-serialize a message, so it distinguishes content within
/// an account rather than identifying a message globally.
pub fn content_hash_of(raw_bytes: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(raw_bytes);
hex::encode(hasher.finalize())
}
/// Domain-separated, length-prefixed digest over an identity tier's inputs.
///
/// The tier label participates in the hash so that a value appearing in two
/// different tiers (a server whose `EMAILID` happens to equal another's
/// `X-GM-MSGID`) cannot produce one key. Length prefixes stop two distinct
/// field tuples serializing to the same byte stream, exactly as in
/// [`Email::surrogate_id`].
fn identity_digest(tier: &str, account_id: &str, parts: &[&str]) -> String {
let mut hasher = Sha256::new();
for field in [tier, account_id].into_iter().chain(parts.iter().copied()) {
hasher.update((field.len() as u64).to_le_bytes());
hasher.update(field.as_bytes());
}
hex::encode(hasher.finalize())
}
/// Derive the account-scoped message key by identity precedence.
///
/// `EMAILID` → `X-GM-MSGID` → (`Message-ID` + content hash) → surrogate.
/// Every tier is account-scoped: identity is only ever claimed within the
/// account that fetched it, so one account's server cannot mint a key that
/// addresses another account's mail.
///
/// The third tier requires **both** a `Message-ID` and a content hash. A
/// `Message-ID` is public -- it is echoed in the `References` of every reply
/// -- and forgeable, so keying on it alone would let a crafted message
/// address, and with an upsert overwrite, a message the user already trusts,
/// and would evade filtering by never being classified as new. Pairing it
/// with the content hash also gets the ordinary cases right: a mailing-list
/// copy and a direct copy share a `Message-ID` but differ in content, and so
/// are correctly two messages.
///
/// The surrogate tier is folder-scoped and therefore *not* stable across a
/// move; it is the honest answer when the server offered nothing better.
///
/// # Keys do not yet converge across engine types
///
/// Precedence makes a key reproducible from the same inputs, not from the
/// same *message*. The IMAP engine does not currently request `EMAILID` or
/// `X-GM-MSGID`, so it can only ever reach the `Message-ID`+content tier or
/// the surrogate, while JMAP supplies a server id and enters at the top
/// tier. Two engines syncing one account therefore produce disjoint key
/// spaces: the same message is two identities, one per engine. This is a
/// known gap in the convergent multi-engine model, not a property to rely
/// on -- do not assume a key derived by one engine addresses the row
/// another engine wrote.
pub fn derive_message_key(
account_id: &str,
remote: RemoteIdentity<'_>,
folder_id: &str,
uid_validity: &str,
remote_id: &str,
) -> MessageIdentity {
if let Some(email_id) = remote.email_id.filter(|v| !v.is_empty()) {
return MessageIdentity {
key: Self::identity_digest("emailid", account_id, &[email_id]),
source: IdentitySource::EmailId,
};
}
if let Some(gm_msgid) = remote.gm_msgid.filter(|v| !v.is_empty()) {
return MessageIdentity {
key: Self::identity_digest("gmsgid", account_id, &[gm_msgid]),
source: IdentitySource::GmailMsgId,
};
}
if let (Some(message_id), Some(content_hash)) = (
remote.message_id.filter(|v| !v.is_empty()),
remote.content_hash.filter(|v| !v.is_empty()),
) {
return MessageIdentity {
key: Self::identity_digest(
"msgid_content",
account_id,
&[message_id, content_hash],
),
source: IdentitySource::MessageIdContent,
};
}
MessageIdentity {
key: Self::identity_digest(
"surrogate",
account_id,
&[folder_id, uid_validity, remote_id],
),
source: IdentitySource::Surrogate,
}
}
}
/// Mailbox folder entity.
@@ -247,4 +518,330 @@ mod tests {
Email::surrogate_id("ab", "c", "1", "1"),
);
}
#[test]
fn normalize_message_id_unfolds_and_unwraps_to_one_canonical_form() {
// Every spelling a transport might hand us for the same id must
// collapse to identical bytes, or two engines holding the same message
// cannot be shown to agree.
let canonical = "abc123@mail.example.com";
for raw in [
"<abc123@mail.example.com>",
"abc123@mail.example.com",
" <abc123@mail.example.com> ",
"<abc123@mail.\r\n example.com>",
"<abc123@mail.\t example.com>",
] {
assert_eq!(
Email::normalize_message_id(raw).as_deref(),
Some(canonical),
"input {raw:?} did not canonicalize"
);
}
}
#[test]
fn normalize_message_id_preserves_case_and_rejects_empty() {
// Only the domain half is case-insensitive per RFC 5322 3.6.4;
// lowercasing wholesale would merge ids the origin considers distinct.
assert_eq!(
Email::normalize_message_id("<AbC@Example.COM>").as_deref(),
Some("AbC@Example.COM")
);
assert_eq!(Email::normalize_message_id(""), None);
assert_eq!(Email::normalize_message_id("<>"), None);
assert_eq!(Email::normalize_message_id(" "), None);
}
#[test]
fn content_hash_distinguishes_identical_message_ids_with_different_bodies() {
// The case the hash exists for: a list copy and a direct copy share a
// Message-ID but are not the same message.
let direct = b"Message-ID: <shared@example.com>\r\n\r\nbody";
let via_list = b"Message-ID: <shared@example.com>\r\nList-Id: l\r\n\r\nbody\n--\nfooter";
assert_ne!(
Email::content_hash_of(direct),
Email::content_hash_of(via_list)
);
assert_eq!(
Email::content_hash_of(direct),
Email::content_hash_of(direct),
"the hash must be deterministic for identical octets"
);
}
#[test]
fn message_key_precedence_prefers_emailid_over_everything_below_it() {
let remote = RemoteIdentity {
email_id: Some("M00000001"),
gm_msgid: Some("1234567890"),
message_id: Some("a@b.example"),
content_hash: Some("deadbeef"),
};
let id = Email::derive_message_key("acct-1", remote, "INBOX", "42", "5");
assert_eq!(id.source, IdentitySource::EmailId);
// Same EMAILID in a different folder is the SAME message.
let elsewhere = Email::derive_message_key(
"acct-1",
RemoteIdentity {
email_id: Some("M00000001"),
gm_msgid: None,
message_id: None,
content_hash: None,
},
"Archive",
"99",
"7",
);
assert_eq!(id.key, elsewhere.key);
}
#[test]
fn message_key_is_account_scoped_at_every_tier() {
let emailid = || RemoteIdentity {
email_id: Some("M00000001"),
gm_msgid: None,
message_id: None,
content_hash: None,
};
let a = Email::derive_message_key("acct-1", emailid(), "INBOX", "42", "5");
let b = Email::derive_message_key("acct-2", emailid(), "INBOX", "42", "5");
assert_ne!(
a.key, b.key,
"the same EMAILID in two accounts must not collide"
);
let gmsgid = || RemoteIdentity {
email_id: None,
gm_msgid: Some("1234567890"),
message_id: None,
content_hash: None,
};
let a = Email::derive_message_key("acct-1", gmsgid(), "INBOX", "42", "5");
let b = Email::derive_message_key("acct-2", gmsgid(), "INBOX", "42", "5");
assert_ne!(
a.key, b.key,
"the same X-GM-MSGID in two accounts must not collide"
);
let msgid_content = || RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: Some("a@b.example"),
content_hash: Some("deadbeef"),
};
let a = Email::derive_message_key("acct-1", msgid_content(), "INBOX", "42", "5");
let b = Email::derive_message_key("acct-2", msgid_content(), "INBOX", "42", "5");
assert_ne!(
a.key, b.key,
"the same Message-ID and content hash in two accounts must not collide"
);
let surrogate = RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: None,
content_hash: None,
};
let a = Email::derive_message_key("acct-1", surrogate, "INBOX", "42", "5");
let b = Email::derive_message_key("acct-2", surrogate, "INBOX", "42", "5");
assert_ne!(
a.key, b.key,
"the same folder/uidvalidity/remote-id in two accounts must not collide"
);
}
#[test]
fn a_message_id_without_a_content_hash_never_reaches_the_content_tier() {
// The non-negotiable: a Message-ID is public and forgeable, so it must
// never key storage on its own. Missing content hash falls through to the
// folder-scoped surrogate rather than trusting the header alone.
let id = Email::derive_message_key(
"acct-1",
RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: Some("a@b.example"),
content_hash: None,
},
"INBOX",
"42",
"5",
);
assert_eq!(id.source, IdentitySource::Surrogate);
let surrogate_only = Email::derive_message_key(
"acct-1",
RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: None,
content_hash: None,
},
"INBOX",
"42",
"5",
);
assert_eq!(id.key, surrogate_only.key);
}
#[test]
fn the_same_message_id_with_different_content_is_two_messages() {
let mk = |hash: &str| {
Email::derive_message_key(
"acct-1",
RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: Some("a@b.example"),
content_hash: Some(hash),
},
"INBOX",
"42",
"5",
)
};
let list_copy = mk("1111");
let direct_copy = mk("2222");
assert_eq!(list_copy.source, IdentitySource::MessageIdContent);
assert_ne!(list_copy.key, direct_copy.key);
}
#[test]
fn tiers_are_domain_separated_so_a_shared_value_cannot_collide_across_them() {
let as_emailid = Email::derive_message_key(
"acct-1",
RemoteIdentity {
email_id: Some("X"),
gm_msgid: None,
message_id: None,
content_hash: None,
},
"INBOX",
"42",
"5",
);
let as_gmsgid = Email::derive_message_key(
"acct-1",
RemoteIdentity {
email_id: None,
gm_msgid: Some("X"),
message_id: None,
content_hash: None,
},
"INBOX",
"42",
"5",
);
assert_ne!(as_emailid.key, as_gmsgid.key);
}
fn sample_placement() -> Placement {
Placement {
account_id: "acct-1".to_string(),
folder_id: "INBOX".to_string(),
uid_validity: "42".to_string(),
remote_id: "5".to_string(),
read: true,
}
}
#[test]
fn placement_key_carries_every_addressing_coordinate_and_drops_the_read_flag() {
// The key must reproduce all four coordinates exactly. A key that
// dropped or reordered one would address a different occupancy, and a
// sync diffing reported-against-stored would delete live mail.
let placement = sample_placement();
let key = placement.key();
assert_eq!(key.account_id, "acct-1");
assert_eq!(key.folder_id, "INBOX");
assert_eq!(key.uid_validity, "42");
assert_eq!(key.remote_id, "5");
// The read flag is the one field a key must NOT carry: it describes an
// occupancy rather than naming it, so including it would make the same
// occupancy compare unequal to itself after the user opened the message.
let mut unread = placement.clone();
unread.read = false;
assert_ne!(placement, unread);
assert_eq!(placement.key(), unread.key());
}
#[test]
fn placement_keys_distinguish_every_coordinate_that_can_differ() {
let base = sample_placement().key();
for mutated in [
PlacementKey {
account_id: "acct-2".to_string(),
..base.clone()
},
PlacementKey {
folder_id: "Archive".to_string(),
..base.clone()
},
PlacementKey {
uid_validity: "43".to_string(),
..base.clone()
},
PlacementKey {
remote_id: "6".to_string(),
..base.clone()
},
] {
assert_ne!(
base, mutated,
"changing one coordinate must yield a different occupancy"
);
}
// Usable as a set key and sortable, which is what the sync path needs
// to diff a reported UID set against what the store holds.
let mut set = std::collections::HashSet::new();
assert!(set.insert(base.clone()));
assert!(!set.insert(base.clone()), "an equal key must not re-insert");
let mut keys = [
PlacementKey {
remote_id: "6".to_string(),
..base.clone()
},
base.clone(),
];
keys.sort();
assert_eq!(keys[0], base);
}
#[test]
fn placement_key_round_trips_through_serde() {
let key = sample_placement().key();
let json = serde_json::to_string(&key).unwrap();
let parsed: PlacementKey = serde_json::from_str(&json).unwrap();
assert_eq!(parsed, key);
}
#[test]
fn placement_round_trips_through_serde_including_its_read_flag() {
let placement = sample_placement();
let json = serde_json::to_string(&placement).unwrap();
let parsed: Placement = serde_json::from_str(&json).unwrap();
assert_eq!(parsed, placement);
assert!(parsed.read);
}
#[test]
fn identity_source_round_trips_through_its_stored_string() {
for source in [
IdentitySource::EmailId,
IdentitySource::GmailMsgId,
IdentitySource::MessageIdContent,
IdentitySource::Surrogate,
] {
assert_eq!(IdentitySource::from_token(source.as_str()), Some(source));
}
assert_eq!(
IdentitySource::from_token("not_a_tier"),
None,
"an unrecognised token must fail loudly, never default to a tier"
);
}
}
+300 -82
View File
@@ -6,13 +6,27 @@ use crate::ast::{
};
use arc_swap::ArcSwap;
use hmac::{Hmac, Mac};
use nuncio_core::model::Email;
use nuncio_core::model::{Email, Placement};
use regex::Regex;
use sha2::Sha256;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tracing::{debug, info, warn};
/// A message together with the one mailbox occupancy being evaluated.
///
/// `WHERE FOLDER = 'INBOX'` has no answer for a message that sits in three
/// folders at once, so evaluation is defined against a single placement and the
/// caller fans out over however many the message has. Borrowing both halves
/// keeps that fan-out free of clones.
#[derive(Debug, Clone, Copy)]
pub struct PlacedEmail<'a> {
/// Identity and content -- the same for every placement.
pub email: &'a Email,
/// The occupancy this evaluation is about.
pub placement: &'a Placement,
}
/// Pre-compiled single filter rule optimizing regex and predicate execution.
#[derive(Clone)]
pub struct CompiledFilter {
@@ -55,31 +69,40 @@ impl CompiledFilter {
}
}
/// Evaluate email message against compiled rule condition tree.
/// Evaluate one placement against the compiled rule condition tree.
///
/// This checks only the WHERE-clause conditions; it does NOT apply the
/// rule's account scope. Callers must gate on
/// [`FilterRule::matches_account`] first (as every `FilterEngine` entry
/// point does), or a rule scoped to one account will fire on messages of
/// another.
pub fn evaluate_condition(&self, email: &Email) -> bool {
Self::eval_node(&self.rule.conditions, email, &self.compiled_regexes)
pub fn evaluate_condition(&self, placed: PlacedEmail<'_>) -> bool {
Self::eval_node(&self.rule.conditions, placed, &self.compiled_regexes)
}
fn eval_node(node: &ConditionNode, email: &Email, regexes: &[(String, Regex)]) -> bool {
fn eval_node(
node: &ConditionNode,
placed: PlacedEmail<'_>,
regexes: &[(String, Regex)],
) -> bool {
match node {
ConditionNode::Leaf(leaf) => Self::eval_leaf(leaf, email, regexes),
ConditionNode::Leaf(leaf) => Self::eval_leaf(leaf, placed, regexes),
ConditionNode::And(children) => {
children.iter().all(|c| Self::eval_node(c, email, regexes))
children.iter().all(|c| Self::eval_node(c, placed, regexes))
}
ConditionNode::Or(children) => {
children.iter().any(|c| Self::eval_node(c, email, regexes))
children.iter().any(|c| Self::eval_node(c, placed, regexes))
}
ConditionNode::Not(inner) => !Self::eval_node(inner, email, regexes),
ConditionNode::Not(inner) => !Self::eval_node(inner, placed, regexes),
}
}
fn eval_leaf(leaf: &ConditionLeaf, email: &Email, regexes: &[(String, Regex)]) -> bool {
fn eval_leaf(
leaf: &ConditionLeaf,
placed: PlacedEmail<'_>,
regexes: &[(String, Regex)],
) -> bool {
let email = placed.email;
match &leaf.field {
FilterField::Subject => {
Self::eval_string_op(&email.subject, &leaf.operator, &leaf.value, regexes)
@@ -98,12 +121,18 @@ impl CompiledFilter {
.unwrap_or("");
Self::eval_string_op(body, &leaf.operator, &leaf.value, regexes)
}
FilterField::Folder => {
Self::eval_string_op(&email.folder_id, &leaf.operator, &leaf.value, regexes)
}
FilterField::Account => {
Self::eval_string_op(&email.account_id, &leaf.operator, &leaf.value, regexes)
}
FilterField::Folder => Self::eval_string_op(
&placed.placement.folder_id,
&leaf.operator,
&leaf.value,
regexes,
),
FilterField::Account => Self::eval_string_op(
&placed.placement.account_id,
&leaf.operator,
&leaf.value,
regexes,
),
FilterField::HasAttachment => {
let has = !email.attachments.is_empty();
if let FilterValue::Boolean(b) = leaf.value {
@@ -277,18 +306,26 @@ impl FilterEngine {
Ok(())
}
/// Evaluate email message returning matching rule actions.
/// Evaluate one placement of a message, returning the matching rules' actions.
///
/// Defined per placement, not per message: `FOLDER` and `ACCOUNT` are
/// properties of where the message sits, and a message can sit in several
/// places at once. Callers holding a multi-placement message evaluate each
/// placement and are responsible for not acting twice on one message -- see
/// the fire-once claim in the store.
///
/// A rule only ever fires for the account it was created against
/// (`FilterRule::target_account`, set via `ON ACCOUNT` or `*` for all
/// accounts) — condition matching alone is not account-scoped, so this
/// check must happen before `evaluate_condition` runs.
pub fn evaluate(&self, email: &Email) -> Vec<(FilterRule, Vec<RuleAction>)> {
pub fn evaluate(&self, placed: PlacedEmail<'_>) -> Vec<(FilterRule, Vec<RuleAction>)> {
let guard = self.cache.load();
let mut results = Vec::new();
for filter in &guard.filters {
if filter.rule.matches_account(&email.account_id) && filter.evaluate_condition(email) {
if filter.rule.matches_account(&placed.placement.account_id)
&& filter.evaluate_condition(placed)
{
debug!(
rule_id = %filter.rule.id,
rule_name = %filter.rule.name,
@@ -315,21 +352,25 @@ impl FilterEngine {
/// the remaining rules rather than aborting the whole batch.
pub async fn evaluate_with_timeout(
&self,
email: &Email,
placed: PlacedEmail<'_>,
timeout_duration: Duration,
) -> Vec<(FilterRule, Vec<RuleAction>)> {
let guard = self.cache.load();
let mut results = Vec::new();
for filter in &guard.filters {
if !filter.rule.matches_account(&email.account_id) {
if !filter.rule.matches_account(&placed.placement.account_id) {
continue;
}
let filter_for_task = filter.clone();
let email_for_task = email.clone();
let email_for_task = placed.email.clone();
let placement_for_task = placed.placement.clone();
let eval_task = tokio::task::spawn_blocking(move || {
filter_for_task.evaluate_condition(&email_for_task)
filter_for_task.evaluate_condition(PlacedEmail {
email: &email_for_task,
placement: &placement_for_task,
})
});
match tokio::time::timeout(timeout_duration, eval_task).await {
@@ -370,7 +411,7 @@ impl FilterEngine {
/// Rules whose account scope does not match the message are reported as
/// `SKIPPED (account scope mismatch)` rather than omitted, so the trace
/// distinguishes "condition did not match" from "rule does not apply here".
pub fn preview(&self, email: &Email) -> FilterPreviewResult {
pub fn preview(&self, placed: PlacedEmail<'_>) -> FilterPreviewResult {
let start = Instant::now();
let guard = self.cache.load();
let mut matched_rule_id = None;
@@ -380,7 +421,7 @@ impl FilterEngine {
let mut matched = false;
for filter in &guard.filters {
if !filter.rule.matches_account(&email.account_id) {
if !filter.rule.matches_account(&placed.placement.account_id) {
traces.push(format!(
"Rule '{}' (priority {}): SKIPPED (account scope mismatch)",
filter.rule.name, filter.rule.priority
@@ -388,7 +429,7 @@ impl FilterEngine {
continue;
}
let is_match = filter.evaluate_condition(email);
let is_match = filter.evaluate_condition(placed);
traces.push(format!(
"Rule '{}' (priority {}): {}",
filter.rule.name,
@@ -406,7 +447,7 @@ impl FilterEngine {
let elapsed = start.elapsed().as_micros() as u64;
FilterPreviewResult {
message_id: email.id.clone(),
message_id: placed.email.id.clone(),
matched,
matched_rule_id,
matched_rule_name,
@@ -435,22 +476,31 @@ mod tests {
use super::*;
use nuncio_core::model::Email;
fn test_email(account_id: &str, subject: &str, folder_id: &str) -> Email {
Email {
/// Build an `Email`/`Placement` pair that shares one account between the
/// message and the mailbox it is placed in, matching how a message and
/// its own placement always agree on account.
fn test_placed_email(account_id: &str, subject: &str, folder_id: &str) -> (Email, Placement) {
let email = Email {
id: "msg-1".to_string(),
account_id: account_id.to_string(),
folder_id: folder_id.to_string(),
remote_id: "1".to_string(),
uid_validity: "1".to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1700000000,
read: false,
body_plain: Some("Hello".to_string()),
body_html: None,
attachments: Vec::new(),
}
message_id: None,
content_hash: None,
};
let placement = Placement {
account_id: account_id.to_string(),
folder_id: folder_id.to_string(),
uid_validity: "1".to_string(),
remote_id: "1".to_string(),
read: false,
};
(email, placement)
}
#[test]
@@ -459,27 +509,19 @@ mod tests {
let rule = crate::parser::NsqlParser::parse_rule("Urgent", 1, nsql).unwrap();
let engine = FilterEngine::new(vec![rule]).unwrap();
let email = Email {
id: "msg-1".to_string(),
account_id: "acct-1".to_string(),
folder_id: "inbox".to_string(),
remote_id: "1".to_string(),
uid_validity: "1".to_string(),
subject: "Urgent Meeting".to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1700000000,
read: false,
body_plain: Some("Hello".to_string()),
body_html: None,
attachments: Vec::new(),
};
let (email, placement) = test_placed_email("acct-1", "Urgent Meeting", "inbox");
let results = engine.evaluate(&email);
let results = engine.evaluate(PlacedEmail {
email: &email,
placement: &placement,
});
assert_eq!(results.len(), 1);
assert_eq!(results[0].1.len(), 2);
let preview = engine.preview(&email);
let preview = engine.preview(PlacedEmail {
email: &email,
placement: &placement,
});
assert!(preview.matched);
assert_eq!(preview.matched_rule_name, Some("Urgent".to_string()));
}
@@ -499,14 +541,27 @@ mod tests {
// Condition matches the subject, but the message belongs to a
// different account than the rule was created for.
let email_b = test_email("acct-b", "Weekly Report", "inbox");
let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox");
assert!(
engine.evaluate(&email_b).is_empty(),
engine
.evaluate(PlacedEmail {
email: &email_b,
placement: &placement_b
})
.is_empty(),
"rule scoped to acct-a must not match a message from acct-b"
);
let email_a = test_email("acct-a", "Weekly Report", "inbox");
assert_eq!(engine.evaluate(&email_a).len(), 1);
let (email_a, placement_a) = test_placed_email("acct-a", "Weekly Report", "inbox");
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &email_a,
placement: &placement_a
})
.len(),
1
);
}
#[tokio::test]
@@ -515,15 +570,27 @@ mod tests {
let rule = crate::parser::NsqlParser::parse_rule("Scoped To A", 1, nsql).unwrap();
let engine = FilterEngine::new(vec![rule]).unwrap();
let email_b = test_email("acct-b", "Weekly Report", "inbox");
let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox");
let results = engine
.evaluate_with_timeout(&email_b, Duration::from_secs(1))
.evaluate_with_timeout(
PlacedEmail {
email: &email_b,
placement: &placement_b,
},
Duration::from_secs(1),
)
.await;
assert!(results.is_empty());
let email_a = test_email("acct-a", "Weekly Report", "inbox");
let (email_a, placement_a) = test_placed_email("acct-a", "Weekly Report", "inbox");
let results = engine
.evaluate_with_timeout(&email_a, Duration::from_secs(1))
.evaluate_with_timeout(
PlacedEmail {
email: &email_a,
placement: &placement_a,
},
Duration::from_secs(1),
)
.await;
assert_eq!(results.len(), 1);
}
@@ -534,8 +601,11 @@ mod tests {
let rule = crate::parser::NsqlParser::parse_rule("Scoped To A", 1, nsql).unwrap();
let engine = FilterEngine::new(vec![rule]).unwrap();
let email_b = test_email("acct-b", "Weekly Report", "inbox");
let preview = engine.preview(&email_b);
let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox");
let preview = engine.preview(PlacedEmail {
email: &email_b,
placement: &placement_b,
});
assert!(!preview.matched);
assert!(preview
.condition_traces
@@ -549,8 +619,16 @@ mod tests {
let rule = crate::parser::NsqlParser::parse_rule("Global", 1, nsql).unwrap();
let engine = FilterEngine::new(vec![rule]).unwrap();
let email = test_email("any-account", "Weekly Report", "inbox");
assert_eq!(engine.evaluate(&email).len(), 1);
let (email, placement) = test_placed_email("any-account", "Weekly Report", "inbox");
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &email,
placement: &placement
})
.len(),
1
);
}
#[test]
@@ -560,20 +638,30 @@ mod tests {
let engine = FilterEngine::new(vec![rule]).unwrap();
// Same folder_id, different account_id: a rule matching on the
// `account` condition field must key off email.account_id, never
// fall through to comparing folder_id.
let email_a = test_email("acct-a", "Anything", "shared-folder");
// `account` condition field must key off the placement's account_id,
// never fall through to comparing folder_id.
let (email_a, placement_a) = test_placed_email("acct-a", "Anything", "shared-folder");
assert_eq!(
engine.evaluate(&email_a).len(),
engine
.evaluate(PlacedEmail {
email: &email_a,
placement: &placement_a
})
.len(),
1,
"rule on account = 'acct-a' must match a message with account_id 'acct-a'"
"rule on account = 'acct-a' must match a placement with account_id 'acct-a'"
);
let email_b = test_email("acct-b", "Anything", "shared-folder");
let (email_b, placement_b) = test_placed_email("acct-b", "Anything", "shared-folder");
assert!(
engine.evaluate(&email_b).is_empty(),
"rule on account = 'acct-a' must not match a message with account_id 'acct-b', \
even when it shares the same folder_id as an acct-a message"
engine
.evaluate(PlacedEmail {
email: &email_b,
placement: &placement_b
})
.is_empty(),
"rule on account = 'acct-a' must not match a placement with account_id 'acct-b', \
even when it shares the same folder_id as an acct-a placement"
);
}
@@ -583,11 +671,24 @@ mod tests {
let rule = crate::parser::NsqlParser::parse_rule("Not In Spam Or Trash", 1, nsql).unwrap();
let engine = FilterEngine::new(vec![rule]).unwrap();
let inbox_email = test_email("acct-1", "Anything", "inbox");
assert_eq!(engine.evaluate(&inbox_email).len(), 1);
let (inbox_email, inbox_placement) = test_placed_email("acct-1", "Anything", "inbox");
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &inbox_email,
placement: &inbox_placement
})
.len(),
1
);
let spam_email = test_email("acct-1", "Anything", "spam");
assert!(engine.evaluate(&spam_email).is_empty());
let (spam_email, spam_placement) = test_placed_email("acct-1", "Anything", "spam");
assert!(engine
.evaluate(PlacedEmail {
email: &spam_email,
placement: &spam_placement
})
.is_empty());
}
#[test]
@@ -612,8 +713,16 @@ mod tests {
};
let engine = FilterEngine::new(vec![rule]).unwrap();
let email = test_email("acct-1", "Anything", "inbox");
assert_eq!(engine.evaluate(&email).len(), 1);
let (email, placement) = test_placed_email("acct-1", "Anything", "inbox");
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &email,
placement: &placement
})
.len(),
1
);
}
#[test]
@@ -640,8 +749,16 @@ mod tests {
};
let engine = FilterEngine::new(vec![rule]).unwrap();
let email = test_email("acct-1", "Anything", "inbox");
assert_eq!(engine.evaluate(&email).len(), 1);
let (email, placement) = test_placed_email("acct-1", "Anything", "inbox");
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &email,
placement: &placement
})
.len(),
1
);
}
#[tokio::test]
@@ -672,11 +789,17 @@ mod tests {
};
let engine = FilterEngine::new(vec![rule]).unwrap();
let mut email = test_email("acct-1", "Anything", "inbox");
let (mut email, placement) = test_placed_email("acct-1", "Anything", "inbox");
email.body_plain = Some("A".repeat(60_000_000));
let results = engine
.evaluate_with_timeout(&email, Duration::from_millis(1))
.evaluate_with_timeout(
PlacedEmail {
email: &email,
placement: &placement,
},
Duration::from_millis(1),
)
.await;
assert!(
@@ -693,4 +816,99 @@ mod tests {
"expected a WARN log carrying the rule id for the timed-out evaluation, got: {captured:?}"
);
}
fn sample_email(key: &str) -> Email {
Email {
id: key.to_string(),
account_id: "acct-1".to_string(),
subject: "Anything".to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1700000000,
body_plain: Some("Hello".to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
}
}
fn sample_placement(folder_id: &str, remote_id: &str) -> Placement {
Placement {
account_id: "acct-1".to_string(),
folder_id: folder_id.to_string(),
uid_validity: "1".to_string(),
remote_id: remote_id.to_string(),
read: false,
}
}
fn rule_where_folder_is(folder: &str) -> FilterRule {
let nsql = format!("WHERE folder = '{folder}' ACTION MARK READ");
crate::parser::NsqlParser::parse_rule("Folder Filter", 1, &nsql).unwrap()
}
fn rule_where_account_is(account: &str) -> FilterRule {
let nsql = format!("WHERE account = '{account}' ACTION MARK READ");
crate::parser::NsqlParser::parse_rule("Account Filter", 1, &nsql).unwrap()
}
#[test]
fn a_folder_condition_matches_only_the_placement_it_is_evaluated_against() {
let engine = FilterEngine::new(vec![rule_where_folder_is("INBOX")]).unwrap();
let email = sample_email("key-1");
let inbox = sample_placement("INBOX", "5");
let archive = sample_placement("Archive", "9");
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &email,
placement: &inbox
})
.len(),
1,
"the INBOX placement matches"
);
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &email,
placement: &archive
})
.len(),
0,
"the same message's Archive placement does not"
);
}
#[test]
fn an_account_condition_reads_the_placement_account() {
let engine = FilterEngine::new(vec![rule_where_account_is("acct-1")]).unwrap();
let email = sample_email("key-1");
let mine = sample_placement("INBOX", "5");
let theirs = Placement {
account_id: "acct-2".into(),
..sample_placement("INBOX", "5")
};
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &email,
placement: &mine
})
.len(),
1
);
assert_eq!(
engine
.evaluate(PlacedEmail {
email: &email,
placement: &theirs
})
.len(),
0
);
}
}
+4
View File
@@ -92,6 +92,10 @@ impl NsqlValidator {
"header field conditions are not yet supported: message headers are not indexed"
.to_string(),
)),
// `Folder` and `Account` are validated the same as the
// content fields here: only the value's shape (string or
// string list) matters at this pass. At evaluation time they
// read a placement's coordinates, not the message's content.
FilterField::Subject
| FilterField::From
| FilterField::To
+40 -10
View File
@@ -1,10 +1,28 @@
//! Protocol-agnostic async mail backend trait definitions.
use async_trait::async_trait;
use nuncio_core::model::{Attachment, Email, Folder};
use nuncio_core::model::{Attachment, Email, Folder, IdentitySource, Placement, PlacementKey};
use crate::parser::MailError;
/// One message as a backend surfaced it: identity, content, and the mailbox
/// occupancy it was found in.
///
/// Backends emit this rather than a bare [`Email`] because a fetch observes two
/// separate facts at once: *which message this is* -- folder-independent, and
/// the same in every mailbox that holds a copy -- and *where this pass found
/// it*, which is per-mailbox and changes under a move. Collapsing them into one
/// record is what made a moved message read as a new one.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PlacedMessage {
/// Identity and content. `email.id` is the derived message key.
pub email: Email,
/// Which precedence tier produced `email.id`.
pub source: IdentitySource,
/// The occupancy this fetch found it in.
pub placement: Placement,
}
/// The remote change a [`RemoteMutationSpec`] applies to one addressed message.
///
/// This is the closed vocabulary of genuinely-remote filter actions that the
@@ -72,21 +90,28 @@ pub struct RemoteMutationSpec {
/// or deleted by any other client -- another Nuncio daemon, a phone, webmail --
/// simply stays in the local store forever. Expressing absence is the whole
/// point of the type.
///
/// Absence is reported as [`PlacementKey`]s, not message keys, because what a
/// folder stops mentioning is an *occupancy*. The same message may still sit in
/// another mailbox, and deleting the message on the strength of one folder's
/// silence would destroy a copy the user still has. A message goes only when
/// its last placement does.
#[derive(Debug, Default, Clone)]
pub struct FolderChanges {
/// Messages to store, whether newly arrived or changed.
pub upserts: Vec<Email>,
/// Surrogate ids the server **explicitly reported as gone**, via QRESYNC
/// Messages to store, whether newly arrived or changed, each with the
/// occupancy this pass found it in.
pub upserts: Vec<PlacedMessage>,
/// Occupancies the server **explicitly reported as gone**, via QRESYNC
/// `VANISHED`. Always safe to delete.
pub removals: Vec<String>,
/// Surrogate ids of everything the folder currently holds, when this pass
/// enumerated the complete UID set. The caller may delete anything it
/// stores for this folder that is absent from this list.
pub removals: Vec<PlacementKey>,
/// Every occupancy the folder currently holds, when this pass enumerated
/// the complete UID set. The caller may delete any placement it stores for
/// this folder that is absent from this list.
///
/// `None` and `Some(vec![])` mean different things, and conflating them
/// deletes a mailbox. `None` is "this pass was incremental and cannot
/// speak to absence"; `Some(vec![])` is "the folder is genuinely empty".
pub present: Option<Vec<String>>,
pub present: Option<Vec<PlacementKey>>,
/// The checkpoint to resume from next time.
pub next_state: String,
}
@@ -158,11 +183,16 @@ pub trait MailBackend: Send + Sync {
///
/// Provided rather than required so no implementation can satisfy the
/// trait by supplying this and quietly never reporting a removal.
///
/// Every message comes back as a [`PlacedMessage`] so the caller can tell
/// a message it already stores in another folder from one it has never
/// seen: the derived key is the same in both folders, only the placement
/// differs.
async fn sync_messages(
&self,
folder_id: &str,
since_state: Option<&str>,
) -> Result<(Vec<Email>, String), MailError> {
) -> Result<(Vec<PlacedMessage>, String), MailError> {
let changes = self.sync_changes(folder_id, since_state).await?;
Ok((changes.upserts, changes.next_state))
}
+579 -102
View File
@@ -1,7 +1,7 @@
//! IMAP4rev1 dual-socket connection engine and IDLE push stream manager.
use async_trait::async_trait;
use nuncio_core::model::{Email, Folder};
use nuncio_core::model::{Email, Folder, MessageIdentity, Placement, PlacementKey, RemoteIdentity};
use nuncio_core::TlsMode;
use std::fmt::Debug;
use std::pin::Pin;
@@ -52,7 +52,10 @@ const MAX_MESSAGES_PER_FETCH_BATCH: usize = 200;
/// would ever accept is pointless to transfer only to reject.
const MAX_MESSAGE_BODY_BYTES: u32 = 25 * 1024 * 1024;
use crate::backend::{MailBackend, MutationOutcome, RemoteMutationKind, RemoteMutationSpec};
use crate::backend::{
FolderChanges, MailBackend, MutationOutcome, PlacedMessage, RemoteMutationKind,
RemoteMutationSpec,
};
use crate::parser::{MailError, MimeParserAdapter};
/// Recover a raw IMAP UID from a [`RemoteMutationSpec::remote_id`] (a decimal
@@ -783,7 +786,8 @@ impl ImapEngine {
}
/// Fetch and parse the messages in a folder over an active IMAP session,
/// returning the fetched messages and the new sync checkpoint.
/// returning each message with the occupancy it was found in, plus the new
/// sync checkpoint.
///
/// `since_state` is the checkpoint returned by a previous sync of this
/// folder, encoded as `"{uidvalidity}:{uidnext}"` (see [`CHECKPOINT_DELIM`]).
@@ -799,7 +803,7 @@ impl ImapEngine {
folder_id: &str,
since_state: Option<&str>,
session: &mut async_imap::Session<S>,
) -> Result<crate::backend::FolderChanges, MailError>
) -> Result<FolderChanges, MailError>
where
S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static,
{
@@ -818,15 +822,21 @@ impl ImapEngine {
/// exercise the multi-batch path with a small window instead of a huge
/// fixture.
///
/// The sync runs in two phases so the transient working set stays bounded
/// regardless of mailbox size:
/// The sync runs in two phases:
/// 1. A cheap enumeration ([`build_enumerate_command_query`]) reads every
/// in-range message's UID and `RFC822.SIZE` with NO body. Messages at or
/// below [`MAX_MESSAGE_BODY_BYTES`] are queued for a full-body fetch;
/// oversized ones are queued for a headers-only fetch and WARN-logged.
/// 2. Each queue is drained in explicit UID-set batches of at most
/// `batch_size`, so the full bodies held in memory at once never exceed
/// one batch.
/// `batch_size`, so a single `UID FETCH` request never asks the server
/// for more than that many bodies at a time.
///
/// Batching bounds the size of each *request*, not the memory this function
/// holds: the parsed messages accumulate into one `Vec` across every batch,
/// so the returned [`FolderChanges`] carries the whole in-range set and the
/// peak working set still scales with the mailbox. That is the shape a
/// caller receiving `FolderChanges` depends on, so anything downstream must
/// be sized for a whole folder rather than a batch.
#[tracing::instrument(
skip(self, session),
fields(account_id = %self.account_id, folder_id = %folder_id),
@@ -838,7 +848,7 @@ impl ImapEngine {
since_state: Option<&str>,
session: &mut async_imap::Session<S>,
batch_size: usize,
) -> Result<crate::backend::FolderChanges, MailError>
) -> Result<FolderChanges, MailError>
where
S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static,
{
@@ -968,16 +978,44 @@ impl ImapEngine {
.unwrap_or_else(|| "full-resync-required".to_string()),
};
let removals: Vec<String> = plan
.vanished_uids
.iter()
.map(|uid| self.surrogate_for(folder_id, server_uid_validity, *uid))
.collect();
let present = plan.present_uids.as_ref().map(|uids| {
uids.iter()
.map(|uid| self.surrogate_for(folder_id, server_uid_validity, *uid))
.collect()
});
// Absence can only be reported against a known UIDVALIDITY generation.
//
// `[UIDVALIDITY]` is REQUIRED on a successful SELECT (RFC 3501 section
// 6.3.1), so this should not happen -- but a server that omits it would
// otherwise produce keys carrying an empty generation, and NONE of them
// would match placements stored under a real one. Under the documented
// contract ("delete any placement absent from `present`") that reads as
// "the folder holds nothing", and the caller deletes the whole mailbox.
// Saying nothing is the only safe answer, and it mirrors the checkpoint
// above, which likewise refuses to write a validity-less value.
let (removals, present) = match server_uid_validity {
Some(_) => {
// Both are occupancies, not messages: a UID that vanished from
// this folder says nothing about copies of the same message
// elsewhere.
let removals: Vec<PlacementKey> = plan
.vanished_uids
.iter()
.map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid))
.collect();
let present = plan.present_uids.as_ref().map(|uids| {
uids.iter()
.map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid))
.collect()
});
(removals, present)
}
None => {
tracing::warn!(
folder_id,
vanished = plan.vanished_uids.len(),
enumerated = plan.present_uids.as_ref().map_or(0, Vec::len),
"mailbox reported no UIDVALIDITY, so this pass cannot key an occupancy and \
reports no absence; upserts are still surfaced"
);
(Vec::new(), None)
}
};
tracing::info!(
account_id = %self.account_id,
@@ -985,13 +1023,13 @@ impl ImapEngine {
rung = ?plan.rung,
fetched = emails.len(),
removed = removals.len(),
enumerated = plan.present_uids.as_ref().map_or(0, Vec::len),
enumerated = present.as_ref().map_or(0, Vec::len),
oversized = oversized_uids.len(),
skipped = skipped_no_uid,
"imap folder sync complete"
);
Ok(crate::backend::FolderChanges {
Ok(FolderChanges {
upserts: emails,
removals,
present,
@@ -1179,45 +1217,126 @@ impl ImapEngine {
Ok(uids)
}
/// Surrogate id for a UID in this account/folder/UIDVALIDITY scope, so
/// removals name messages the same way stored rows do.
fn surrogate_for(&self, folder_id: &str, uid_validity: Option<u32>, uid: u32) -> String {
let validity = uid_validity.map(|v| v.to_string()).unwrap_or_default();
Email::surrogate_id(&self.account_id, folder_id, &validity, &uid.to_string())
/// The placement key a UID addresses in this account/folder/UIDVALIDITY
/// scope, so absence reports name occupancies the same way stored rows do.
///
/// Deliberately NOT a message key. A UID is only ever evidence about one
/// mailbox: `VANISHED` says this folder no longer holds this UID, not that
/// the message is gone -- another mailbox may still hold a copy. It also
/// could not be a message key even in principle, because the key a message
/// is stored under depends on which identity tier the server supported,
/// and a UID cannot reconstruct that.
fn placement_key_for(
&self,
folder_id: &str,
uid_validity: Option<u32>,
uid: u32,
) -> PlacementKey {
PlacementKey {
account_id: self.account_id.clone(),
folder_id: folder_id.to_string(),
uid_validity: uid_validity.map(|v| v.to_string()).unwrap_or_default(),
remote_id: uid.to_string(),
}
}
/// Build an [`Email`] from a full-body (or envelope-only) FETCH item. When
/// the item carries a `BODY[]` it is parsed as a full MIME message; when it
/// does not (a server that returned only the ENVELOPE), a minimal record is
/// synthesised from the envelope so the message is still surfaced honestly.
/// Build the identity and placement for one fetched message.
///
/// `EMAILID` and `X-GM-MSGID` are only present when the server advertised
/// `OBJECTID` / `X-GM-EXT-1` and the FETCH asked for them; when neither is
/// available the pair of `Message-ID` and content hash carries the identity,
/// and only if both are missing does this fall back to the folder-scoped
/// surrogate -- which is the one tier that does not survive a move.
///
/// The server-supplied hints arrive as one [`RemoteIdentity`] rather than as
/// four loose arguments: they are already a single value at every call site,
/// and spreading them out buys no clarity while pushing this function past
/// the argument-count lint.
///
/// In practice every IMAP call site passes `email_id: None` and
/// `gm_msgid: None` today, because the FETCH never asks for them, so this
/// engine reaches only the `Message-ID`+content tier or the surrogate. The
/// JMAP engine enters at the top tier, which means an IMAP-derived key and
/// a JMAP-derived key for the *same* message will not match. Syncing one
/// account over both engines does not converge on a shared identity yet.
fn place(
&self,
folder_id: &str,
uid_validity: &str,
remote_id: &str,
remote: RemoteIdentity<'_>,
read: bool,
) -> (MessageIdentity, Placement) {
let identity =
Email::derive_message_key(&self.account_id, remote, folder_id, uid_validity, remote_id);
let placement = Placement {
account_id: self.account_id.clone(),
folder_id: folder_id.to_string(),
uid_validity: uid_validity.to_string(),
remote_id: remote_id.to_string(),
read,
};
(identity, placement)
}
/// Build a [`PlacedMessage`] from a full-body (or envelope-only) FETCH item.
/// When the item carries a `BODY[]` it is parsed as a full MIME message;
/// when it does not (a server that returned only the ENVELOPE), a minimal
/// record is synthesised from the envelope so the message is still surfaced
/// honestly.
fn build_email_from_fetch(
&self,
folder_id: &str,
server_uid_validity: Option<u32>,
fetch_data: &async_imap::types::Fetch,
) -> Result<Email, MailError> {
) -> Result<PlacedMessage, MailError> {
let uid_num = fetch_data.uid.unwrap_or(0);
// The message is addressed on the wire by its UID within the folder's
// UIDVALIDITY scope; the persisted id is an opaque surrogate hashed over
// both plus the account and folder, so the same UID in another
// folder/account can never collide.
// UIDVALIDITY scope. That pair, with the folder and account, is the
// *placement*; the message's own identity is derived separately so that
// moving it between folders does not make it a different message.
let remote_id = uid_num.to_string();
let uid_validity = server_uid_validity
.map(|v| v.to_string())
.unwrap_or_default();
let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id);
let is_read = fetch_data
.flags()
.any(|flag| matches!(flag, async_imap::types::Flag::Seen));
// EMAILID (RFC 8474) and X-GM-MSGID are not requested by the current
// FETCH, so identity rests on the Message-ID/content pair where the
// message carried a Message-ID and on the folder-scoped surrogate
// otherwise.
//
// Adding those FETCH items is not a local change. Every already-stored
// message would derive a DIFFERENT key on its next resync, so each
// re-reported placement repoints to the new key -- and the old message
// row, now referenced by nothing, is never reaped. Its decrypted body
// stays in the plaintext FTS index forever, which is exactly the leak
// the reap on placement removal exists to prevent. The FETCH items can
// only be requested once the repoint path reaps the message it
// abandons; until then, requesting them leaks plaintext.
if let Some(raw_bytes) = fetch_data.body() {
let mut email =
MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, raw_bytes)?;
email.remote_id = remote_id;
email.uid_validity = uid_validity;
email.read = is_read;
return Ok(email);
let mut email = MimeParserAdapter::parse_mime(&self.account_id, raw_bytes)?;
let (identity, placement) = self.place(
folder_id,
&uid_validity,
&remote_id,
RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: email.message_id.as_deref(),
content_hash: email.content_hash.as_deref(),
},
is_read,
);
email.id = identity.key;
return Ok(PlacedMessage {
email,
source: identity.source,
placement,
});
}
let subject = fetch_data
@@ -1241,24 +1360,51 @@ impl ImapEngine {
let received_at = fetch_data.internal_date().map_or(0, |dt| dt.timestamp());
Ok(Email {
id: email_id,
account_id: self.account_id.clone(),
folder_id: folder_id.to_string(),
remote_id,
uid_validity,
subject,
sender,
recipient,
received_at,
read: is_read,
body_plain: None,
body_html: None,
attachments: Vec::new(),
// No body was returned, so there are no octets to hash -- but the
// ENVELOPE carries `message-id` (RFC 3501 section 7.4.2), so identity
// capture does not have to be skipped on this path.
let message_id = fetch_data
.envelope()
.and_then(|env| env.message_id.as_ref())
.and_then(|raw| Email::normalize_message_id(&String::from_utf8_lossy(raw)));
// With no octets there is no content hash, and the `Message-ID` alone
// is never enough to key on, so this path lands on the surrogate tier.
// That is the honest answer: nothing was fetched that could prove two
// folders hold the same message.
let (identity, placement) = self.place(
folder_id,
&uid_validity,
&remote_id,
RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: message_id.as_deref(),
content_hash: None,
},
is_read,
);
Ok(PlacedMessage {
email: Email {
id: identity.key,
account_id: self.account_id.clone(),
subject,
sender,
recipient,
received_at,
body_plain: None,
body_html: None,
attachments: Vec::new(),
message_id,
content_hash: None,
},
source: identity.source,
placement,
})
}
/// Build an [`Email`] for a message whose body exceeded
/// Build a [`PlacedMessage`] for a message whose body exceeded
/// [`MAX_MESSAGE_BODY_BYTES`]: its headers (`BODY[HEADER]`) are parsed for
/// honest metadata (subject/sender/date) but the oversized body is left
/// unfetched, so the record carries an EMPTY body rather than a fabricated
@@ -1269,30 +1415,50 @@ impl ImapEngine {
folder_id: &str,
server_uid_validity: Option<u32>,
fetch_data: &async_imap::types::Fetch,
) -> Email {
) -> PlacedMessage {
let uid_num = fetch_data.uid.unwrap_or(0);
let remote_id = uid_num.to_string();
let uid_validity = server_uid_validity
.map(|v| v.to_string())
.unwrap_or_default();
let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id);
let is_read = fetch_data
.flags()
.any(|flag| matches!(flag, async_imap::types::Flag::Seen));
if let Some(header_bytes) = fetch_data.header() {
if let Ok(mut email) =
MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, header_bytes)
{
if let Ok(mut email) = MimeParserAdapter::parse_mime(&self.account_id, header_bytes) {
// A headers-only parse yields no body, but be explicit that the
// oversized body was intentionally not fetched.
email.remote_id = remote_id;
email.uid_validity = uid_validity;
email.read = is_read;
email.body_plain = None;
email.body_html = None;
email.attachments = Vec::new();
return email;
// `parse_mime` hashed the header section it was handed, but
// `content_hash` is defined over a message's FULL octets.
// Keeping that value would publish a hash that silently means
// something else, so drop it; the `Message-ID` it recovered
// from those headers is still correct and is kept. Dropping it
// also costs this message the content tier -- a `Message-ID`
// never keys on its own -- so an oversized message identifies
// by its folder-scoped surrogate until the octets are fetched.
email.content_hash = None;
let (identity, placement) = self.place(
folder_id,
&uid_validity,
&remote_id,
RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: email.message_id.as_deref(),
content_hash: None,
},
is_read,
);
email.id = identity.key;
return PlacedMessage {
email,
source: identity.source,
placement,
};
}
}
@@ -1300,20 +1466,31 @@ impl ImapEngine {
// `build_email_from_fetch` cannot error on this path (no body to parse),
// but if it ever did, surface a minimal honest record rather than panic.
self.build_email_from_fetch(folder_id, server_uid_validity, fetch_data)
.unwrap_or_else(|_| Email {
id: email_id,
account_id: self.account_id.clone(),
folder_id: folder_id.to_string(),
remote_id,
uid_validity,
subject: "No Subject".to_string(),
sender: "unknown@nuncio.mx".to_string(),
recipient: "me@nuncio.mx".to_string(),
received_at: 0,
read: is_read,
body_plain: None,
body_html: None,
attachments: Vec::new(),
.unwrap_or_else(|_| {
let (identity, placement) = self.place(
folder_id,
&uid_validity,
&remote_id,
RemoteIdentity::default(),
is_read,
);
PlacedMessage {
email: Email {
id: identity.key,
account_id: self.account_id.clone(),
subject: "No Subject".to_string(),
sender: "unknown@nuncio.mx".to_string(),
recipient: "me@nuncio.mx".to_string(),
received_at: 0,
body_plain: None,
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
},
source: identity.source,
placement,
}
})
}
@@ -1325,7 +1502,7 @@ impl ImapEngine {
&self,
folder_id: &str,
since_state: Option<&str>,
) -> Result<crate::backend::FolderChanges, MailError> {
) -> Result<FolderChanges, MailError> {
let (username, password) = match (&self.username, &self.password) {
(Some(u), Some(p)) => (u.as_str(), p.as_str()),
_ => {
@@ -1793,7 +1970,7 @@ impl MailBackend for ImapEngine {
&self,
folder_id: &str,
since_state: Option<&str>,
) -> Result<crate::backend::FolderChanges, MailError> {
) -> Result<FolderChanges, MailError> {
self.sync_folder_messages(folder_id, since_state).await
}
@@ -1808,6 +1985,7 @@ impl MailBackend for ImapEngine {
#[cfg(test)]
mod tests {
use super::*;
use nuncio_core::model::IdentitySource;
use tracing_test::traced_test;
#[tokio::test]
@@ -2193,17 +2371,25 @@ mod tests {
}
/// The RFC822 body of a scripted fixture message.
///
/// Carries a `Message-ID` so the full-body path reaches the content tier.
/// That is what makes the headers-only path's tier meaningful to assert:
/// the same message identifies differently there, and only because the
/// header-section hash is deliberately discarded.
fn fixture_body(uid: u32) -> String {
format!(
"Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\
Message-ID: <msg{uid}@example.test>\r\n\
To: me@example.test\r\n\r\nBody of message {uid}.\r\n"
)
}
/// The header block (no body) of a scripted fixture message.
/// The header block (no body) of a scripted fixture message. Same headers
/// as [`fixture_body`], including the `Message-ID`.
fn fixture_header(uid: u32) -> String {
format!(
"Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\
Message-ID: <msg{uid}@example.test>\r\n\
To: me@example.test\r\n\r\n"
)
}
@@ -2237,13 +2423,17 @@ mod tests {
/// and records every `UID FETCH` command line the client issued so a test
/// can assert on the exact batching. Returns the synced emails, the new
/// checkpoint, and the captured `UID FETCH` command lines.
///
/// `uidvalidity` is an `Option` so a test can script the one thing RFC 3501
/// section 6.3.1 says a SELECT must never do -- omit `[UIDVALIDITY]` -- and
/// check the sync stays safe when a server does it anyway.
async fn run_batched_sync(
messages: Vec<(u32, u32)>,
uidvalidity: u32,
uidvalidity: Option<u32>,
uidnext: u32,
since_state: Option<String>,
batch_size: usize,
) -> (Vec<Email>, String, Vec<String>) {
) -> (FolderChanges, Vec<String>) {
let (client_io, server_io) = tokio::io::duplex(65536);
let fetch_cmds = Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let server_cmds = fetch_cmds.clone();
@@ -2305,9 +2495,13 @@ mod tests {
.write_all(format!("{tag} OK Logged in\r\n").as_bytes())
.await;
} else if upper.contains("SELECT") {
let validity_line = match uidvalidity {
Some(v) => format!("* OK [UIDVALIDITY {v}] ok\r\n"),
None => String::new(),
};
let resp = format!(
"* FLAGS (\\Seen)\r\n* {exists} EXISTS\r\n* 0 RECENT\r\n\
* OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\
{validity_line}* OK [UIDNEXT {uidnext}] ok\r\n\
{tag} OK [READ-WRITE] SELECT done\r\n"
);
let _ = io.write_all(resp.as_bytes()).await;
@@ -2331,12 +2525,11 @@ mod tests {
.sync_folder_messages_batched("INBOX", since_state.as_deref(), &mut session, batch_size)
.await
.expect("batched sync succeeds");
let (emails, checkpoint) = (changes.upserts, changes.next_state);
drop(session);
let _ = server.await;
let cmds = fetch_cmds.lock().expect("lock captured commands").clone();
(emails, checkpoint, cmds)
(changes, cmds)
}
/// The captured `UID FETCH` command lines that pulled FULL bodies (a
@@ -2358,10 +2551,11 @@ mod tests {
// `1:*` body fetch. The enumeration discovers the UIDs; the bodies are
// then fetched in explicit UID-set chunks of at most the batch size.
let messages: Vec<(u32, u32)> = (1..=5).map(|u| (u, 100)).collect();
let (emails, checkpoint, cmds) = run_batched_sync(messages, 1, 6, None, 2).await;
let (changes, cmds) = run_batched_sync(messages, Some(1), 6, None, 2).await;
let emails = &changes.upserts;
assert_eq!(emails.len(), 5, "every message must be synced");
assert_eq!(checkpoint, "v1:uidnext:1:6");
assert_eq!(changes.next_state, "v1:uidnext:1:6");
let body_cmds = body_fetch_commands(&cmds);
assert_eq!(
@@ -2391,7 +2585,8 @@ mod tests {
// normal messages get full bodies.
let huge = MAX_MESSAGE_BODY_BYTES + 1;
let messages: Vec<(u32, u32)> = vec![(1, 100), (2, huge), (3, 100)];
let (emails, _checkpoint, cmds) = run_batched_sync(messages, 1, 4, None, 200).await;
let (changes, cmds) = run_batched_sync(messages, Some(1), 4, None, 200).await;
let emails = &changes.upserts;
assert_eq!(emails.len(), 3, "no message may be silently dropped");
@@ -2416,27 +2611,54 @@ mod tests {
// body -- never a fabricated one.
let oversized = emails
.iter()
.find(|e| e.remote_id == "2")
.find(|e| e.placement.remote_id == "2")
.expect("oversized message must be present");
assert!(
oversized.body_plain.is_none() && oversized.body_html.is_none(),
oversized.email.body_plain.is_none() && oversized.email.body_html.is_none(),
"oversized message must carry no body, got: {oversized:?}"
);
assert!(
oversized.subject.contains('2'),
oversized.email.subject.contains('2'),
"oversized message must keep honest header metadata, got: {oversized:?}"
);
// The header section it WAS given hashes to something, but that hash
// covers only the headers, not the message. Publishing it would key the
// message on the wrong octets -- and then the same message would key
// differently once its body was finally fetched, resurrecting it as a
// duplicate. So the hash is dropped, which costs this message the
// content tier even though it carried a Message-ID.
assert!(
oversized.email.message_id.is_some(),
"the headers-only parse must still recover the Message-ID"
);
assert_eq!(
oversized.email.content_hash, None,
"a hash over only the header section must never be published"
);
assert_eq!(
oversized.source,
IdentitySource::Surrogate,
"without octets to hash, a Message-ID alone must not reach the content tier"
);
// The normal messages DID get full bodies.
for uid in ["1", "3"] {
let email = emails
.iter()
.find(|e| e.remote_id == uid)
.find(|e| e.placement.remote_id == uid)
.expect("normal message present");
assert!(
email.body_plain.is_some(),
email.email.body_plain.is_some(),
"normal message {uid} must have a body, got: {email:?}"
);
// The contrast that gives the assertion above its force: the same
// headers, plus the octets, DO reach the content tier.
assert_eq!(
email.source,
IdentitySource::MessageIdContent,
"a full-body fetch with a Message-ID must reach the content tier"
);
}
}
@@ -2447,24 +2669,24 @@ mod tests {
// checkpoint enumerates only the narrowed `{uidnext}:*` range (so a
// fire-once-per-new-message consumer never re-sees old mail).
let messages: Vec<(u32, u32)> = (1..=3).map(|u| (u, 100)).collect();
let (first, checkpoint, first_cmds) =
run_batched_sync(messages.clone(), 1, 4, None, 2).await;
assert_eq!(first.len(), 3);
let (first, first_cmds) = run_batched_sync(messages.clone(), Some(1), 4, None, 2).await;
let checkpoint = first.next_state.clone();
assert_eq!(first.upserts.len(), 3);
assert_eq!(checkpoint, "v1:uidnext:1:4");
assert!(
first_cmds.iter().any(|c| c.contains("1:*")),
"first sync enumerates the full range, got: {first_cmds:?}"
);
let (second, checkpoint2, second_cmds) =
run_batched_sync(messages, 1, 4, Some(checkpoint.clone()), 2).await;
let (second, second_cmds) =
run_batched_sync(messages, Some(1), 4, Some(checkpoint.clone()), 2).await;
assert_eq!(
second.len(),
second.upserts.len(),
0,
"no new mail arrived, so an incremental sync returns nothing"
);
assert_eq!(
checkpoint2, "v1:uidnext:1:4",
second.next_state, "v1:uidnext:1:4",
"checkpoint stays stable across an empty pass"
);
let second_bodies = body_fetch_ranges(&second_cmds);
@@ -3258,4 +3480,259 @@ mod tests {
);
Ok(())
}
/// Drive a folder sync over a scripted server that serves exactly ONE
/// message: `uid`, with `raw` as its full RFC822 octets and `seen`
/// controlling its `\Seen` flag. Holding the octets fixed while varying the
/// folder, UIDVALIDITY and UID is the shape of "the same message, found in
/// two mailboxes" -- which is what the identity/placement split exists for.
///
/// Passing `raw = None` scripts the envelope-only path instead: the server
/// answers the body fetch with an `ENVELOPE` and no `BODY[]`, which is what
/// a server that will not hand over octets looks like.
async fn run_single_message_sync(
folder_id: &str,
uidvalidity: u32,
uid: u32,
seen: bool,
raw: Option<&str>,
) -> PlacedMessage {
let (client_io, server_io) = tokio::io::duplex(65536);
let raw = raw.map(str::to_string);
let uidnext = uid.saturating_add(1);
let flags = if seen { "\\Seen" } else { "" };
let server = tokio::spawn(async move {
let mut io = server_io;
while let Some(line) = read_scripted_line(&mut io).await {
let tag = line.split_whitespace().next().unwrap_or("").to_string();
let upper = line.to_ascii_uppercase();
if upper.contains("UID FETCH") {
let size = raw.as_ref().map_or(64, String::len);
let resp = if !upper.contains("BODY.PEEK[]") {
// Enumeration pass: UID + size only, no body.
format!(
"* 1 FETCH (UID {uid} RFC822.SIZE {size})\r\n\
{tag} OK FETCH completed\r\n"
)
} else if let Some(raw) = raw.as_ref() {
format!(
"* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \
BODY[] {{{size}}}\r\n{raw})\r\n{tag} OK FETCH completed\r\n"
)
} else {
// A server that answers with metadata but no octets.
// ENVELOPE still carries `message-id` (RFC 3501
// section 7.4.2), so identity capture is not skipped --
// but there is nothing to hash.
format!(
"* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \
ENVELOPE (\"Mon, 1 Jan 2024 00:00:00 +0000\" \"Env subject\" \
((\"A\" NIL \"alice\" \"b.example\")) \
((\"A\" NIL \"alice\" \"b.example\")) \
((\"A\" NIL \"alice\" \"b.example\")) \
((\"B\" NIL \"bob\" \"b.example\")) \
NIL NIL NIL \"<env@b.example>\"))\r\n\
{tag} OK FETCH completed\r\n"
)
};
let _ = io.write_all(resp.as_bytes()).await;
} else if upper.contains("CAPABILITY") {
// The planner negotiates capabilities before selecting; a
// script that never answers this blocks the client forever.
let _ = io
.write_all(
format!(
"* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n"
)
.as_bytes(),
)
.await;
} else if upper.contains("LOGIN") {
let _ = io
.write_all(format!("{tag} OK Logged in\r\n").as_bytes())
.await;
} else if upper.contains("SELECT") {
let resp = format!(
"* FLAGS (\\Seen)\r\n* 1 EXISTS\r\n* 0 RECENT\r\n\
* OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\
{tag} OK [READ-WRITE] SELECT done\r\n"
);
let _ = io.write_all(resp.as_bytes()).await;
} else if upper.contains("LOGOUT") {
let _ = io
.write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes())
.await;
break;
}
}
});
let client = async_imap::Client::new(client_io);
let mut session = client
.login("user", "pass")
.await
.map_err(|(e, _)| e)
.expect("scripted login succeeds");
let engine = ImapEngine::new("acct-1", "example.test", 993);
let changes = engine
.sync_folder_messages_batched(folder_id, None, &mut session, 8)
.await
.expect("scripted sync succeeds");
drop(session);
let _ = server.await;
changes
.upserts
.into_iter()
.next()
.expect("the scripted message must be surfaced")
}
#[tokio::test]
async fn the_imap_backend_gives_one_identity_to_a_message_found_in_two_folders() {
// Byte-identical octets under two different UIDs in two different
// mailboxes -- an IMAP COPY, or a Gmail label. Identity comes from the
// Message-ID/content pair, so the derived key is ONE value while the
// placements stay distinct. Before the split this was two messages.
let raw = "Message-ID: <a@b.example>\r\nSubject: Hi\r\n\
From: alice@b.example\r\n\r\nbody\r\n";
let inbox = run_single_message_sync("INBOX", 42, 5, true, Some(raw)).await;
let archive = run_single_message_sync("Archive", 77, 9, false, Some(raw)).await;
assert_eq!(inbox.source, IdentitySource::MessageIdContent);
assert_eq!(archive.source, IdentitySource::MessageIdContent);
assert_eq!(
inbox.email.id, archive.email.id,
"the same octets in two folders must derive one message key"
);
assert_eq!(inbox.placement.account_id, "acct-1");
assert_eq!(inbox.placement.folder_id, "INBOX");
assert_eq!(inbox.placement.uid_validity, "42");
assert_eq!(inbox.placement.remote_id, "5");
assert_eq!(archive.placement.folder_id, "Archive");
assert_eq!(archive.placement.uid_validity, "77");
assert_eq!(archive.placement.remote_id, "9");
// IMAP `\Seen` is per-mailbox (RFC 3501 section 2.3.2), so read state
// belongs to the occupancy, not to the message.
assert!(inbox.placement.read);
assert!(!archive.placement.read);
}
#[tokio::test]
async fn a_message_with_no_message_id_stays_on_the_folder_scoped_surrogate() {
// RFC 5322 section 3.6.4 makes Message-ID a SHOULD, so a message may
// carry none. Without it there is no content tier, and the backend must
// report two honest folder-scoped keys rather than claim a stability
// the server gave it no basis for.
let raw = "Subject: No id\r\nFrom: alice@b.example\r\n\r\nbody\r\n";
let inbox = run_single_message_sync("INBOX", 42, 5, false, Some(raw)).await;
let archive = run_single_message_sync("Archive", 77, 9, false, Some(raw)).await;
assert_eq!(inbox.source, IdentitySource::Surrogate);
assert_ne!(
inbox.email.id, archive.email.id,
"a surrogate is folder-scoped and must not be claimed to survive a move"
);
}
#[tokio::test]
async fn an_envelope_only_fetch_keeps_its_message_id_but_stays_on_the_surrogate() {
// A server that returns metadata and no octets. ENVELOPE carries
// `message-id` (RFC 3501 section 7.4.2) so the header is still
// captured, but there is nothing to hash -- and a Message-ID alone is
// public and forgeable, so it must never key storage by itself.
let placed = run_single_message_sync("INBOX", 42, 5, false, None).await;
assert_eq!(placed.email.subject, "Env subject");
assert_eq!(placed.email.message_id.as_deref(), Some("env@b.example"));
assert_eq!(
placed.email.content_hash, None,
"no octets were fetched, so there is nothing honest to hash"
);
assert_eq!(
placed.source,
IdentitySource::Surrogate,
"an envelope-only fetch must not reach the content tier"
);
assert_eq!(placed.placement.remote_id, "5");
assert_eq!(placed.placement.uid_validity, "42");
}
#[traced_test]
#[tokio::test]
async fn a_select_without_uidvalidity_reports_no_absence_at_all() {
// RFC 3501 section 6.3.1 makes `[UIDVALIDITY]` REQUIRED on SELECT, so a
// server omitting it is broken -- but the failure mode is a wiped
// mailbox, not a bad sync. Every placement key this pass could build
// would carry an empty generation and match nothing the store holds,
// and under the `present` contract that reads as "the folder is empty".
// The pass must therefore decline to speak about absence, while still
// surfacing what it fetched.
let messages: Vec<(u32, u32)> = (1..=2).map(|u| (u, 100)).collect();
let (changes, _cmds) = run_batched_sync(messages, None, 3, None, 200).await;
assert_eq!(
changes.upserts.len(),
2,
"the messages themselves are still surfaced"
);
assert_eq!(
changes.present, None,
"a pass that cannot key an occupancy must not claim to know what is present"
);
assert!(
changes.removals.is_empty(),
"nor may it report a removal, got: {:?}",
changes.removals
);
// And it must say why, rather than failing silently.
assert!(logs_contain("mailbox reported no UIDVALIDITY"));
// The same pass against a compliant server DOES report presence, so the
// assertions above are about the missing UIDVALIDITY and nothing else.
let messages: Vec<(u32, u32)> = (1..=2).map(|u| (u, 100)).collect();
let (compliant, _cmds) = run_batched_sync(messages, Some(7), 3, None, 200).await;
let present = compliant
.present
.expect("a compliant SELECT must let the pass report presence");
assert_eq!(present.len(), 2);
assert!(present.iter().all(|k| k.uid_validity == "7"));
}
#[tokio::test]
async fn a_message_fetched_from_two_folders_keeps_one_identity() {
// Both folders return the same Message-ID and the same octets, so the
// content tier must produce one key from two different UIDs.
let raw = b"Message-ID: <a@b.example>\r\nSubject: Hi\r\n\r\nbody";
let hash = Email::content_hash_of(raw);
let inbox = Email::derive_message_key(
"acct-1",
RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: Some("a@b.example"),
content_hash: Some(&hash),
},
"INBOX",
"42",
"5",
);
let archive = Email::derive_message_key(
"acct-1",
RemoteIdentity {
email_id: None,
gm_msgid: None,
message_id: Some("a@b.example"),
content_hash: Some(&hash),
},
"Archive",
"77",
"9",
);
assert_eq!(inbox.key, archive.key);
assert_eq!(inbox.source, IdentitySource::MessageIdContent);
}
}
+91 -26
View File
@@ -1,19 +1,22 @@
//! JMAP (RFC 8620 / RFC 8621) protocol engine, session discovery, and differential update parser.
use async_trait::async_trait;
use nuncio_core::model::{Email, Folder};
use nuncio_core::model::{Email, Folder, Placement, RemoteIdentity};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use crate::backend::{MailBackend, MutationOutcome, RemoteMutationKind, RemoteMutationSpec};
use crate::backend::{
FolderChanges, MailBackend, MutationOutcome, PlacedMessage, RemoteMutationKind,
RemoteMutationSpec,
};
use crate::parser::MailError;
/// Stand-in for the UIDVALIDITY scope on JMAP, which has no such concept: a
/// JMAP Email object id is already globally stable within its account (RFC
/// 8621), so a fixed sentinel keeps the surrogate-id computation uniform with
/// IMAP without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats
/// this non-numeric value as "no validity to compare" and is never engaged for
/// a JMAP mutation, which addresses messages by object id.
/// JMAP Email object id is already stable within its account (RFC 8621), so a
/// fixed sentinel keeps a placement key the same shape on both protocols
/// without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats this
/// non-numeric value as "no validity to compare" and is never engaged for a
/// JMAP mutation, which addresses messages by object id.
const JMAP_UID_VALIDITY_SENTINEL: &str = "jmap";
/// JMAP Session Object (RFC 8620 Section 2).
@@ -92,6 +95,10 @@ pub struct JmapEmail {
pub received_at: Option<i64>,
pub is_unread: Option<bool>,
pub body_snippet: Option<String>,
/// RFC 8621 models `messageId` as a list, because a message may carry more
/// than one `Message-ID` header. The first entry is the one used.
#[serde(default)]
pub message_id: Option<Vec<String>>,
}
/// JMAP email address object.
@@ -184,7 +191,7 @@ impl JmapEngine {
{
"accountId": account_id,
"ids": ids,
"properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet"]
"properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet", "messageId"]
},
"c1"
]
@@ -364,11 +371,12 @@ impl JmapEngine {
}
}
/// Parse raw JMAP `Email/get` JSON response payload into domain [`Email`] list and new state string.
/// Parse a raw JMAP `Email/get` JSON response payload into a list of
/// [`PlacedMessage`] and the new state string.
pub fn parse_email_get_response(
&self,
raw_json: &str,
) -> Result<(Vec<Email>, String), MailError> {
) -> Result<(Vec<PlacedMessage>, String), MailError> {
let val: Value = serde_json::from_str(raw_json)
.map_err(|e| MailError::ParseFailed(format!("invalid JMAP JSON response: {e}")))?;
let payload = Self::extract_method_response_payload(&val)?;
@@ -394,30 +402,58 @@ impl JmapEngine {
.unwrap_or_else(|| "me@nuncio.mx".to_string());
let folder_id = "inbox".to_string();
// The JMAP object id is the protocol-native id; the persisted
// id is an opaque surrogate hashed over it plus the account,
// folder, and (sentinel) scope so it stays uniform with IMAP.
let remote_id = item.id;
let id = Email::surrogate_id(
let message_id = item
.message_id
.as_ref()
.and_then(|ids| ids.first())
.and_then(|raw| Email::normalize_message_id(raw));
// A JMAP Email id is server-assigned and stable across mailboxes
// within the account, which is the same guarantee RFC 8474
// EMAILID gives, so it enters at the top tier rather than
// through the surrogate.
let identity = Email::derive_message_key(
&self.account_id,
RemoteIdentity {
email_id: Some(&remote_id),
gm_msgid: None,
message_id: message_id.as_deref(),
// `Email/get` returns a `bodySnippet`, not the
// message's octets, so there is nothing to hash here
// without an extra download per message.
content_hash: None,
},
&folder_id,
JMAP_UID_VALIDITY_SENTINEL,
&remote_id,
);
Email {
id,
let placement = Placement {
account_id: self.account_id.clone(),
folder_id,
remote_id,
// JMAP has no UIDVALIDITY; the sentinel keeps the placement
// key the same shape as IMAP's without implying a guard.
uid_validity: JMAP_UID_VALIDITY_SENTINEL.to_string(),
subject: item.subject.unwrap_or_else(|| "No Subject".to_string()),
sender,
recipient,
received_at: item.received_at.unwrap_or(0),
remote_id: remote_id.clone(),
read: !item.is_unread.unwrap_or(false),
body_plain: item.body_snippet,
body_html: None,
attachments: Vec::new(),
};
PlacedMessage {
email: Email {
id: identity.key,
account_id: self.account_id.clone(),
subject: item.subject.unwrap_or_else(|| "No Subject".to_string()),
sender,
recipient,
received_at: item.received_at.unwrap_or(0),
body_plain: item.body_snippet,
body_html: None,
attachments: Vec::new(),
message_id,
content_hash: None,
},
source: identity.source,
placement,
}
})
.collect();
@@ -565,7 +601,7 @@ impl MailBackend for JmapEngine {
&self,
folder_id: &str,
_since_state: Option<&str>,
) -> Result<crate::backend::FolderChanges, MailError> {
) -> Result<FolderChanges, MailError> {
if !self.has_credentials() {
return Err(MailError::AuthError(
"JMAP message sync requires credentials".to_string(),
@@ -589,7 +625,7 @@ impl MailBackend for JmapEngine {
"jmap message sync complete"
);
Ok(crate::backend::FolderChanges {
Ok(FolderChanges {
upserts: emails,
removals: Vec::new(),
// `Email/query` was filtered to one mailbox, but this engine does
@@ -722,6 +758,35 @@ mod tests {
assert_eq!(new_state, "s-2");
}
#[test]
fn parse_email_get_response_captures_normalized_message_id() {
// RFC 8621 models `messageId` as a list; the first entry is taken and
// normalized to the same canonical form the IMAP paths produce, so two
// engines syncing one account via different protocols agree.
let raw = r#"{
"methodResponses": [
["Email/get", {"state": "s-1", "list": [
{"id": "m-1", "subject": "Hi", "messageId": ["<Abc.123@mail.nuncio.mx>"]},
{"id": "m-2", "subject": "No id"}
]}, "c1"]
]
}"#;
let engine = JmapEngine::new("acct-1");
let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get");
assert_eq!(
emails[0].email.message_id.as_deref(),
Some("Abc.123@mail.nuncio.mx")
);
assert_eq!(
emails[1].email.message_id, None,
"an absent messageId must stay None, not become empty"
);
// `Email/get` returns a snippet, never the octets, so there is nothing
// honest to hash on this path.
assert_eq!(emails[0].email.content_hash, None);
}
#[test]
fn parse_invalid_jmap_json_fails() {
let engine = JmapEngine::new("acct-1");
+1 -1
View File
@@ -18,7 +18,7 @@ pub mod smtp;
pub mod test_server;
pub use backend::{
FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage,
FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage, PlacedMessage,
RemoteMutationKind, RemoteMutationSpec,
};
pub use imap::{IdleSocketState, ImapDualSocketManager, ImapEngine};
+170 -28
View File
@@ -1,11 +1,12 @@
//! Deterministic mock mail backend for offline testing and integration verification.
use async_trait::async_trait;
use nuncio_core::model::{Email, Folder};
use nuncio_core::model::{Email, Folder, IdentitySource, Placement, PlacementKey};
use std::sync::{Arc, Mutex};
use crate::backend::{
MailBackend, MessageSender, MutationOutcome, OutboundMessage, RemoteMutationSpec,
FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage, PlacedMessage,
RemoteMutationSpec,
};
use crate::parser::MailError;
@@ -13,7 +14,7 @@ use crate::parser::MailError;
#[derive(Debug, Clone, Default)]
pub struct MockMailBackend {
folders: Arc<Mutex<Vec<Folder>>>,
messages: Arc<Mutex<Vec<Email>>>,
messages: Arc<Mutex<Vec<PlacedMessage>>>,
should_fail: Arc<Mutex<bool>>,
/// Every `since_state` argument this mock's `sync_messages` was called
/// with, in call order, so tests can prove a caller threads the returned
@@ -25,6 +26,10 @@ pub struct MockMailBackend {
/// so a daemon E2E can prove the outbox executor genuinely invoked the
/// backend op (not a fabricated completion).
applied_mutations: Arc<Mutex<Vec<RemoteMutationSpec>>>,
/// Occupancies this mock reports as explicitly gone, the way QRESYNC
/// `VANISHED` does. Handed back on the pass for the folder each key names,
/// so a test can prove a caller acts on a server-confirmed removal.
removals: Arc<Mutex<Vec<PlacementKey>>>,
}
impl MockMailBackend {
@@ -56,6 +61,15 @@ impl MockMailBackend {
.unwrap_or_default()
}
/// Stage occupancies this mock reports as explicitly gone, as a QRESYNC
/// `VANISHED` would. Each key is handed back on the pass for the folder it
/// names, so staging a removal for `Archive` does not leak into `INBOX`.
pub fn set_removals(&self, removals: Vec<PlacementKey>) {
if let Ok(mut guard) = self.removals.lock() {
*guard = removals;
}
}
/// Configure the mock to simulate network failure errors.
pub fn set_should_fail(&self, fail: bool) {
if let Ok(mut flag) = self.should_fail.lock() {
@@ -70,13 +84,81 @@ impl MockMailBackend {
}
}
/// Add a mock email message to the storage.
pub fn add_message(&self, email: Email) {
/// Add a mock message to the storage, as the occupancy a backend would
/// have found it in. Taking a [`PlacedMessage`] rather than a bare `Email`
/// lets a test stage the same message key in two folders and prove the
/// caller treats it as one message in two places.
pub fn add_message(&self, message: PlacedMessage) {
if let Ok(mut guard) = self.messages.lock() {
guard.push(email);
guard.push(message);
}
}
/// Account every message staged by [`Self::shared_message_in`] and
/// [`Self::with_same_message_in`] belongs to.
pub const SHARED_ACCOUNT_ID: &'static str = "acct-1";
/// The one derived message key every occupancy staged by
/// [`Self::shared_message_in`] carries, whatever folder it sits in.
pub const SHARED_MESSAGE_KEY: &'static str = "msg-shared-identity";
/// One occupancy of the single shared message, in `folder_id` under
/// `remote_id`.
///
/// Identity is byte-identical across every folder -- that is the whole
/// point: a caller cannot tell a first arrival in a second mailbox from a
/// message it already stores unless it compares occupancies rather than
/// message keys.
pub fn shared_message_in(folder_id: &str, remote_id: &str) -> PlacedMessage {
PlacedMessage {
email: Email {
id: Self::SHARED_MESSAGE_KEY.to_string(),
account_id: Self::SHARED_ACCOUNT_ID.to_string(),
subject: "Shared across folders".to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1_700_000_000,
body_plain: Some("one message, several mailboxes".to_string()),
body_html: None,
message_id: Some("shared@nuncio.mx".to_string()),
content_hash: None,
attachments: Vec::new(),
},
// EmailId, not Surrogate: a folder-independent key is exactly what
// a server-assigned identity buys, and it is the tier under which
// one message legitimately reports from several mailboxes.
source: IdentitySource::EmailId,
placement: Placement {
account_id: Self::SHARED_ACCOUNT_ID.to_string(),
folder_id: folder_id.to_string(),
uid_validity: "1".to_string(),
remote_id: remote_id.to_string(),
read: false,
},
}
}
/// A backend serving `folders`, each holding the same message under its own
/// `remote_id` -- one identity, one occupancy per folder.
///
/// Stages the folders as well as the messages, so a caller that enumerates
/// folders and then syncs each one sees every occupancy.
pub fn with_same_message_in(folders: &[&str]) -> Self {
let backend = Self::new();
for (index, folder_id) in folders.iter().enumerate() {
backend.add_folder(Folder {
id: (*folder_id).to_string(),
name: (*folder_id).to_string(),
total_messages: 1,
unread_messages: 1,
});
// A distinct UID per folder: UIDs are folder-scoped, so the same
// mail in two mailboxes is addressed by two different ones.
backend.add_message(Self::shared_message_in(folder_id, &(index + 1).to_string()));
}
backend
}
/// Every [`RemoteMutationSpec`] this mock's `apply_mutation` was called
/// with, in call order. A failing configuration (`set_should_fail(true)`)
/// records nothing, so a test can prove a failed op is never counted as
@@ -112,7 +194,7 @@ impl MailBackend for MockMailBackend {
&self,
folder_id: &str,
since_state: Option<&str>,
) -> Result<crate::backend::FolderChanges, MailError> {
) -> Result<FolderChanges, MailError> {
// Record the checkpoint arg before any early return, so even a failing
// call is visible to a test asserting on how the caller resumes.
if let Ok(mut calls) = self.since_state_calls.lock() {
@@ -134,11 +216,25 @@ impl MailBackend for MockMailBackend {
.map_err(|e| MailError::ParseFailed(e.to_string()))?
.clone();
// Staged removals are scoped to the folder each key names, and are
// reported on both the full and the incremental pass -- a `VANISHED`
// report is exactly the kind of thing an incremental resync carries.
let removals: Vec<PlacementKey> = self
.removals
.lock()
.map_err(|e| MailError::ParseFailed(e.to_string()))?
.iter()
.filter(|key| key.folder_id == folder_id)
.cloned()
.collect();
// Simulate a since_state-aware backend: with a prior checkpoint there
// is nothing new to hand back, so the incremental fetch is genuinely
// narrower (empty) rather than a full re-report of every message.
// `present` stays `None`: an incremental pass cannot speak to absence.
if since_state.is_some() {
return Ok(crate::backend::FolderChanges {
return Ok(FolderChanges {
removals,
next_state: returned_state,
..Default::default()
});
@@ -150,15 +246,16 @@ impl MailBackend for MockMailBackend {
.map_err(|e| MailError::ParseFailed(e.to_string()))?;
let matches: Vec<_> = messages
.iter()
.filter(|m| m.folder_id == folder_id)
.filter(|m| m.placement.folder_id == folder_id)
.cloned()
.collect();
// A full pass over the mock's store did see everything in the folder,
// so it can honestly report what is present.
let present = matches.iter().map(|m| m.id.clone()).collect();
Ok(crate::backend::FolderChanges {
// so it can honestly report what is present -- as occupancies, since
// that is what a folder can speak to.
let present = matches.iter().map(|m| m.placement.key()).collect();
Ok(FolderChanges {
upserts: matches,
removals: Vec::new(),
removals,
present: Some(present),
next_state: returned_state,
})
@@ -256,22 +353,30 @@ mod tests {
unread_messages: 1,
});
let email = Email {
id: "msg-mock-1".to_string(),
account_id: "acct-1".to_string(),
folder_id: "inbox".to_string(),
remote_id: "1".to_string(),
uid_validity: "1".to_string(),
subject: "Mock Test".to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1700000000,
read: false,
body_plain: Some("Mock body".to_string()),
body_html: None,
attachments: Vec::new(),
let placed = PlacedMessage {
email: Email {
id: "msg-mock-1".to_string(),
account_id: "acct-1".to_string(),
subject: "Mock Test".to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1700000000,
body_plain: Some("Mock body".to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
},
source: IdentitySource::Surrogate,
placement: Placement {
account_id: "acct-1".to_string(),
folder_id: "inbox".to_string(),
uid_validity: "1".to_string(),
remote_id: "1".to_string(),
read: false,
},
};
mock.add_message(email.clone());
mock.add_message(placed.clone());
let folders = mock.sync_folders().await.expect("sync folders succeeds");
assert_eq!(folders.len(), 1);
@@ -281,6 +386,10 @@ mod tests {
.await
.expect("sync messages succeeds");
assert_eq!(msgs.len(), 1);
assert_eq!(
msgs[0], placed,
"the mock must hand back exactly the placement it was staged with"
);
assert_eq!(state, "mock-state-token-100");
// Test error simulation
@@ -289,6 +398,39 @@ mod tests {
assert!(mock.sync_messages("inbox", None).await.is_err());
}
/// The staging helper is only useful if it really produces one identity in
/// two places: same message key, different occupancies. If it ever drifted
/// into two keys, every test built on it would pass for the wrong reason.
#[tokio::test]
async fn with_same_message_in_stages_one_identity_across_every_folder() {
let mock = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]);
let folders = mock.sync_folders().await.expect("sync folders succeeds");
let folder_ids: Vec<&str> = folders.iter().map(|f| f.id.as_str()).collect();
assert_eq!(folder_ids, vec!["INBOX", "Archive"]);
let inbox = mock
.sync_changes("INBOX", None)
.await
.expect("inbox pass succeeds");
let archive = mock
.sync_changes("Archive", None)
.await
.expect("archive pass succeeds");
assert_eq!(inbox.upserts.len(), 1);
assert_eq!(archive.upserts.len(), 1);
assert_eq!(
inbox.upserts[0].email.id, archive.upserts[0].email.id,
"both occupancies must carry the same derived message key"
);
assert_ne!(
inbox.upserts[0].placement.key(),
archive.upserts[0].placement.key(),
"the occupancies themselves must differ, or there is nothing to classify"
);
}
fn sample_outbound_message() -> OutboundMessage {
OutboundMessage {
from: "alice@nuncio.mx".to_string(),
+56 -19
View File
@@ -61,12 +61,15 @@ impl MimeParserAdapter {
pub const MAX_PAYLOAD_BYTES: usize = 25 * 1024 * 1024;
/// Parse a raw RFC 5322 byte slice into an [`Email`] domain entity.
pub fn parse_mime(
id: &str,
account_id: &str,
folder_id: &str,
raw_bytes: &[u8],
) -> Result<Email, MailError> {
///
/// The returned `Email` carries an **empty** `id`, and it is the caller's
/// job to fill it from [`Email::derive_message_key`]. The derivation needs
/// inputs this parser cannot see -- the transport's addressing coordinates
/// and any server-assigned identity (RFC 8474 `EMAILID`, `X-GM-MSGID`) --
/// alongside the `Message-ID` and content hash recovered here, so the id
/// can only be settled once both halves are in hand. Minting a provisional
/// value here would risk one reaching the store as though it were derived.
pub fn parse_mime(account_id: &str, raw_bytes: &[u8]) -> Result<Email, MailError> {
if raw_bytes.len() > Self::MAX_PAYLOAD_BYTES {
return Err(MailError::ParseFailed(
"MIME payload exceeds maximum allowed limit of 25MB".to_string(),
@@ -95,6 +98,9 @@ impl MimeParserAdapter {
let received_at = msg.date().map_or(0, |d| d.to_timestamp());
let message_id = msg.message_id().and_then(Email::normalize_message_id);
let content_hash = Some(Email::content_hash_of(raw_bytes));
let body_plain = msg.body_text(0).map(|b| b.to_string());
let body_html = msg.body_html(0).map(|b| b.to_string());
@@ -126,21 +132,19 @@ impl MimeParserAdapter {
}
Ok(Email {
id: id.to_string(),
// Assigned by the caller from the derived message key; see the
// method doc for why it cannot be settled here.
id: String::new(),
account_id: account_id.to_string(),
folder_id: folder_id.to_string(),
// The MIME body carries no protocol addressing; the caller (the
// IMAP sync) fills these from the transport before persisting.
remote_id: String::new(),
uid_validity: String::new(),
subject,
sender,
recipient,
received_at,
read: false,
body_plain,
body_html,
attachments,
message_id,
content_hash,
})
}
}
@@ -158,9 +162,12 @@ mod tests {
\r\n\
Hello Bob, this is a plain text email.";
let email =
MimeParserAdapter::parse_mime("msg-1", "acct-1", "inbox", raw).expect("parse succeeds");
assert_eq!(email.id, "msg-1");
let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds");
assert_eq!(
email.id, "",
"the parser must not mint an id; the caller derives it"
);
assert_eq!(email.account_id, "acct-1");
assert_eq!(email.subject, "Test Subject");
assert_eq!(email.sender, "alice@nuncio.mx");
assert_eq!(email.recipient, "bob@nuncio.mx");
@@ -170,11 +177,41 @@ mod tests {
);
}
#[test]
fn parse_captures_normalized_message_id_and_content_hash() {
let raw = b"From: Alice <alice@nuncio.mx>\r\n\
To: Bob <bob@nuncio.mx>\r\n\
Message-ID: <Abc.123@mail.nuncio.mx>\r\n\
Subject: Test\r\n\
\r\n\
body";
let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds");
assert_eq!(email.message_id.as_deref(), Some("Abc.123@mail.nuncio.mx"));
assert_eq!(
email.content_hash,
Some(nuncio_core::model::Email::content_hash_of(raw)),
"the hash must cover the octets actually parsed"
);
}
#[test]
fn parse_reports_no_message_id_when_the_header_is_absent() {
// RFC 5322 3.6.4 makes Message-ID a SHOULD. An absent header must read
// as "none present", never as an invented or empty value.
let raw = b"From: Alice <alice@nuncio.mx>\r\n\
Subject: No id\r\n\
\r\n\
body";
let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds");
assert_eq!(email.message_id, None);
assert!(email.content_hash.is_some());
}
#[test]
fn parse_invalid_mime_returns_error() {
let raw = b"";
let err = MimeParserAdapter::parse_mime("msg-2", "acct-1", "inbox", raw)
.expect_err("should fail");
let err = MimeParserAdapter::parse_mime("acct-1", raw).expect_err("should fail");
assert_eq!(
err,
MailError::ParseFailed("invalid RFC 5322 MIME structure".to_string())
@@ -188,7 +225,7 @@ mod tests {
#[test]
fn parse_oversized_mime_returns_error() {
let oversized = vec![0u8; MimeParserAdapter::MAX_PAYLOAD_BYTES + 1];
let err = MimeParserAdapter::parse_mime("msg-huge", "acct-1", "inbox", &oversized)
let err = MimeParserAdapter::parse_mime("acct-1", &oversized)
.expect_err("should fail for oversized payload");
assert!(err
.to_string()
+2 -4
View File
@@ -357,17 +357,15 @@ mod tests {
Email {
id: "msg-1".to_string(),
account_id: "acct-1".to_string(),
folder_id: "outbox".to_string(),
remote_id: "1".to_string(),
uid_validity: "1".to_string(),
subject: "Status Update".to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1700000000,
read: true,
body_plain: Some("Plaintext status update".to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
}
}
+26 -9
View File
@@ -11,6 +11,7 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use nuncio_core::model::PlacementKey;
use nuncio_mail::imap_raw;
use nuncio_mail::test_server::{MockImapServer, ServerProfile};
@@ -345,14 +346,17 @@ async fn every_rung_reports_a_message_that_disappeared() {
let second = sync_folder(&server, Some(&first.next_state)).await;
// However the rung learned it, the caller must end up able to identify
// the removed message and only that one.
let mut reported: Vec<String> = second.removals.clone();
// the occupancy that went and only that one. Absence is reported per
// mailbox, so what is compared is the placement each upsert was found
// under, not the message key -- the same message may still sit
// elsewhere and must not be swept up by this folder's silence.
let mut reported: Vec<PlacementKey> = second.removals.clone();
if let Some(present) = &second.present {
let stored_gone = first
.upserts
.iter()
.map(|e| e.id.clone())
.filter(|id| !present.contains(id));
.map(|e| e.placement.key())
.filter(|key| !present.contains(key));
reported.extend(stored_gone);
}
reported.sort();
@@ -366,12 +370,17 @@ async fn every_rung_reports_a_message_that_disappeared() {
let surviving = first
.upserts
.iter()
.find(|e| e.remote_id == keep.to_string())
.find(|e| e.placement.remote_id == keep.to_string())
.expect("the kept message was in the first pass");
assert!(
!reported.contains(&surviving.id),
!reported.contains(&surviving.placement.key()),
"{profile:?}: the surviving message must not be reported gone"
);
assert_eq!(
reported[0].remote_id,
gone.to_string(),
"{profile:?}: the reported occupancy must be the expunged UID"
);
}
}
@@ -474,9 +483,17 @@ async fn a_uidvalidity_change_refetches_rather_than_deleting_the_folder() {
second.removals.is_empty(),
"a renumbering is not a removal report"
);
// The re-fetched message has a different surrogate, because UIDVALIDITY is
// part of identity -- so the caller replaces rather than merges.
assert_ne!(second.upserts[0].id, first.upserts[0].id);
// The re-fetched message lands under a new occupancy, because UIDVALIDITY
// is part of a placement key -- so the caller writes a new placement rather
// than updating the old one, and the stale UID never addresses live mail.
assert_ne!(
second.upserts[0].placement.uid_validity,
first.upserts[0].placement.uid_validity
);
assert_ne!(second.upserts[0].placement, first.upserts[0].placement);
// These two are also genuinely different messages, so their derived keys
// differ as well.
assert_ne!(second.upserts[0].email.id, first.upserts[0].email.id);
}
/// Build an engine pointed at the mock and apply one mutation through the
+22 -15
View File
@@ -2,6 +2,7 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use nuncio_core::model::IdentitySource;
use nuncio_mail::{JmapEngine, MailBackend, RemoteMutationKind, RemoteMutationSpec};
use serde_json::json;
use wiremock::matchers::{body_partial_json, method, path};
@@ -84,13 +85,16 @@ async fn wiremock_jmap_session_discovery_and_email_get_sync() {
.expect("parse email get");
assert_eq!(emails.len(), 1);
// The JMAP object id is preserved as the protocol-native `remote_id`, while
// the persisted id is an opaque 64-char surrogate hash (never the raw id).
assert_eq!(emails[0].remote_id, "msg-wm-100");
assert_ne!(emails[0].id, "msg-wm-100");
assert_eq!(emails[0].id.len(), 64);
assert_eq!(emails[0].subject, "WireMock JMAP E2E Test");
assert_eq!(emails[0].sender, "sender@nuncio.mx");
// The JMAP object id is preserved as the placement's protocol-native
// `remote_id`, while the message's own id is an opaque 64-char derived key
// (never the raw id). The object id is account-stable, so it enters the
// precedence at the EMAILID tier rather than as a folder-scoped surrogate.
assert_eq!(emails[0].placement.remote_id, "msg-wm-100");
assert_eq!(emails[0].source, IdentitySource::EmailId);
assert_ne!(emails[0].email.id, "msg-wm-100");
assert_eq!(emails[0].email.id.len(), 64);
assert_eq!(emails[0].email.subject, "WireMock JMAP E2E Test");
assert_eq!(emails[0].email.sender, "sender@nuncio.mx");
assert_eq!(new_state, "sync-state-500");
}
@@ -204,15 +208,18 @@ async fn jmap_engine_sync_folders_and_messages_through_mail_backend_trait() {
.await
.expect("sync_messages succeeds");
assert_eq!(emails.len(), 1);
assert_eq!(emails[0].remote_id, "msg-wm-200");
assert_ne!(emails[0].id, "msg-wm-200");
assert_eq!(emails[0].id.len(), 64);
assert_eq!(emails[0].subject, "Trait-Level JMAP Sync");
assert_eq!(emails[0].sender, "sender@nuncio.mx");
assert_eq!(emails[0].received_at, 1700002000);
assert!(!emails[0].read);
assert_eq!(emails[0].placement.remote_id, "msg-wm-200");
assert_eq!(emails[0].placement.folder_id, "inbox");
assert_eq!(emails[0].source, IdentitySource::EmailId);
assert_ne!(emails[0].email.id, "msg-wm-200");
assert_eq!(emails[0].email.id.len(), 64);
assert_eq!(emails[0].email.subject, "Trait-Level JMAP Sync");
assert_eq!(emails[0].email.sender, "sender@nuncio.mx");
assert_eq!(emails[0].email.received_at, 1700002000);
// Read state is a property of the occupancy, not of the message.
assert!(!emails[0].placement.read);
assert_eq!(
emails[0].body_plain,
emails[0].email.body_plain,
Some("Fetched through the MailBackend trait.".to_string())
);
assert_eq!(new_state, "sync-state-900");
@@ -519,10 +519,19 @@ message Attachment {
// (rather than `Email`) to avoid colliding with common generated-code
// identifiers in hand-written Rust using this module.
message Message {
// Unique message identifier. OPAQUE and server-assigned: a deterministic
// surrogate derived from the message's stable protocol identity, never a
// raw upstream protocol id (e.g. an IMAP UID, which is only stable within
// a single IMAP session and is NOT a durable cross-session identifier).
// Unique message identifier. OPAQUE and server-assigned: content-addressed
// from the strongest identity the message's server offered. Never a raw
// upstream protocol id (e.g. an IMAP UID, which is only stable within a
// single IMAP session and is NOT a durable cross-session identifier).
//
// Stability across a move depends on what the server offered. Where a
// server-assigned object id or a usable Message-ID was available, the id is
// folder-independent and survives a move between folders. Where none was --
// a message carrying no Message-ID, or one fetched headers-only -- the id
// falls back to a derivation that includes the folder, and moving such a
// message yields a different id. Clients MUST NOT assume an id outlives a
// move.
//
// Clients MUST treat this as an opaque token -- pass it back verbatim in
// later requests, but never parse, construct, or assume any structure in
// its value.
@@ -530,6 +539,11 @@ message Message {
// Account identifier owning the message.
string account_id = 2;
// Mailbox folder identifier (e.g. "inbox").
//
// A message can occupy several folders at once. This names the one the
// message was reached through: the requested folder for a listing, and the
// lowest-sorting folder id for a lookup by id. A field carrying the complete
// placement set is not yet part of this contract.
string folder_id = 3;
// Subject line.
string subject = 4;
@@ -777,6 +791,13 @@ service Filters {
// applied retroactively to mail that was synced before those rules
// existed. Streams cumulative progress as it works, ending with a final
// `done: true` message once the whole store has been scanned.
//
// Each (rule, message) pair fires at most once, ever -- the same claim
// ledger live sync uses, keyed by rule id and message id. This is not a
// repeatable rescan: running Triage again with the same rules finds
// nothing left to claim and reports `matched_count: 0`. Only messages
// stored since the last run, or rules created since then, produce further
// matches; editing an existing rule's contents does not reset its claims.
rpc Triage(TriageRequest) returns (stream TriageProgress);
}
File diff suppressed because it is too large. Load diff
+61 -33
View File
@@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize};
/// Search hit result for full-text query matches.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SearchHit {
/// Entity ID matching the query.
/// Entity ID matching the query. For a message hit this is the
/// folder-independent message key: the index holds one row per message, so
/// a hit names a message and never one of its mailbox placements. Resolve
/// it through [`DatabaseEngine::get_message`].
pub id: String,
/// Matching title or subject.
pub title: String,
@@ -50,7 +53,7 @@ impl<'a> SearchEngine<'a> {
let rows: Vec<(String, String, String)> = sqlx::query_as(
r#"
SELECT id, subject, snippet(messages_fts, 3, '<b>', '</b>', '...', 10) as snippet
SELECT message_key, subject, snippet(messages_fts, 3, '<b>', '</b>', '...', 10) as snippet
FROM messages_fts
WHERE messages_fts MATCH ?
ORDER BY rank
@@ -94,7 +97,7 @@ impl<'a> SearchEngine<'a> {
let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new(
"SELECT id, title, snippet, rk FROM ( \
SELECT id, subject AS title, \
SELECT message_key AS id, subject AS title, \
snippet(messages_fts, 3, '<b>', '</b>', '...', 10) AS snippet, rank AS rk \
FROM messages_fts WHERE messages_fts MATCH ",
);
@@ -255,21 +258,39 @@ mod tests {
nuncio_core::model::Email {
id: id.to_string(),
account_id: "acct-1".to_string(),
folder_id: "inbox".to_string(),
remote_id: id.to_string(),
uid_validity: "1".to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1_700_000_000,
read: false,
body_plain: Some(body_plain.to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
}
}
/// The crux test: proves body search works over the REAL `save_email` write path (no
fn sample_placement(uid: &str) -> nuncio_core::model::Placement {
nuncio_core::model::Placement {
account_id: "acct-1".to_string(),
folder_id: "inbox".to_string(),
uid_validity: "1".to_string(),
remote_id: uid.to_string(),
read: false,
}
}
async fn save(db: &DatabaseEngine, email: &nuncio_core::model::Email) {
db.save_email_at(
email,
nuncio_core::model::IdentitySource::Surrogate,
&sample_placement(&email.id),
)
.await
.unwrap();
}
/// The crux test: proves body search works over the REAL `save_email_at` write path (no
/// raw-SQL plaintext injection bypassing encryption), while the stored body column stays
/// ciphertext. This is what the old `fts5_message_search_and_triggers` test failed to
/// prove -- it inserted plaintext directly via raw SQL, which trivially "worked" but never
@@ -288,7 +309,7 @@ mod tests {
"Quarterly Financial Meeting",
"Let's discuss the annual budget revenue forecast",
);
db.save_email(&email).await.unwrap();
save(&db, &email).await;
// Body search matches a plaintext body term -- proves the FTS index holds real,
// decrypted-at-write-time plaintext trigrams, not the AES-256-GCM ciphertext that is
@@ -305,39 +326,44 @@ mod tests {
// The body column at rest must remain encrypted ciphertext -- never the plaintext
// search term, and never equal to the original plaintext body.
let (stored_body,): (String,) =
sqlx::query_as("SELECT body_plain FROM messages WHERE id = 'msg-1'")
sqlx::query_as("SELECT body_plain FROM messages WHERE message_key = 'msg-1'")
.fetch_one(db.pool())
.await
.unwrap();
assert_ne!(stored_body, email.body_plain.clone().unwrap());
assert!(!stored_body.to_lowercase().contains("revenue"));
// Re-saving via save_email (an update, not a fresh insert) replaces the FTS row rather
// than duplicating or leaving a stale entry: the old term no longer matches, the new
// one does.
let mut updated = email.clone();
updated.body_plain = Some("Updated strategy review notes".to_string());
db.save_email(&updated).await.unwrap();
// Re-saving the same key with a different body must leave the index alone: the message
// row is first-write-wins, so a second write that cannot change the stored body must
// not change what search returns for it either. Anything else would have search
// disagree with the store -- either finding a body that was never persisted, or holding
// two rows for one message.
let mut rewritten = email.clone();
rewritten.body_plain = Some("Updated strategy review notes".to_string());
save(&db, &rewritten).await;
assert!(search.search_messages("revenue").await.unwrap().is_empty());
let new_hits = search.search_messages("strategy").await.unwrap();
assert_eq!(new_hits.len(), 1);
assert_eq!(new_hits[0].id, "msg-1");
let unchanged = search.search_messages("revenue").await.unwrap();
assert_eq!(unchanged.len(), 1, "the stored body is still searchable");
assert_eq!(unchanged[0].id, "msg-1");
assert!(
search.search_messages("strategy").await.unwrap().is_empty(),
"a body the store rejected must never become searchable"
);
// Delete message trigger still cleans up the FTS row (subject/sender/id only -- no
// Delete message trigger still cleans up the FTS row (subject/sender/key only -- no
// ciphertext involved).
sqlx::query("DELETE FROM messages WHERE id = 'msg-1'")
sqlx::query("DELETE FROM messages WHERE message_key = 'msg-1'")
.execute(db.pool())
.await
.unwrap();
let deleted_hits = search.search_messages("strategy").await.unwrap();
let deleted_hits = search.search_messages("revenue").await.unwrap();
assert!(deleted_hits.is_empty());
}
/// Proves the backfill half of the fix: a message saved through the real write path,
/// whose FTS entry is then lost (simulating either a row written before the FTS5 index
/// existed, or one written by a process that bypassed `save_email`'s explicit indexing),
/// existed, or one written by a process that bypassed `save_email_at`'s explicit indexing),
/// is NOT searchable until a migration pass runs -- at which point `backfill_message_fts`
/// decrypts the ciphertext body column with this engine's real storage key and repopulates
/// the plaintext-derived trigram index, with no search call ever required to trigger it.
@@ -349,11 +375,11 @@ mod tests {
"Legacy Roadmap Notes",
"Confidential quarterly roadmap details",
);
db.save_email(&email).await.unwrap();
save(&db, &email).await;
// Simulate pre-existing data whose FTS entry is missing (e.g. written before the FTS5
// index existed) by dropping its messages_fts row directly.
sqlx::query("DELETE FROM messages_fts WHERE id = 'msg-legacy-1'")
sqlx::query("DELETE FROM messages_fts WHERE message_key = 'msg-legacy-1'")
.execute(db.pool())
.await
.unwrap();
@@ -383,13 +409,15 @@ mod tests {
async fn search_messages_page_returns_every_hit_once_no_gaps() {
let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
for i in 0..5 {
db.save_email(&sample_email(
&format!("msg-{i}"),
"Budget Planning Meeting",
"annual budget revenue forecast discussion",
))
.await
.unwrap();
save(
&db,
&sample_email(
&format!("msg-{i}"),
"Budget Planning Meeting",
"annual budget revenue forecast discussion",
),
)
.await;
}
let search = SearchEngine::new(&db);
+753 -164
View File
File diff suppressed because it is too large. Load diff
+191 -18
View File
@@ -289,7 +289,7 @@ async fn execute_one(
match mutation.mutation_type.as_str() {
"MOVE" | "COPY" | "FLAG" | "UNFLAG" | "DELETE" => {
dispatch_mailbox_mutation(env, mutation, &email, &payload).await
dispatch_mailbox_mutation(db, env, mutation, &email, &payload).await
}
"FORWARD" => dispatch_forward(db, env, &email, &payload).await,
"WEBHOOK" => dispatch_webhook(env, mutation, &email, &payload).await,
@@ -298,10 +298,30 @@ async fn execute_one(
}
/// Execute a mailbox-affecting mutation (move/copy/flag/unflag/delete) against
/// the account's mail backend, recovering the remote addressing (protocol id +
/// UIDVALIDITY scope) from the stored message row so the backend can enforce
/// its UIDVALIDITY guard.
/// the account's mail backend, recovering the remote addressing (folder,
/// protocol id, UIDVALIDITY scope) from the message's stored placement so the
/// backend can enforce its UIDVALIDITY guard.
///
/// The addressing comes from a placement rather than the message row, because
/// those coordinates are properties of a mailbox occupancy and a message can
/// hold several. The outbox row records only the message key, so which occupancy
/// the rule matched is not recoverable here -- and when the message occupies
/// more than one mailbox, this **refuses** rather than picking one.
///
/// Guessing is not the conservative option. A rule that matched the `INBOX`
/// copy would `MOVE` or `DELETE` whichever copy happened to sort first, against
/// mail the user never targeted and with no undo; Gmail labels make
/// multi-placement ordinary rather than exotic. Unresolvable *addressing* is no
/// more a licence to act than an unverifiable claim is in
/// [`crate::sync::apply_filter_actions`]: refusing is inert and visible,
/// guessing is destructive and irreversible.
///
/// The refusal is permanent, not retryable: nothing about a later attempt makes
/// the outbox row remember a placement it never recorded. Carrying the
/// originating placement on the row is the real fix, and belongs with the
/// placement-aware wire surface.
async fn dispatch_mailbox_mutation(
db: &DatabaseEngine,
env: &dyn RemoteExecutionEnv,
mutation: &PendingRemoteMutation,
email: &nuncio_core::model::Email,
@@ -330,20 +350,56 @@ async fn dispatch_mailbox_mutation(
other => return Disposition::Permanent(format!("unexpected mailbox mutation '{other}'")),
};
let backend = match env.mail_backend(&email.account_id).await {
// Recover the remote addressing from a stored placement rather than
// parsing it out of the opaque message key. The backend still
// SELECT-verifies the UIDVALIDITY before mutating.
let placements = match db.placements_of(&email.id).await {
Ok(placements) => placements,
Err(e) => {
return Disposition::Retry(format!("failed to resolve the message's placements: {e}"))
}
};
let placement = match placements.as_slice() {
[only] => only,
[] => {
// No occupancy means no way to address the message on any server,
// and no retry can conjure one back.
return Disposition::Permanent(format!(
"message '{}' occupies no mailbox to address",
email.id
));
}
several => {
let folders: Vec<&str> = several.iter().map(|p| p.folder_id.as_str()).collect();
tracing::warn!(
mutation_id = %mutation.id,
op = %mutation.mutation_type,
message_id = %email.id,
placement_count = several.len(),
"outbox: refusing an ambiguously-addressed mutation"
);
return Disposition::Permanent(format!(
"message '{}' occupies {} mailboxes ({}) and the outbox row does not record \
which one rule '{}' matched; refusing to guess a {} target",
email.id,
several.len(),
folders.join(", "),
mutation.rule_id,
mutation.mutation_type
));
}
};
let backend = match env.mail_backend(&placement.account_id).await {
Ok(backend) => backend,
Err(e) => return Disposition::Retry(format!("failed to build mail backend: {e}")),
};
// Recover the remote addressing from the stored message row -- the
// protocol-native id and the UIDVALIDITY scope it was captured under --
// rather than parsing it out of the opaque surrogate id. The backend still
// SELECT-verifies the UIDVALIDITY before mutating.
let spec = RemoteMutationSpec {
message_id: email.id.clone(),
remote_id: email.remote_id.clone(),
folder_id: email.folder_id.clone(),
uid_validity: email.uid_validity.clone(),
remote_id: placement.remote_id.clone(),
folder_id: placement.folder_id.clone(),
uid_validity: placement.uid_validity.clone(),
kind,
};
@@ -580,7 +636,7 @@ mod tests {
use super::*;
use crate::lifecycle::{ShutdownController, ShutdownSignal};
use crate::test_tracing::with_recorder;
use nuncio_core::model::Email;
use nuncio_core::model::{Email, IdentitySource, Placement};
use nuncio_core::EventBus;
use nuncio_filter::MutationPayload;
use nuncio_mail::MockMailBackend;
@@ -634,17 +690,27 @@ mod tests {
Email {
id: "msg-outbox-log".to_string(),
account_id: "acct-outbox-log".to_string(),
folder_id: "INBOX".to_string(),
remote_id: "77".to_string(),
uid_validity: "9".to_string(),
subject: "Confidential outbox subject".to_string(),
sender: "alice@example.com".to_string(),
recipient: "owner@nuncio.mx".to_string(),
received_at: 1_700_000_000,
read: false,
body_plain: Some("secret outbox body".to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
}
}
/// The one mailbox occupancy `sample_email` is stored in -- the addressing
/// the outbox recovers to build its `RemoteMutationSpec`.
fn sample_placement() -> Placement {
Placement {
account_id: "acct-outbox-log".to_string(),
folder_id: "INBOX".to_string(),
uid_validity: "9".to_string(),
remote_id: "77".to_string(),
read: false,
}
}
@@ -683,7 +749,13 @@ mod tests {
db.save_folder_sync_state("acct-outbox-log", "INBOX", "9:100")
.await
.expect("save folder checkpoint");
db.save_email(&sample_email()).await.expect("save email");
db.save_email_at(
&sample_email(),
IdentitySource::Surrogate,
&sample_placement(),
)
.await
.expect("save email");
db.save_pending_mutation(&flag_mutation())
.await
.expect("save pending mutation");
@@ -727,4 +799,105 @@ mod tests {
);
}
}
/// A second occupancy of `sample_email`, in a different mailbox -- an
/// ordinary Gmail label, or a `COPY` another client made.
fn second_placement() -> Placement {
Placement {
account_id: "acct-outbox-log".to_string(),
folder_id: "Archive".to_string(),
uid_validity: "9".to_string(),
remote_id: "78".to_string(),
read: false,
}
}
/// Seed the message into every given occupancy and dispatch one FLAG
/// mutation against it. Returns the honest disposition and every spec the
/// backend was actually asked to apply.
///
/// Calls [`dispatch_mailbox_mutation`] rather than draining through
/// [`execute_pending_mutations`] deliberately. The drain's own
/// "attempting"/"completed" `tracing` callsites are what
/// `drain_logs_attempt_and_completion_without_leaking_message_content`
/// asserts on, and `tracing` caches callsite interest globally: a
/// concurrent test hitting those callsites with no recorder installed
/// caches them as uninteresting and makes that test fail. Addressing is
/// decided here anyway, so the narrower call is also the tighter assertion.
async fn dispatch_with_placements(
placements: &[Placement],
) -> (Disposition, Vec<RemoteMutationSpec>) {
let (db, _dir) = DatabaseEngine::connect_ephemeral()
.await
.expect("ephemeral db");
for placement in placements {
db.save_email_at(&sample_email(), IdentitySource::Surrogate, placement)
.await
.expect("store the occupancy");
}
let backend = MockMailBackend::new();
let env = SucceedingEnv {
backend: backend.clone(),
};
let mutation = flag_mutation();
let payload =
serde_json::from_str::<MutationPayload>(&mutation.payload).expect("parse payload");
let disposition =
dispatch_mailbox_mutation(&db, &env, &mutation, &sample_email(), &payload).await;
(disposition, backend.applied_mutations())
}
/// The unambiguous case still works: exactly one occupancy, so the row's
/// target is fully determined and the mutation is addressed from it.
#[tokio::test]
async fn a_single_placement_mutation_is_addressed_from_that_placement() {
let (disposition, applied) = dispatch_with_placements(&[sample_placement()]).await;
assert!(
matches!(disposition, Disposition::Completed),
"a fully-addressed mutation must go through"
);
assert_eq!(applied.len(), 1, "the backend really was asked to act");
assert_eq!(applied[0].folder_id, "INBOX");
assert_eq!(applied[0].remote_id, "77");
assert_eq!(applied[0].uid_validity, "9");
assert_eq!(applied[0].message_id, "msg-outbox-log");
}
/// The ambiguous case refuses. The outbox row names only a message, so for
/// a message in two mailboxes there is no way to know which copy the rule
/// matched -- and a guessed MOVE or DELETE lands on mail the user never
/// targeted, with no undo.
#[tokio::test]
async fn a_multi_placement_mutation_is_refused_rather_than_guessed() {
let (disposition, applied) =
dispatch_with_placements(&[sample_placement(), second_placement()]).await;
assert!(
applied.is_empty(),
"an ambiguously-addressed mutation must never reach the server"
);
match disposition {
// Permanent, not Retry: no later attempt makes the outbox row
// remember a placement it never recorded.
Disposition::Permanent(reason) => {
assert!(
reason.contains("INBOX") && reason.contains("Archive"),
"the refusal must name the competing mailboxes: {reason}"
);
assert!(
reason.contains("refusing"),
"and say plainly that it refused: {reason}"
);
}
Disposition::Completed => panic!("expected a permanent refusal, got Completed"),
Disposition::Conflict(observed) => {
panic!("expected a permanent refusal, got Conflict({observed})")
}
Disposition::Retry(reason) => {
panic!("expected a permanent refusal, got Retry({reason})")
}
}
}
}
+489 -174
View File
@@ -7,7 +7,8 @@
//! account it connects a real [`MailBackend`] (a real [`ImapEngine`] for an
//! IMAP/SMTP transport, built from [`AccountConfig`] plus the
//! account's keyring-stored password), fetches folders and messages, and
//! persists every message through [`DatabaseEngine::save_email`].
//! persists every message, together with the mailbox occupancy it was found
//! in, through [`DatabaseEngine::save_email_at`].
//!
//! # Testability
//!
@@ -20,6 +21,7 @@
//! all-accounts sync fans these per-account runs out through
//! [`crate::sync_dispatcher::sync_all_configured`], so concurrency and the
//! per-account timeout are bounded in one place rather than serialized here.
use nuncio_core::model::PlacementKey;
use nuncio_core::{AccountConfig, CoreCommand, CoreEvent, EventBus, Transport};
use nuncio_filter::{FilterEngine, OutboxManager, RuleAction};
use nuncio_mail::{ImapEngine, JmapEngine, MailBackend, MailError};
@@ -68,24 +70,39 @@ fn remote_action_tag(action: &RuleAction) -> Option<(&'static str, Option<String
}
}
/// Evaluate `email` against `filter_engine` and route every matched rule's
/// actions to their real, persisted effect.
/// Evaluate `email` **as it sits in `placement`** against `filter_engine` and
/// route every matched rule's actions to their real, persisted effect.
///
/// Filters fire EXACTLY ONCE per newly-arrived message: callers (see
/// [`fetch_and_persist`]) must only invoke this for a message that was
/// genuinely new to the store on this sync pass, never for one that already
/// existed. Sync is not yet incremental -- every sync re-fetches and
/// re-persists messages the backend still reports -- so calling this
/// unconditionally on every persisted message would re-fire a rule's actions
/// on every single re-sync of the same mailbox: a fresh
/// `PendingRemoteMutation` and a fresh `FilterExecutionLog` entry per cycle
/// for a message that arrived once, growing the outbox and the audit ledger
/// without bound and (once a real mutation transport lands) duplicating the
/// real remote operation. This function itself has no way to tell "new" from
/// "already seen" -- that determination is the caller's responsibility.
/// Evaluation is per placement because the rule language can ask where a
/// message is: `WHERE FOLDER = 'INBOX'` has no answer for a message that
/// occupies three mailboxes, so the caller fans out over the occupancies and
/// each call answers for exactly one.
///
/// `MarkRead`/`MarkUnread` apply immediately to the stored message via
/// [`DatabaseEngine::set_message_read`]. Every other action enqueues a real
/// Acting, in contrast, is once per (rule, message). Every matched rule must
/// win [`DatabaseEngine::claim_filter_fire`] before any of its actions run, and
/// the claim is keyed on message identity, not on the placement. `MOVE` and
/// `FLAG` converge under repetition -- ten moves leave one net effect -- but
/// `FORWARD` and `CALL WEBHOOK` land on third parties and have no shared state
/// to converge against: one message in ten folders would page an on-call
/// engineer ten times. The claim fails closed, so an unverifiable claim skips
/// the actions rather than licensing them.
///
/// The claim is spent the moment it is won, before the actions are known to
/// have succeeded. A rule whose enqueue then fails is therefore not retried on
/// a later pass. That is the deliberate trade: re-firing FORWARD and CALL
/// WEBHOOK on a partial failure would re-deliver to a third party, and a
/// duplicate mail cannot be un-received, while a dropped one is visible in the
/// warning below and recoverable by hand.
///
/// The claim also subsumes the older "only call me for a new message" contract
/// for re-arrivals of the same message. Callers should still gate on a genuinely
/// new occupancy (see [`fetch_and_persist`]) -- the claim stops repeat *actions*,
/// not the wasted evaluation of a mailbox nothing changed in.
///
/// `MarkRead`/`MarkUnread` apply immediately to the occupancy that was
/// evaluated, via [`DatabaseEngine::set_placement_read`]: IMAP `\Seen` is
/// per-mailbox, so "mark this read" is only well-defined once it names a
/// mailbox. Every other action enqueues a real
/// [`nuncio_filter::PendingRemoteMutation`] through the existing outbox
/// (`OutboxManager::create_mutation` + `DatabaseEngine::save_pending_mutation`)
/// for the background outbox worker to pick up -- this function never
@@ -104,40 +121,69 @@ fn remote_action_tag(action: &RuleAction) -> Option<(&'static str, Option<String
///
/// Standalone and reusable by design: this is the exact evaluate-and-route
/// logic other sync paths (e.g. a bulk retroactive triage over the whole
/// store) need, so it takes only a `DatabaseEngine`/`FilterEngine`/`Email`
/// and has no dependency on the live inbound-sync call chain.
/// store) need, so it takes only a `DatabaseEngine`/`FilterEngine`/`Email`/
/// `Placement` and has no dependency on the live inbound-sync call chain.
pub async fn apply_filter_actions(
db: &nuncio_store::db::DatabaseEngine,
filter_engine: &FilterEngine,
email: &nuncio_core::model::Email,
placement: &nuncio_core::model::Placement,
) -> usize {
let mut applied = 0usize;
let placed = nuncio_filter::PlacedEmail { email, placement };
for (rule, actions) in filter_engine.evaluate(email) {
for (rule, actions) in filter_engine.evaluate(placed) {
// Claim before acting, never after. The claim is keyed on message
// identity, so the same mail reaching a second mailbox -- an ordinary
// MOVE, or a Gmail label -- evaluates again but acts only once.
match db.claim_filter_fire(&rule.id, &email.id).await {
Ok(true) => {}
Ok(false) => {
tracing::debug!(
rule_id = %rule.id,
message_key = %email.id,
folder_id = %placement.folder_id,
"rule already fired for this message; skipping actions for this placement"
);
continue;
}
Err(e) => {
// Fail closed. An unverifiable claim must not become a licence
// to act: a lost claim check on a FORWARD is a duplicate mail
// the recipient cannot un-receive.
tracing::error!(
rule_id = %rule.id,
message_key = %email.id,
error = %e,
"could not claim the filter fire; skipping actions"
);
continue;
}
}
let placement_key = placement.key();
for action in actions {
let immediate_ok = match &action {
RuleAction::MarkRead => match db.set_message_read(&email.id, true).await {
Ok(()) => true,
Err(e) => {
tracing::warn!(
"filter rule '{}' MARK READ failed for message '{}': {e}",
rule.id,
email.id
);
false
// One write with a boolean rather than two near-identical
// arms. Both name the occupancy that was evaluated, because
// `\Seen` is per-mailbox: a message read in INBOX is not
// thereby read in Archive.
RuleAction::MarkRead | RuleAction::MarkUnread => {
let read = matches!(action, RuleAction::MarkRead);
let label = if read { "MARK READ" } else { "MARK UNREAD" };
match db.set_placement_read(&placement_key, read).await {
Ok(()) => true,
Err(e) => {
tracing::warn!(
"filter rule '{}' {label} failed for message '{}' in folder '{}': {e}",
rule.id,
email.id,
placement.folder_id
);
false
}
}
},
RuleAction::MarkUnread => match db.set_message_read(&email.id, false).await {
Ok(()) => true,
Err(e) => {
tracing::warn!(
"filter rule '{}' MARK UNREAD failed for message '{}': {e}",
rule.id,
email.id
);
false
}
},
}
RuleAction::MoveTo(_)
| RuleAction::CopyTo(_)
| RuleAction::Flag
@@ -197,31 +243,42 @@ pub async fn apply_filter_actions(
applied
}
/// Fetch every folder and every message in every folder from `backend`,
/// persisting each message via [`DatabaseEngine::save_email`] and, for
/// messages genuinely new to the store, evaluating it against
/// `filter_engine` via [`apply_filter_actions`].
/// Fetch every folder from `backend`, persist each reported occupancy via
/// [`DatabaseEngine::save_email_at`], evaluate the genuinely-new occupancies
/// against `filter_engine` through [`apply_filter_actions`], and delete the
/// occupancies the server no longer reports.
///
/// Sync is not yet incremental (a re-sync re-fetches every message the
/// backend still reports, not just genuinely new ones), so new-vs-seen must
/// be determined BEFORE `save_email` persists (or re-persists) anything --
/// `save_email` is `INSERT OR REPLACE` and would otherwise erase the
/// distinction between "arriving for the first time" and "seen again". Doing
/// that with one existence lookup per message (`DatabaseEngine::get_message`)
/// is an N+1 query pattern over a fetched chunk; instead, this classifies an
/// entire folder's chunk with a single batched
/// `DatabaseEngine::existing_message_ids` lookup up front, then folds in a
/// per-batch "already seen this pass" set while iterating so a chunk that
/// itself contains a repeated id (e.g. a backend surfacing the same message
/// twice in one fetch) still fires at most once -- exactly as if each
/// message's existence had been checked one at a time immediately before its
/// own `save_email`.
/// # Why classification is per placement, not per message
///
/// Returns the total number of messages processed (a message id "processed"
/// more than once, e.g. by a backend that returns the same id from multiple
/// folders, is counted once per occurrence -- `save_email` itself is
/// `INSERT OR REPLACE`, so storage stays deduplicated by message id
/// regardless).
/// Message identity is folder-independent: a message already stored from
/// `INBOX` has the same key when it turns up in `Archive`. Asking "have I seen
/// this *message*?" therefore answers `yes` for a mailbox this daemon has never
/// filtered, and the arrival silently fires nothing -- no error, no log. The
/// question that matches the intent is "have I seen this *occupancy*?", so this
/// classifies on [`PlacementKey`] via
/// [`DatabaseEngine::existing_placements`].
///
/// Classification happens BEFORE the write, because `save_email_at` upserts and
/// would otherwise erase the distinction between a first arrival and a
/// re-report. One batched lookup covers the whole chunk rather than an N+1 of
/// per-row existence checks, and a per-pass "already seen" set folds in on top
/// so a backend that surfaces the same occupancy twice in one fetch still
/// classifies it as new exactly once.
///
/// Firing once per *arrival* is not the same as acting once per *message*; the
/// second guarantee lives in the claim inside [`apply_filter_actions`].
///
/// # Absence
///
/// [`nuncio_mail::FolderChanges::present`] is honoured as written: `None` means
/// the pass was incremental and cannot speak to absence, and treating it as
/// "nothing is present" would delete the whole folder. Only `Some(list)` --
/// which `nuncio-mail` withholds when a SELECT omitted UIDVALIDITY, since the
/// keys it could build would name no stored row -- licenses deleting the stored
/// placements missing from it.
///
/// Returns the total number of occupancies processed: one message reported from
/// two folders counts twice, because two mailboxes really did report it.
///
/// Contains no event-bus or credential logic -- pure fetch-and-persist, so
/// it is the smallest unit tests can exercise directly with a
@@ -248,25 +305,29 @@ async fn fetch_and_persist(
let changes = backend
.sync_changes(&folder.id, last_state.as_deref())
.await?;
let emails = changes.upserts;
let fetched = emails.len();
let placed = changes.upserts;
let fetched = placed.len();
let new_state = changes.next_state;
// One round trip classifies the whole chunk instead of one lookup per
// message. A lookup failure that is not "not found" would be a
// genuine DB error; `existing_message_ids` surfaces those as `Err`
// rather than silently treating the batch as "not new", so callers
// never persist and skip-filter on an unverified assumption.
let chunk_ids: Vec<String> = emails.iter().map(|e| e.id.clone()).collect();
let already_present = db.existing_message_ids(&chunk_ids).await?;
// occupancy. This asks about *placements*, not messages: identity is
// folder-independent, so a message already stored from another folder
// is still a first arrival here, and testing the message key instead
// would classify it as seen and skip filtering it entirely. A lookup
// failure that is not "not found" surfaces as `Err` rather than
// silently reading as "not new", so nothing is persisted and
// skip-filtered on an unverified assumption.
let chunk_keys: Vec<PlacementKey> = placed.iter().map(|p| p.placement.key()).collect();
let already_present = db.existing_placements(&chunk_keys).await?;
let mut seen_this_pass = std::collections::HashSet::new();
for email in emails {
let is_new =
!already_present.contains(&email.id) && seen_this_pass.insert(email.id.clone());
db.save_email(&email).await?;
for message in placed {
let key = message.placement.key();
let is_new = !already_present.contains(&key) && seen_this_pass.insert(key);
db.save_email_at(&message.email, message.source, &message.placement)
.await?;
if is_new {
apply_filter_actions(db, filter_engine, &email).await;
apply_filter_actions(db, filter_engine, &message.email, &message.placement).await;
}
synced += 1;
}
@@ -274,31 +335,39 @@ async fn fetch_and_persist(
// could only ever add: a message moved or deleted by any other client
// -- another daemon, a phone, webmail -- stayed in the local store
// forever, and every account accumulated ghosts.
if let Some(acct) = account_id {
let mut gone: Vec<String> = changes.removals;
//
// What a folder stops mentioning is an occupancy, never a message: the
// same mail may still sit in another mailbox, and it goes only when its
// last placement does -- which `delete_placements` decides, not this.
//
// A `PlacementKey` names its own account, so an explicitly-reported
// removal is actionable with no ambient account context at all. Only
// the `present` diff needs one, because its other half is a lookup
// keyed by (account, folder).
let mut gone: Vec<PlacementKey> = changes.removals;
// `present` is the folder's complete contents when the pass was
// able to enumerate them. `None` and `Some(vec![])` are NOT the
// same: `None` means the pass was incremental and cannot speak to
// absence, and treating it as "nothing is present" would delete
// the folder.
if let Some(present) = changes.present {
let present: std::collections::HashSet<String> = present.into_iter().collect();
let stored = db.message_ids_in_folder(acct, &folder.id).await?;
gone.extend(stored.into_iter().filter(|id| !present.contains(id)));
}
// `present` is the folder's complete contents when the pass was
// able to enumerate them. `None` and `Some(vec![])` are NOT the
// same: `None` means the pass was incremental and cannot speak to
// absence, and treating it as "nothing is present" would delete
// the folder.
if let (Some(acct), Some(present)) = (account_id, changes.present) {
let present: std::collections::HashSet<PlacementKey> = present.into_iter().collect();
let stored = db.placements_in_folder(acct, &folder.id).await?;
gone.extend(stored.into_iter().filter(|key| !present.contains(key)));
}
gone.sort_unstable();
gone.dedup();
if !gone.is_empty() {
let deleted = db.delete_messages(&gone).await?;
tracing::info!(
account_id = %acct,
folder_id = %folder.id,
deleted,
"removed messages that are no longer on the server"
);
}
gone.sort_unstable();
gone.dedup();
if !gone.is_empty() {
let outcome = db.delete_placements(&gone).await?;
tracing::info!(
account_id = ?account_id,
folder_id = %folder.id,
placements_removed = outcome.placements_removed,
messages_reaped = outcome.messages_reaped,
"removed placements that are no longer on the server"
);
}
// Persist the returned checkpoint only AFTER this folder's messages
@@ -446,9 +515,9 @@ pub async fn run_account_sync(
#[cfg(test)]
mod tests {
use super::*;
use nuncio_core::model::{Email, Folder};
use nuncio_core::model::{Email, Folder, IdentitySource, Placement};
use nuncio_core::{CoreEvent, EngineStatus, TlsMode};
use nuncio_mail::MockMailBackend;
use nuncio_mail::{MockMailBackend, PlacedMessage};
use nuncio_store::db::DatabaseEngine;
use serde_json::json;
use wiremock::matchers::{body_partial_json, method, path};
@@ -568,24 +637,58 @@ mod tests {
}
}
fn mock_email(id: &str, folder_id: &str, subject: &str) -> Email {
Email {
id: id.to_string(),
account_id: "acct-mock-1".to_string(),
folder_id: folder_id.to_string(),
remote_id: id.to_string(),
uid_validity: "1".to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1_700_000_000,
read: false,
body_plain: Some(format!("body for {id}")),
body_html: None,
attachments: Vec::new(),
/// One mock message as a backend would surface it: identity plus the
/// occupancy it was found in. `id` doubles as the folder-scoped UID, which
/// keeps distinct ids in one folder distinct occupancies.
fn mock_placed(id: &str, folder_id: &str, subject: &str) -> PlacedMessage {
PlacedMessage {
email: Email {
id: id.to_string(),
account_id: "acct-mock-1".to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "bob@nuncio.mx".to_string(),
received_at: 1_700_000_000,
body_plain: Some(format!("body for {id}")),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
},
source: IdentitySource::Surrogate,
placement: Placement {
account_id: "acct-mock-1".to_string(),
folder_id: folder_id.to_string(),
uid_validity: "1".to_string(),
remote_id: id.to_string(),
read: false,
},
}
}
/// The read flag of a message's one occupancy, for the single-placement
/// mock fixtures. Fails loudly rather than defaulting if the message turns
/// out to sit in no mailbox at all -- "unread" and "not there" are
/// different answers and must not be conflated by a test helper.
async fn read_flag_of(db: &DatabaseEngine, message_key: &str) -> bool {
let placements = db
.placements_of(message_key)
.await
.expect("read the message's placements");
assert_eq!(
placements.len(),
1,
"fixture expects exactly one occupancy of '{message_key}'"
);
placements[0].read
}
/// The message key every [`MockMailBackend::with_same_message_in`]
/// occupancy carries.
fn shared_key() -> &'static str {
MockMailBackend::SHARED_MESSAGE_KEY
}
#[tokio::test]
async fn sync_with_backend_persists_mock_messages_and_emits_events() {
let (db, _dir) = DatabaseEngine::connect_ephemeral()
@@ -601,8 +704,8 @@ mod tests {
total_messages: 2,
unread_messages: 2,
});
mock.add_message(mock_email("m1", "inbox", "Hello"));
mock.add_message(mock_email("m2", "inbox", "World"));
mock.add_message(mock_placed("m1", "inbox", "Hello"));
mock.add_message(mock_placed("m2", "inbox", "World"));
let filter_engine = empty_filter_engine();
let synced = sync_with_backend(
@@ -617,11 +720,14 @@ mod tests {
assert_eq!(synced, 2);
let persisted = db
.list_messages("inbox", 10)
.list_messages("acct-mock-1", "inbox", 10)
.await
.expect("list persisted messages");
assert_eq!(persisted.len(), 2);
let subjects: Vec<String> = persisted.iter().map(|e| e.subject.clone()).collect();
let subjects: Vec<String> = persisted
.iter()
.map(|(email, _placement)| email.subject.clone())
.collect();
assert!(subjects.contains(&"Hello".to_string()));
assert!(subjects.contains(&"World".to_string()));
@@ -815,12 +921,13 @@ mod tests {
// folder, and its protocol-native JMAP object id round-trips in
// `remote_id`.
let persisted = db
.list_messages("inbox", 10)
.list_messages("acct-jmap-real-1", "inbox", 10)
.await
.expect("jmap message persisted");
assert_eq!(persisted.len(), 1);
assert_eq!(persisted[0].subject, "Welcome to JMAP Sync");
assert_eq!(persisted[0].remote_id, "jmap-msg-1");
let (email, placement) = &persisted[0];
assert_eq!(email.subject, "Welcome to JMAP Sync");
assert_eq!(placement.remote_id, "jmap-msg-1");
assert_eq!(
events.recv().await.expect("start event"),
@@ -927,17 +1034,16 @@ mod tests {
total_messages: 1,
unread_messages: 1,
});
mock.add_message(mock_email("m-urgent", "inbox", "Urgent: server down"));
mock.add_message(mock_placed("m-urgent", "inbox", "Urgent: server down"));
let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
.await
.expect("sync succeeds");
assert_eq!(synced, 1);
let stored = db.get_message("m-urgent").await.expect("message persisted");
assert!(
stored.read,
"MARK READ action must genuinely flip the stored read flag"
read_flag_of(&db, "m-urgent").await,
"MARK READ action must genuinely flip the occupancy's read flag"
);
let logs = db
@@ -979,7 +1085,7 @@ mod tests {
total_messages: 1,
unread_messages: 1,
});
mock.add_message(mock_email("m-archive", "inbox", "Please Archive Me"));
mock.add_message(mock_placed("m-archive", "inbox", "Please Archive Me"));
let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
.await
@@ -1004,11 +1110,7 @@ mod tests {
assert_eq!(logs[0].message_id, "m-archive");
// The remote action must not have applied a local read-flag change.
let stored = db
.get_message("m-archive")
.await
.expect("message persisted");
assert!(!stored.read);
assert!(!read_flag_of(&db, "m-archive").await);
}
#[tokio::test]
@@ -1033,15 +1135,17 @@ mod tests {
total_messages: 1,
unread_messages: 1,
});
mock.add_message(mock_email("m-plain", "inbox", "Just a normal update"));
mock.add_message(mock_placed("m-plain", "inbox", "Just a normal update"));
let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
.await
.expect("sync succeeds");
assert_eq!(synced, 1);
let stored = db.get_message("m-plain").await.expect("message persisted");
assert!(!stored.read, "non-matching message must stay untouched");
assert!(
!read_flag_of(&db, "m-plain").await,
"non-matching message must stay untouched"
);
let logs = db
.list_filter_execution_logs(10)
@@ -1092,7 +1196,9 @@ mod tests {
total_messages: 1,
unread_messages: 1,
});
mock.add_message(mock_email("m-repeat", "inbox", "Urgent: server down"));
let repeat = mock_placed("m-repeat", "inbox", "Urgent: server down");
let repeat_key = repeat.placement.key();
mock.add_message(repeat);
// First sync: the message is genuinely new, so both rules must fire.
let synced_first = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
@@ -1100,11 +1206,7 @@ mod tests {
.expect("first sync succeeds");
assert_eq!(synced_first, 1);
let stored_after_first = db
.get_message("m-repeat")
.await
.expect("message persisted after first sync");
assert!(stored_after_first.read);
assert!(read_flag_of(&db, "m-repeat").await);
let logs_after_first = db
.list_filter_execution_logs(10)
@@ -1121,7 +1223,7 @@ mod tests {
// Manually flip the message back to unread, exactly as a user
// reading and then un-reading it would -- proves the second sync
// doesn't re-flip it back to read via a repeat MARK READ fire.
db.set_message_read("m-repeat", false)
db.set_placement_read(&repeat_key, false)
.await
.expect("manually mark unread");
@@ -1132,12 +1234,8 @@ mod tests {
.expect("second sync succeeds");
assert_eq!(synced_second, 1);
let stored_after_second = db
.get_message("m-repeat")
.await
.expect("message persisted after second sync");
assert!(
!stored_after_second.read,
!read_flag_of(&db, "m-repeat").await,
"a re-sync of an already-seen message must not re-fire MARK READ"
);
@@ -1164,10 +1262,10 @@ mod tests {
#[tokio::test]
async fn sync_with_backend_fires_filters_only_for_new_messages_in_a_mixed_batch() {
// A single fetched chunk containing a mix of an already-seen message
// (persisted by a prior sync) and genuinely new ones must fire filter
// actions ONLY for the new ones, via the batched
// `existing_message_ids` classification -- not a per-message lookup.
// A single fetched chunk containing a mix of an already-stored
// occupancy (persisted by a prior sync) and genuinely new ones must
// fire filter actions ONLY for the new ones, via the batched
// `existing_placements` classification -- not a per-row lookup.
let (db, _dir) = DatabaseEngine::connect_ephemeral()
.await
.expect("ephemeral db");
@@ -1181,16 +1279,13 @@ mod tests {
.expect("parse rule");
let filter_engine = FilterEngine::new(vec![mark_read_rule]).expect("compile rule");
// Pre-seed one message directly, simulating it having landed in a
// Pre-seed one occupancy directly, simulating it having landed in a
// prior sync pass. It must NOT re-fire even though this pass's
// backend still reports it (sync is not yet incremental).
db.save_email(&mock_email(
"m-already-seen",
"inbox",
"Urgent: known issue",
))
.await
.expect("pre-seed existing message");
let seeded = mock_placed("m-already-seen", "inbox", "Urgent: known issue");
db.save_email_at(&seeded.email, seeded.source, &seeded.placement)
.await
.expect("pre-seed existing occupancy");
let mock = MockMailBackend::new();
mock.add_folder(Folder {
@@ -1200,9 +1295,13 @@ mod tests {
unread_messages: 3,
});
// Mixed chunk: one already-present id, two genuinely new ids.
mock.add_message(mock_email("m-already-seen", "inbox", "Urgent: known issue"));
mock.add_message(mock_email("m-new-1", "inbox", "Urgent: brand new"));
mock.add_message(mock_email("m-new-2", "inbox", "Urgent: also new"));
mock.add_message(mock_placed(
"m-already-seen",
"inbox",
"Urgent: known issue",
));
mock.add_message(mock_placed("m-new-1", "inbox", "Urgent: brand new"));
mock.add_message(mock_placed("m-new-2", "inbox", "Urgent: also new"));
let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
.await
@@ -1224,16 +1323,12 @@ mod tests {
assert!(fired_ids.contains("m-new-2"));
assert!(!fired_ids.contains("m-already-seen"));
// The pre-seeded message must still be persisted (INSERT OR REPLACE
// semantics are unaffected by the batched new/seen classification)
// but its read flag must be untouched by this pass's MARK READ rule.
let already_seen = db
.get_message("m-already-seen")
.await
.expect("still persisted");
// The pre-seeded occupancy must still be persisted (the upsert is
// unaffected by the batched new/seen classification) but its read flag
// must be untouched by this pass's MARK READ rule.
assert!(
!already_seen.read,
"a message that already existed before this sync must not re-fire MARK READ"
!read_flag_of(&db, "m-already-seen").await,
"an occupancy that already existed before this sync must not re-fire MARK READ"
);
}
@@ -1256,7 +1351,7 @@ mod tests {
total_messages: 1,
unread_messages: 1,
});
mock.add_message(mock_email("m-inc-1", "inbox", "First"));
mock.add_message(mock_placed("m-inc-1", "inbox", "First"));
let filter_engine = empty_filter_engine();
@@ -1301,4 +1396,224 @@ mod tests {
vec![None, Some("uidnext-4242".to_string())],
);
}
/// A rule that matches the one message the shared-identity mock stages.
fn rule_matching_the_shared_message() -> nuncio_filter::FilterRule {
nuncio_filter::NsqlParser::parse_rule(
"Archive Shared",
1,
"WHERE subject CONTAINS 'Shared' ACTION MOVE TO 'Archive'",
)
.expect("parse rule")
}
#[tokio::test]
async fn a_message_arriving_in_a_second_folder_still_gets_evaluated() {
// The regression this guards: with identity now folder-independent, a
// message already stored from INBOX has a known message_key, so a
// message-scoped "already present" test would classify its arrival in
// Archive as old and skip filtering it entirely -- silently, with no
// error and nothing in the log.
let (db, _dir) = DatabaseEngine::connect_ephemeral()
.await
.expect("ephemeral db");
let event_bus = EventBus::new();
let rule = rule_matching_the_shared_message();
let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule");
// Already stored, from INBOX. The message key is therefore known
// before this sync begins -- but nothing has ever filtered Archive.
let in_inbox = MockMailBackend::shared_message_in("INBOX", "1");
db.save_email_at(&in_inbox.email, in_inbox.source, &in_inbox.placement)
.await
.expect("pre-seed the INBOX occupancy");
let backend = MockMailBackend::with_same_message_in(&["Archive"]);
let processed = fetch_and_persist(
&db,
&event_bus,
&backend,
&filter_engine,
Some(MockMailBackend::SHARED_ACCOUNT_ID),
)
.await
.expect("sync succeeds");
assert_eq!(processed, 1, "the Archive occupancy is processed");
let placements = db
.placements_of(shared_key())
.await
.expect("read the message's placements");
assert_eq!(
placements.len(),
2,
"one message, now occupying both mailboxes"
);
assert!(
db.has_filter_fired(&rule.id, shared_key())
.await
.expect("read the claim ledger"),
"an arrival in a second folder must be evaluated, not skipped as already-seen"
);
let pending = db
.list_pending_mutations(10)
.await
.expect("list pending mutations");
assert_eq!(
pending.len(),
1,
"with a real outbox mutation behind the fire, not just a ledger row"
);
assert_eq!(pending[0].message_id, shared_key());
}
#[tokio::test]
async fn a_rule_fires_once_for_a_message_that_lands_in_two_folders() {
// MOVE and FLAG converge under repetition; FORWARD and CALL WEBHOOK do
// not. Two occupancies of one message must not page an on-call
// engineer twice.
let (db, _dir) = DatabaseEngine::connect_ephemeral()
.await
.expect("ephemeral db");
let event_bus = EventBus::new();
let rule = rule_matching_the_shared_message();
let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule");
let backend = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]);
let processed = fetch_and_persist(
&db,
&event_bus,
&backend,
&filter_engine,
Some(MockMailBackend::SHARED_ACCOUNT_ID),
)
.await
.expect("sync succeeds");
assert_eq!(processed, 2, "both occupancies are processed");
let placements = db
.placements_of(shared_key())
.await
.expect("read the message's placements");
assert_eq!(placements.len(), 2);
assert_eq!(
db.filter_fire_count().await.expect("read the claim ledger"),
1,
"one claim, therefore one set of actions"
);
assert!(db
.has_filter_fired(&rule.id, shared_key())
.await
.expect("read the claim ledger"));
let pending = db
.list_pending_mutations(10)
.await
.expect("list pending mutations");
assert_eq!(
pending.len(),
1,
"one outbox mutation for the message, not one per folder"
);
let logs = db
.list_filter_execution_logs(10)
.await
.expect("list execution logs");
assert_eq!(logs.len(), 1, "and one audit entry, not one per folder");
}
#[tokio::test]
async fn an_unverifiable_claim_skips_the_actions_rather_than_licensing_them() {
// Fail closed. If the claim itself errors, the engine cannot know
// whether this rule has already acted on this message -- and a lost
// claim check on a FORWARD is a duplicate mail the recipient cannot
// un-receive. Dropping the ledger table makes the claim error for real
// while every other write still works, so a mutation appearing here
// would be a genuine "acted without a claim", not a dead store.
let (db, _dir) = DatabaseEngine::connect_ephemeral()
.await
.expect("ephemeral db");
let rule = rule_matching_the_shared_message();
let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule");
let placed = MockMailBackend::shared_message_in("INBOX", "1");
db.save_email_at(&placed.email, placed.source, &placed.placement)
.await
.expect("store the occupancy");
sqlx::query("DROP TABLE filter_fired")
.execute(db.pool())
.await
.expect("drop the claim ledger");
let applied =
apply_filter_actions(&db, &filter_engine, &placed.email, &placed.placement).await;
assert_eq!(applied, 0, "an unclaimed rule must apply nothing");
let pending = db
.list_pending_mutations(10)
.await
.expect("list pending mutations");
assert!(
pending.is_empty(),
"no outbox mutation may be enqueued off an unverifiable claim"
);
let logs = db
.list_filter_execution_logs(10)
.await
.expect("list execution logs");
assert!(
logs.is_empty(),
"and nothing may be recorded as having fired"
);
}
#[tokio::test]
async fn an_account_less_pass_still_applies_server_confirmed_removals() {
// A `PlacementKey` names its own account, so an explicitly-reported
// removal needs no ambient account context. Gating the whole removals
// block on `account_id.is_some()` would silently discard a VANISHED
// report on a mock-driven `SyncAll`, leaving a ghost the server has
// already told us is gone.
let (db, _dir) = DatabaseEngine::connect_ephemeral()
.await
.expect("ephemeral db");
let event_bus = EventBus::new();
let filter_engine = empty_filter_engine();
let staged = mock_placed("m-vanished", "inbox", "Deleted elsewhere");
db.save_email_at(&staged.email, staged.source, &staged.placement)
.await
.expect("store the occupancy a prior pass left behind");
let mock = MockMailBackend::new();
mock.add_folder(Folder {
id: "inbox".to_string(),
name: "Inbox".to_string(),
total_messages: 0,
unread_messages: 0,
});
mock.set_removals(vec![staged.placement.key()]);
// `None` account: the pass has nowhere to key a checkpoint and cannot
// run the `present` diff, but the removal is still actionable.
let synced = fetch_and_persist(&db, &event_bus, &mock, &filter_engine, None)
.await
.expect("sync succeeds");
assert_eq!(synced, 0, "nothing new arrived");
let placements = db
.placements_of("m-vanished")
.await
.expect("read the message's placements");
assert!(
placements.is_empty(),
"the server-confirmed removal must be applied without an account id"
);
}
}
+50 -25
View File
@@ -3,7 +3,7 @@
//! Boots a real `nunciod` daemon gRPC server (`nunciod::grpc::
//! serve_on_listener`) on an ephemeral loopback port, backed by a real
//! temp file-backed [`DatabaseEngine`] seeded directly via
//! [`DatabaseEngine::save_email`] (standing in for messages that were synced
//! [`DatabaseEngine::save_email_at`] (standing in for messages that were synced
//! before the active rule set existed) and a live [`FilterEngine`] carrying
//! a real persisted rule.
//!
@@ -25,7 +25,7 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use nuncio_core::model::Email;
use nuncio_core::model::{Email, IdentitySource, Placement};
use nuncio_core::EventBus;
use nuncio_filter::{FilterEngine, NsqlParser};
use nuncio_proto::v1::filters_client::FiltersClient;
@@ -42,20 +42,52 @@ fn sample_email(id: &str, subject: &str) -> Email {
Email {
id: id.to_string(),
account_id: ACCOUNT_ID.to_string(),
folder_id: "inbox".to_string(),
remote_id: id.to_string(),
uid_validity: "1".to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "triage-e2e@nuncio.mx".to_string(),
received_at: 1_700_000_000,
read: false,
body_plain: Some("body".to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
}
}
/// The INBOX occupancy a seeded message sits in. `id` doubles as the UID, so
/// distinct messages occupy distinct rows.
fn sample_placement(id: &str) -> Placement {
Placement {
account_id: ACCOUNT_ID.to_string(),
folder_id: "inbox".to_string(),
uid_validity: "1".to_string(),
remote_id: id.to_string(),
read: false,
}
}
/// Seed a message and its occupancy the way a sync pass would.
async fn seed(db: &DatabaseEngine, id: &str, subject: &str) {
db.save_email_at(
&sample_email(id, subject),
IdentitySource::Surrogate,
&sample_placement(id),
)
.await
.unwrap_or_else(|e| panic!("seed message '{id}': {e}"));
}
/// The read flag of a seeded message's single occupancy.
async fn read_flag_of(db: &DatabaseEngine, id: &str) -> bool {
let placements = db.placements_of(id).await.expect("read placements");
assert_eq!(
placements.len(),
1,
"'{id}' should occupy exactly one folder"
);
placements[0].read
}
/// Boots the real daemon gRPC server (bearer-token authenticated) on an
/// ephemeral loopback port, backed by a real temp file-backed
/// `DatabaseEngine` and the given live `FilterEngine`. Returns the server
@@ -143,15 +175,9 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun
// Seed messages directly into the store, standing in for mail that was
// synced BEFORE these rules existed -- exactly the scenario `Triage` is
// for. Two match (one per rule), one does not.
db.save_email(&sample_email("m-urgent", "Urgent: server down"))
.await
.expect("seed urgent message");
db.save_email(&sample_email("m-archive", "Please Archive Me"))
.await
.expect("seed archive message");
db.save_email(&sample_email("m-plain", "Just saying hi"))
.await
.expect("seed non-matching message");
seed(&db, "m-urgent", "Urgent: server down").await;
seed(&db, "m-archive", "Please Archive Me").await;
seed(&db, "m-plain", "Just saying hi").await;
let mut client = FiltersClient::connect(format!("http://{addr}"))
.await
@@ -188,8 +214,10 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun
// Real side effects, not a fabricated count: the MARK READ action
// actually flipped the stored read flag...
let urgent = db.get_message("m-urgent").await.expect("get urgent");
assert!(urgent.read, "MARK READ must genuinely flip the read flag");
assert!(
read_flag_of(&db, "m-urgent").await,
"MARK READ must genuinely flip the occupancy's read flag"
);
// ...and the MOVE TO action actually queued a real outbox mutation.
let pending = db
@@ -203,8 +231,10 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun
);
assert_eq!(pending[0].message_id, "m-archive");
let plain = db.get_message("m-plain").await.expect("get plain");
assert!(!plain.read, "a non-matching message must be left untouched");
assert!(
!read_flag_of(&db, "m-plain").await,
"a non-matching message must be left untouched"
);
}
#[tokio::test]
@@ -219,12 +249,7 @@ async fn triage_streams_multiple_chunks_for_a_store_bigger_than_one_chunk() {
// updates before the final `done: true` -- proof this is a real
// multi-chunk stream, not one batch disguised as one.
for i in 0..5 {
db.save_email(&sample_email(
&format!("m-{i:02}"),
&format!("Message number {i}"),
))
.await
.expect("seed message");
seed(&db, &format!("m-{i:02}"), &format!("Message number {i}")).await;
}
let mut client = FiltersClient::connect(format!("http://{addr}"))
+31 -8
View File
@@ -27,10 +27,10 @@ use std::sync::{Arc, Mutex};
use nuncio_cal::{CalDavAccountConfig, CalDavClient, CalendarBackend};
use nuncio_contacts::{CardDavAccountConfig, CardDavClient, ContactsBackend};
use nuncio_core::model::{Email, Folder};
use nuncio_core::model::{Email, Folder, IdentitySource, Placement};
use nuncio_core::EventBus;
use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher};
use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender};
use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender, PlacedMessage};
use nuncio_proto::v1::{
AccountConfig, AddAccountRequest, GetStatusRequest, MarkReadRequest, SendMessageRequest,
SyncRequest, TlsMode,
@@ -139,17 +139,35 @@ fn sentinel_inbound_email() -> Email {
Email {
id: "msg-canary-1".to_string(),
account_id: ACCOUNT_ID.to_string(),
folder_id: "inbox".to_string(),
remote_id: "msg-canary-1".to_string(),
uid_validity: "1".to_string(),
subject: SENTINEL_SUBJECT.to_string(),
sender: SENTINEL_SENDER.to_string(),
recipient: SENTINEL_RECIPIENT.to_string(),
received_at: 1_700_000_000,
read: false,
body_plain: Some(SENTINEL_BODY.to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
}
}
/// The INBOX occupancy the sentinel message is found in.
fn sentinel_placement() -> Placement {
Placement {
account_id: ACCOUNT_ID.to_string(),
folder_id: "inbox".to_string(),
uid_validity: "1".to_string(),
remote_id: "msg-canary-1".to_string(),
read: false,
}
}
/// The sentinel message as a backend would surface it.
fn sentinel_placed_message() -> PlacedMessage {
PlacedMessage {
email: sentinel_inbound_email(),
source: IdentitySource::Surrogate,
placement: sentinel_placement(),
}
}
@@ -212,7 +230,7 @@ async fn drive_daemon() {
total_messages: 1,
unread_messages: 1,
});
mock_backend.add_message(sentinel_inbound_email());
mock_backend.add_message(sentinel_placed_message());
let mock_sender = MockMessageSender::new();
let overrides = MailEngineOverrides {
mail_backend: Some(Arc::new(mock_backend)),
@@ -307,7 +325,12 @@ async fn drive_filter_and_webhook() {
)
.expect("parse rule");
let engine = FilterEngine::new(vec![rule]).expect("compile rule");
let matches = engine.evaluate(&sentinel_inbound_email());
let sentinel = sentinel_inbound_email();
let placement = sentinel_placement();
let matches = engine.evaluate(nuncio_filter::PlacedEmail {
email: &sentinel,
placement: &placement,
});
assert_eq!(matches.len(), 1, "rule must fire on the sentinel message");
let mock_server = MockServer::start().await;
@@ -12,7 +12,7 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use async_trait::async_trait;
use nuncio_core::model::Email;
use nuncio_core::model::{Email, IdentitySource, Placement};
use nuncio_core::EventBus;
use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher, WebhookError};
use nuncio_mail::{
@@ -116,17 +116,28 @@ fn sample_email(subject: &str) -> Email {
Email {
id: "surrogate-inbox-42".to_string(),
account_id: ACCOUNT_ID.to_string(),
folder_id: FOLDER_ID.to_string(),
remote_id: "42".to_string(),
uid_validity: "1".to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: "owner@nuncio.mx".to_string(),
received_at: 1_700_000_000,
read: false,
body_plain: Some("original body".to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
}
}
/// The mailbox occupancy the seeded message sits in. The outbox recovers its
/// remote addressing from here now -- the UID and the UIDVALIDITY scope it was
/// captured under are properties of the occupancy, not of the message.
fn sample_placement(remote_id: &str) -> Placement {
Placement {
account_id: ACCOUNT_ID.to_string(),
folder_id: FOLDER_ID.to_string(),
uid_validity: "1".to_string(),
remote_id: remote_id.to_string(),
read: false,
}
}
@@ -142,11 +153,14 @@ async fn seed_and_enqueue(db: &DatabaseEngine, rule_nsql: &str, subject: &str) {
.expect("save folder checkpoint");
let email = sample_email(subject);
db.save_email(&email).await.expect("save email");
let placement = sample_placement("42");
db.save_email_at(&email, IdentitySource::Surrogate, &placement)
.await
.expect("save email");
let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule");
let engine = FilterEngine::new(vec![rule]).expect("compile rule");
apply_filter_actions(db, &engine, &email).await;
apply_filter_actions(db, &engine, &email, &placement).await;
let pending = db.list_pending_mutations(10).await.expect("list pending");
assert_eq!(
@@ -173,14 +187,16 @@ async fn seed_and_enqueue_distinct(
.await
.expect("save folder checkpoint");
// `remote_id`/`uid_validity` (not just `id`) are part of the store's
// `messages` identity UNIQUE index -- leaving them at `sample_email`'s
// shared default would make this INSERT OR REPLACE the OTHER seeded
// message sharing that same remote identity out from under it.
// The occupancy needs its own UID as well as its own message key: the
// placements primary key is (account, folder, uidvalidity, uid), so
// leaving `remote_id` at the shared default would repoint the OTHER
// seeded message's occupancy at this message.
let mut email = sample_email(subject);
email.id = message_id.to_string();
email.remote_id = message_id.to_string();
db.save_email(&email).await.expect("save email");
let placement = sample_placement(message_id);
db.save_email_at(&email, IdentitySource::Surrogate, &placement)
.await
.expect("save email");
let before = db
.list_pending_mutations(100)
@@ -190,7 +206,7 @@ async fn seed_and_enqueue_distinct(
let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule");
let engine = FilterEngine::new(vec![rule]).expect("compile rule");
apply_filter_actions(db, &engine, &email).await;
apply_filter_actions(db, &engine, &email, &placement).await;
let after = db
.list_pending_mutations(100)
+27 -17
View File
@@ -31,10 +31,10 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use nuncio_core::model::{Email, Folder};
use nuncio_core::model::{Email, Folder, IdentitySource, Placement};
use nuncio_core::EventBus;
use nuncio_filter::FilterEngine;
use nuncio_mail::{MockMailBackend, MockMessageSender};
use nuncio_mail::{MockMailBackend, MockMessageSender, PlacedMessage};
use nuncio_proto::v1::event::Kind;
use nuncio_proto::v1::{
AccountConfig, AddAccountRequest, GetMessageRequest, ListAccountsRequest, ListMessagesRequest,
@@ -73,21 +73,31 @@ fn sample_account_config() -> AccountConfig {
}
}
fn mock_inbound_email(id: &str, subject: &str, body: &str) -> Email {
Email {
id: id.to_string(),
account_id: ACCOUNT_ID.to_string(),
folder_id: "inbox".to_string(),
remote_id: id.to_string(),
uid_validity: "1".to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: ACCOUNT_EMAIL.to_string(),
received_at: 1_700_000_000,
read: false,
body_plain: Some(body.to_string()),
body_html: None,
attachments: Vec::new(),
/// One inbound message as a backend surfaces it: identity, the tier the key
/// came from, and the INBOX occupancy this pass found it in.
fn mock_inbound_email(id: &str, subject: &str, body: &str) -> PlacedMessage {
PlacedMessage {
email: Email {
id: id.to_string(),
account_id: ACCOUNT_ID.to_string(),
subject: subject.to_string(),
sender: "alice@nuncio.mx".to_string(),
recipient: ACCOUNT_EMAIL.to_string(),
received_at: 1_700_000_000,
body_plain: Some(body.to_string()),
body_html: None,
attachments: Vec::new(),
message_id: None,
content_hash: None,
},
source: IdentitySource::Surrogate,
placement: Placement {
account_id: ACCOUNT_ID.to_string(),
folder_id: "inbox".to_string(),
uid_validity: "1".to_string(),
remote_id: id.to_string(),
read: false,
},
}
}
+15 -5
View File
@@ -104,7 +104,7 @@ part of the original M1–M7 numbering:
`tls_mode_from_db` silent `ImplicitTls` fallback (#271), the
`NaturalLanguageScheduler` that returns wrong times (#258), plus CLI and audit
cleanups.
- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 44 open). Six tracks
- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 56 open). Six tracks
that must land before `nuncio.v1` can be frozen: **WS-A** contract hardening
(typed errors, `Timestamp`/`Duration` unification, keyset pagination, id
conventions, enum hygiene, account transport `oneof`); **WS-B** mail model and
@@ -228,19 +228,29 @@ The engine becomes production-solid.
### M6.5, WS and OBS — the work between M6 and M7
Opened after M5, when the reshape needed before an honest `v1` freeze proved
larger than M6. Status as of 2026-08-07:
larger than M6. Status as of 2026-08-12:
| Track | Closed / Open | Scope |
| :--- | :---: | :--- |
| **M6.5** Feature completeness & de-fabrication | 3 / 8 | Remove the remaining fabricated or silently-wrong paths (Contacts sync wiring #255, DAV XML namespaces #257, TLS-mode fallback #271, NL scheduler #258) |
| **WS-A** Contract hardening (pre-freeze reshape) | 6 / 3 | Typed errors, `Timestamp`/`Duration`, keyset pagination, id conventions, enum hygiene, account transport `oneof` — **gates M7** |
| **WS-B** Mail model & mutations | 1 / 9 | Message identity, drafts, folder management, attachments on receive |
| **WS-C** Sync, push & lifecycle | 4 / 4 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown |
| **WS-B** Mail model & mutations | 1 / 14 | Message identity, drafts, folder management, attachments on receive |
| **WS-C** Sync, push & lifecycle | 4 / 9 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown |
| **WS-D** Security to 9/10 | 4 / 8 | Whole-DB SQLCipher (#299), UDS / named-pipe IPC (#305), signed updates (#300), HTML sanitization (#302) |
| **WS-E** Calendar/Contacts write & Search | 2 / 10 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service |
| **WS-E** Calendar/Contacts write & Search | 2 / 12 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service |
| **WS-F** Ops, usability & maintainability | 5 / 10 | CLI polish, `System.Backup`, and the `grpc.rs` (#333) / `DatabaseEngine` (#334) decompositions |
| **OBS-1 … OBS-9** Observability | 9 / 0 | **COMPLETE.** Tracing subscriber, RPC spans + request-id, domain/lifecycle coverage, protocol instrumentation, `GetStatus`, CLI verbosity, redaction canary |
WS-B and WS-C grew when **ADR 0002** decided
the convergent multi-engine sync model and broke it into #386–#397. Those are
in flight, not closed: the message/placement identity split, Message-ID and
content-hash capture, `user_version` gating, the raw IMAP command layer, the
four-rung change enumeration ladder, and three-state verified mutations are all
implemented and green on a branch, and the counts above move when it merges.
Still open behind them: the mutation RPCs and durable conflict surface (#395),
DAV href/ETag identity and `sync-collection` (#392, #393), the scheduler and
rate limiting (#396), and the ≥3-engine convergence harness (#397).
The OBS stories (#357–#365) were merged without a milestone assignment; the
epic is complete regardless, and `docs/LOGGING.md` is its reference.
+4 -2
View File
@@ -280,11 +280,13 @@
<p class="cbody">Opened after M5, when the reshape needed before an honest <code>v1</code> freeze proved larger
than M6 alone. <b>M6.5</b> de-fabrication (3/11) removes the remaining silently-wrong paths: Contacts sync
wiring (#255), namespace-aware DAV XML (#257), the TLS-mode fallback (#271), the NL scheduler (#258).
<b>WS-A … WS-F</b> (22/66) is the pre-freeze reshape: contract hardening (WS-A, <b>gates M7</b>), mail model
<b>WS-A … WS-F</b> (22/78) is the pre-freeze reshape: contract hardening (WS-A, <b>gates M7</b>), mail model
&amp; mutations (WS-B), sync/push/lifecycle (WS-C), security to 9/10 (WS-D — SQLCipher #299, UDS &amp;
named-pipe IPC #305, signed updates #300), calendar/contacts write-back &amp; search (WS-E), and ops,
usability &amp; maintainability (WS-F — including the <code>grpc.rs</code> #333 and <code>DatabaseEngine</code>
#334 decompositions). <b>OBS-1 … OBS-9 is complete</b> (9/9, #357–#365): layered tracing subscriber, per-RPC
#334 decompositions). WS-B and WS-C grew when <b>ADR 0002</b> decided the convergent multi-engine sync
model and broke it into #386–#397; the message/placement identity split, change enumeration and verified
mutations are implemented and green on a branch, not yet merged. <b>OBS-1 … OBS-9 is complete</b> (9/9, #357–#365): layered tracing subscriber, per-RPC
spans with request-id correlation, domain and lifecycle coverage, protocol instrumentation, a real
<code>System.GetStatus</code>, CLI <code>-v</code>/<code>-vv</code> with typed errors, and a
<code>Redacted&lt;T&gt;</code> policy with a no-secrets-in-logs canary.</p>
File diff suppressed because it is too large. Load diff