feat(store): split message identity from mailbox placements #405
No files matched your search
@@ -87,10 +87,13 @@ impl ExportEngine {
|
||||
writer.write_all(from_line.as_bytes())?;
|
||||
total_bytes += from_line.len() as u64;
|
||||
|
||||
// Headers
|
||||
// Headers. No X-Nuncio-Folder header: the folder is a property of
|
||||
// a placement now, not of the message, so `Email` has none to
|
||||
// name here. Restoring it would mean threading the relevant
|
||||
// `Placement` into this function's signature.
|
||||
let headers = format!(
|
||||
"Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nX-Nuncio-Folder: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n",
|
||||
email.id, email.sender, email.recipient, email.subject, date_str, email.account_id, email.folder_id
|
||||
"Message-ID: <{}>\r\nFrom: {}\r\nTo: {}\r\nSubject: {}\r\nDate: {}\r\nX-Nuncio-Account: {}\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n",
|
||||
email.id, email.sender, email.recipient, email.subject, date_str, email.account_id
|
||||
);
|
||||
writer.write_all(headers.as_bytes())?;
|
||||
total_bytes += headers.len() as u64;
|
||||
@@ -184,32 +187,28 @@ mod tests {
|
||||
Email {
|
||||
id: "msg-100".to_string(),
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "INBOX".to_string(),
|
||||
remote_id: "100".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: "Test Subject 1".to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1700000000,
|
||||
read: false,
|
||||
body_plain: Some("Hello world body text".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
Email {
|
||||
id: "msg-101".to_string(),
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "INBOX".to_string(),
|
||||
remote_id: "101".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: "Test Subject 2".to_string(),
|
||||
sender: "charlie@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1700000100,
|
||||
read: true,
|
||||
body_plain: Some("Second body text".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
+621
-24
@@ -61,33 +61,153 @@ pub struct Attachment {
|
||||
pub content: Bytes,
|
||||
}
|
||||
|
||||
/// Email message domain entity owned by Nuncio core.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Email {
|
||||
/// Opaque, deterministic surrogate identifier for the message.
|
||||
/// Which tier of the identity precedence produced a [`MessageIdentity`].
|
||||
///
|
||||
/// Persisted alongside the key so a later pass can tell a server-assigned
|
||||
/// identity from a locally-derived fallback without recomputing it, and so a
|
||||
/// store can be audited for how much of it rests on the weakest tier.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum IdentitySource {
|
||||
/// RFC 8474 OBJECTID `EMAILID` -- server-assigned and stable across folders.
|
||||
EmailId,
|
||||
/// Gmail's `X-GM-MSGID`, the same guarantee by a vendor extension.
|
||||
GmailMsgId,
|
||||
/// `Message-ID` **and** a content hash together. Never the header alone.
|
||||
MessageIdContent,
|
||||
/// Folder-scoped last resort: the message is only identified by where it
|
||||
/// currently sits, so the same mail in another folder is a second message.
|
||||
Surrogate,
|
||||
}
|
||||
|
||||
impl IdentitySource {
|
||||
/// Stable token persisted in `messages.identity_source`.
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::EmailId => "emailid",
|
||||
Self::GmailMsgId => "gmsgid",
|
||||
Self::MessageIdContent => "msgid_content",
|
||||
Self::Surrogate => "surrogate",
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a token written by [`Self::as_str`]. Returns `None` for anything
|
||||
/// else so an unrecognised value fails loudly rather than defaulting to a
|
||||
/// tier that would overstate how trustworthy the key is.
|
||||
///
|
||||
/// Derived from [`Email::surrogate_id`] over the account, folder,
|
||||
/// UIDVALIDITY, and protocol-native id. It carries no decodable internal
|
||||
/// encoding (callers must treat it as opaque) and is stable across
|
||||
/// re-syncs of the same message, so persisting it upserts the same row
|
||||
/// rather than duplicating. Distinct messages that happen to share a
|
||||
/// protocol id across folders or accounts hash to distinct surrogates, so
|
||||
/// one can never silently overwrite another.
|
||||
pub id: String,
|
||||
/// Account identifier owning the message.
|
||||
/// Deliberately not `std::str::FromStr`: that trait forces a `Result` and
|
||||
/// therefore an error type, when the only failure here is "not one of four
|
||||
/// known tokens" and `None` already says exactly that.
|
||||
pub fn from_token(s: &str) -> Option<Self> {
|
||||
match s {
|
||||
"emailid" => Some(Self::EmailId),
|
||||
"gmsgid" => Some(Self::GmailMsgId),
|
||||
"msgid_content" => Some(Self::MessageIdContent),
|
||||
"surrogate" => Some(Self::Surrogate),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A content-addressed message key together with the tier that produced it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct MessageIdentity {
|
||||
/// Hex SHA-256 key identifying the message within its account.
|
||||
pub key: String,
|
||||
/// Which precedence tier produced `key`.
|
||||
pub source: IdentitySource,
|
||||
}
|
||||
|
||||
/// Server-supplied identity hints for one message, in precedence order.
|
||||
///
|
||||
/// Borrowed rather than owned because every caller already holds these as
|
||||
/// fields on a parsed response and only needs them for the duration of the
|
||||
/// key derivation.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct RemoteIdentity<'a> {
|
||||
/// RFC 8474 `EMAILID` from a `FETCH ... (EMAILID)`, if the server has OBJECTID.
|
||||
pub email_id: Option<&'a str>,
|
||||
/// Gmail `X-GM-MSGID`, if the server advertises `X-GM-EXT-1`.
|
||||
pub gm_msgid: Option<&'a str>,
|
||||
/// Normalized `Message-ID` (see [`Email::normalize_message_id`]).
|
||||
pub message_id: Option<&'a str>,
|
||||
/// Hex SHA-256 over the full RFC822 octets (see [`Email::content_hash_of`]).
|
||||
pub content_hash: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// One occupancy of a message in one mailbox.
|
||||
///
|
||||
/// A message can sit in several folders at once (an IMAP `COPY`, a Gmail label,
|
||||
/// a message that is both in a thread's folder and in `\All`). The addressing
|
||||
/// coordinates and the read flag are properties of the *occupancy*, not of the
|
||||
/// message: IMAP `\Seen` is per-mailbox, and a UID is only meaningful inside one
|
||||
/// `uid_validity` scope of one folder.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Placement {
|
||||
/// Account owning the mailbox.
|
||||
pub account_id: String,
|
||||
/// Mailbox folder identifier (e.g. "inbox").
|
||||
pub folder_id: String,
|
||||
/// Protocol-native message id used to address the message on its server:
|
||||
/// the IMAP UID as a decimal string, or the JMAP Email object id. This is
|
||||
/// what a remote mutation must use to identify the message on the wire --
|
||||
/// never the opaque [`Email::id`].
|
||||
pub remote_id: String,
|
||||
/// The addressing scope the `remote_id` is only meaningful within: the IMAP
|
||||
/// folder UIDVALIDITY as a decimal string. Protocols without a UIDVALIDITY
|
||||
/// (JMAP) carry a stable sentinel instead, so the surrogate id stays
|
||||
/// deterministic.
|
||||
/// The IMAP UIDVALIDITY the `remote_id` was captured under, as a decimal
|
||||
/// string; a stable sentinel for protocols without one (JMAP). In the
|
||||
/// primary key because UIDs restart at 1 after a bump and would otherwise
|
||||
/// collide with rows written before it.
|
||||
pub uid_validity: String,
|
||||
/// Protocol-native id addressing the message in this mailbox: the IMAP UID
|
||||
/// as a decimal string, or the JMAP Email object id.
|
||||
pub remote_id: String,
|
||||
/// Read/unread state **in this mailbox**.
|
||||
pub read: bool,
|
||||
}
|
||||
|
||||
/// The four coordinates that address one mailbox occupancy.
|
||||
///
|
||||
/// The addressing half of a [`Placement`], without the read flag: enough to
|
||||
/// name an occupancy, not enough to describe it. A protocol backend reports
|
||||
/// these when a folder stops mentioning a message, and the store deletes
|
||||
/// exactly the rows they name -- which is why the type lives here in `core`
|
||||
/// rather than in either crate. What a folder stops reporting is an occupancy,
|
||||
/// never a message; a message goes only when its last placement does.
|
||||
///
|
||||
/// `Ord` is derived because the sync path sorts and dedups batches of these
|
||||
/// before deleting; `Hash`/`Eq` because it is also the key of the
|
||||
/// already-present set used to classify a fetched chunk.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
pub struct PlacementKey {
|
||||
/// Account owning the mailbox.
|
||||
pub account_id: String,
|
||||
/// Mailbox folder identifier.
|
||||
pub folder_id: String,
|
||||
/// The UIDVALIDITY generation `remote_id` was captured under.
|
||||
pub uid_validity: String,
|
||||
/// Protocol-native id addressing the message in this mailbox.
|
||||
pub remote_id: String,
|
||||
}
|
||||
|
||||
impl Placement {
|
||||
/// The addressing coordinates of this occupancy, dropping the read flag.
|
||||
pub fn key(&self) -> PlacementKey {
|
||||
PlacementKey {
|
||||
account_id: self.account_id.clone(),
|
||||
folder_id: self.folder_id.clone(),
|
||||
uid_validity: self.uid_validity.clone(),
|
||||
remote_id: self.remote_id.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Email message domain entity owned by Nuncio core.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Email {
|
||||
/// Opaque, account-scoped identity for the message itself.
|
||||
///
|
||||
/// Derived by [`Email::derive_message_key`], so it is independent of which
|
||||
/// folder the message currently occupies: moving a message between folders
|
||||
/// does not change it. Where the message actually sits, and its read state
|
||||
/// there, live in [`Placement`] rows instead. Callers must treat this as an
|
||||
/// opaque token -- it carries no decodable structure.
|
||||
pub id: String,
|
||||
/// Account identifier owning the message.
|
||||
pub account_id: String,
|
||||
/// Subject line.
|
||||
pub subject: String,
|
||||
/// Sender address (e.g. "alice@nuncio.mx").
|
||||
@@ -96,14 +216,32 @@ pub struct Email {
|
||||
pub recipient: String,
|
||||
/// Unix timestamp of message arrival.
|
||||
pub received_at: i64,
|
||||
/// Read/unread flag status.
|
||||
pub read: bool,
|
||||
/// Plaintext message body.
|
||||
pub body_plain: Option<String>,
|
||||
/// HTML message body.
|
||||
pub body_html: Option<String>,
|
||||
/// List of attached files.
|
||||
pub attachments: Vec<Attachment>,
|
||||
/// The message's RFC 5322 `Message-ID`, normalized by
|
||||
/// [`Email::normalize_message_id`], or `None` when the message carried
|
||||
/// none (the header is `SHOULD`, not `MUST`) or the transport did not
|
||||
/// surface it.
|
||||
///
|
||||
/// Captured but not yet used for identity. It is a *hint* toward a
|
||||
/// folder-independent identity, never an identity on its own: the value is
|
||||
/// public (it is echoed in the `References` of every reply) and forgeable,
|
||||
/// so keying storage on it alone would let a crafted message address, and
|
||||
/// overwrite, one the user already trusts.
|
||||
pub message_id: Option<String>,
|
||||
/// Hex SHA-256 over the full RFC822 octets the message was parsed from, or
|
||||
/// `None` when only headers or an envelope were available.
|
||||
///
|
||||
/// Pairs with [`Email::message_id`]: the same `Message-ID` carrying
|
||||
/// different content is two messages, not one. That case is ordinary --
|
||||
/// a mailing-list copy and a direct copy of the same mail share a
|
||||
/// `Message-ID` while differing in headers and footer, and a draft keeps
|
||||
/// one `Message-ID` across every save.
|
||||
pub content_hash: Option<String>,
|
||||
}
|
||||
|
||||
impl Email {
|
||||
@@ -131,6 +269,139 @@ impl Email {
|
||||
}
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Canonicalize a raw `Message-ID` header value.
|
||||
///
|
||||
/// Every engine syncing an account must derive byte-identical values from
|
||||
/// the same header, or two caches that agree in substance will disagree in
|
||||
/// bytes and no comparison between them can be trusted. The rules are
|
||||
/// therefore fixed and deliberately small:
|
||||
///
|
||||
/// 1. Remove **all** ASCII whitespace. A `msg-id` cannot legally contain
|
||||
/// any, so this unfolds a wrapped header without needing to know how the
|
||||
/// transport folded it -- and transports differ.
|
||||
/// 2. Strip one surrounding pair of angle brackets, which servers include
|
||||
/// or omit inconsistently between `ENVELOPE` and a raw header.
|
||||
/// 3. Preserve case. Only the domain half is case-insensitive per RFC 5322
|
||||
/// §3.6.4; lowercasing the whole value would merge ids that the
|
||||
/// originating server considers distinct, and the same server returns
|
||||
/// the same bytes to every engine anyway.
|
||||
///
|
||||
/// Returns `None` for an absent or empty value so a missing header stays
|
||||
/// distinguishable from a present-but-empty one.
|
||||
pub fn normalize_message_id(raw: &str) -> Option<String> {
|
||||
let stripped: String = raw.chars().filter(|c| !c.is_ascii_whitespace()).collect();
|
||||
let stripped = stripped
|
||||
.strip_prefix('<')
|
||||
.and_then(|s| s.strip_suffix('>'))
|
||||
.unwrap_or(&stripped);
|
||||
if stripped.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(stripped.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Hex SHA-256 over a message's full RFC822 octets, for the
|
||||
/// `content_hash` field.
|
||||
///
|
||||
/// Deterministic for identical bytes, which is what makes it usable as the
|
||||
/// tiebreaker against a duplicated `Message-ID`. It is *not* stable across
|
||||
/// servers that re-serialize a message, so it distinguishes content within
|
||||
/// an account rather than identifying a message globally.
|
||||
pub fn content_hash_of(raw_bytes: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(raw_bytes);
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Domain-separated, length-prefixed digest over an identity tier's inputs.
|
||||
///
|
||||
/// The tier label participates in the hash so that a value appearing in two
|
||||
/// different tiers (a server whose `EMAILID` happens to equal another's
|
||||
/// `X-GM-MSGID`) cannot produce one key. Length prefixes stop two distinct
|
||||
/// field tuples serializing to the same byte stream, exactly as in
|
||||
/// [`Email::surrogate_id`].
|
||||
fn identity_digest(tier: &str, account_id: &str, parts: &[&str]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
for field in [tier, account_id].into_iter().chain(parts.iter().copied()) {
|
||||
hasher.update((field.len() as u64).to_le_bytes());
|
||||
hasher.update(field.as_bytes());
|
||||
}
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Derive the account-scoped message key by identity precedence.
|
||||
///
|
||||
/// `EMAILID` → `X-GM-MSGID` → (`Message-ID` + content hash) → surrogate.
|
||||
/// Every tier is account-scoped: identity is only ever claimed within the
|
||||
/// account that fetched it, so one account's server cannot mint a key that
|
||||
/// addresses another account's mail.
|
||||
///
|
||||
/// The third tier requires **both** a `Message-ID` and a content hash. A
|
||||
/// `Message-ID` is public -- it is echoed in the `References` of every reply
|
||||
/// -- and forgeable, so keying on it alone would let a crafted message
|
||||
/// address, and with an upsert overwrite, a message the user already trusts,
|
||||
/// and would evade filtering by never being classified as new. Pairing it
|
||||
/// with the content hash also gets the ordinary cases right: a mailing-list
|
||||
/// copy and a direct copy share a `Message-ID` but differ in content, and so
|
||||
/// are correctly two messages.
|
||||
///
|
||||
/// The surrogate tier is folder-scoped and therefore *not* stable across a
|
||||
/// move; it is the honest answer when the server offered nothing better.
|
||||
///
|
||||
/// # Keys do not yet converge across engine types
|
||||
///
|
||||
/// Precedence makes a key reproducible from the same inputs, not from the
|
||||
/// same *message*. The IMAP engine does not currently request `EMAILID` or
|
||||
/// `X-GM-MSGID`, so it can only ever reach the `Message-ID`+content tier or
|
||||
/// the surrogate, while JMAP supplies a server id and enters at the top
|
||||
/// tier. Two engines syncing one account therefore produce disjoint key
|
||||
/// spaces: the same message is two identities, one per engine. This is a
|
||||
/// known gap in the convergent multi-engine model, not a property to rely
|
||||
/// on -- do not assume a key derived by one engine addresses the row
|
||||
/// another engine wrote.
|
||||
pub fn derive_message_key(
|
||||
account_id: &str,
|
||||
remote: RemoteIdentity<'_>,
|
||||
folder_id: &str,
|
||||
uid_validity: &str,
|
||||
remote_id: &str,
|
||||
) -> MessageIdentity {
|
||||
if let Some(email_id) = remote.email_id.filter(|v| !v.is_empty()) {
|
||||
return MessageIdentity {
|
||||
key: Self::identity_digest("emailid", account_id, &[email_id]),
|
||||
source: IdentitySource::EmailId,
|
||||
};
|
||||
}
|
||||
if let Some(gm_msgid) = remote.gm_msgid.filter(|v| !v.is_empty()) {
|
||||
return MessageIdentity {
|
||||
key: Self::identity_digest("gmsgid", account_id, &[gm_msgid]),
|
||||
source: IdentitySource::GmailMsgId,
|
||||
};
|
||||
}
|
||||
if let (Some(message_id), Some(content_hash)) = (
|
||||
remote.message_id.filter(|v| !v.is_empty()),
|
||||
remote.content_hash.filter(|v| !v.is_empty()),
|
||||
) {
|
||||
return MessageIdentity {
|
||||
key: Self::identity_digest(
|
||||
"msgid_content",
|
||||
account_id,
|
||||
&[message_id, content_hash],
|
||||
),
|
||||
source: IdentitySource::MessageIdContent,
|
||||
};
|
||||
}
|
||||
MessageIdentity {
|
||||
key: Self::identity_digest(
|
||||
"surrogate",
|
||||
account_id,
|
||||
&[folder_id, uid_validity, remote_id],
|
||||
),
|
||||
source: IdentitySource::Surrogate,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mailbox folder entity.
|
||||
@@ -247,4 +518,330 @@ mod tests {
|
||||
Email::surrogate_id("ab", "c", "1", "1"),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_message_id_unfolds_and_unwraps_to_one_canonical_form() {
|
||||
// Every spelling a transport might hand us for the same id must
|
||||
// collapse to identical bytes, or two engines holding the same message
|
||||
// cannot be shown to agree.
|
||||
let canonical = "abc123@mail.example.com";
|
||||
for raw in [
|
||||
"<abc123@mail.example.com>",
|
||||
"abc123@mail.example.com",
|
||||
" <abc123@mail.example.com> ",
|
||||
"<abc123@mail.\r\n example.com>",
|
||||
"<abc123@mail.\t example.com>",
|
||||
] {
|
||||
assert_eq!(
|
||||
Email::normalize_message_id(raw).as_deref(),
|
||||
Some(canonical),
|
||||
"input {raw:?} did not canonicalize"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_message_id_preserves_case_and_rejects_empty() {
|
||||
// Only the domain half is case-insensitive per RFC 5322 3.6.4;
|
||||
// lowercasing wholesale would merge ids the origin considers distinct.
|
||||
assert_eq!(
|
||||
Email::normalize_message_id("<AbC@Example.COM>").as_deref(),
|
||||
Some("AbC@Example.COM")
|
||||
);
|
||||
assert_eq!(Email::normalize_message_id(""), None);
|
||||
assert_eq!(Email::normalize_message_id("<>"), None);
|
||||
assert_eq!(Email::normalize_message_id(" "), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_hash_distinguishes_identical_message_ids_with_different_bodies() {
|
||||
// The case the hash exists for: a list copy and a direct copy share a
|
||||
// Message-ID but are not the same message.
|
||||
let direct = b"Message-ID: <shared@example.com>\r\n\r\nbody";
|
||||
let via_list = b"Message-ID: <shared@example.com>\r\nList-Id: l\r\n\r\nbody\n--\nfooter";
|
||||
assert_ne!(
|
||||
Email::content_hash_of(direct),
|
||||
Email::content_hash_of(via_list)
|
||||
);
|
||||
assert_eq!(
|
||||
Email::content_hash_of(direct),
|
||||
Email::content_hash_of(direct),
|
||||
"the hash must be deterministic for identical octets"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_key_precedence_prefers_emailid_over_everything_below_it() {
|
||||
let remote = RemoteIdentity {
|
||||
email_id: Some("M00000001"),
|
||||
gm_msgid: Some("1234567890"),
|
||||
message_id: Some("a@b.example"),
|
||||
content_hash: Some("deadbeef"),
|
||||
};
|
||||
let id = Email::derive_message_key("acct-1", remote, "INBOX", "42", "5");
|
||||
assert_eq!(id.source, IdentitySource::EmailId);
|
||||
|
||||
// Same EMAILID in a different folder is the SAME message.
|
||||
let elsewhere = Email::derive_message_key(
|
||||
"acct-1",
|
||||
RemoteIdentity {
|
||||
email_id: Some("M00000001"),
|
||||
gm_msgid: None,
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
"Archive",
|
||||
"99",
|
||||
"7",
|
||||
);
|
||||
assert_eq!(id.key, elsewhere.key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn message_key_is_account_scoped_at_every_tier() {
|
||||
let emailid = || RemoteIdentity {
|
||||
email_id: Some("M00000001"),
|
||||
gm_msgid: None,
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
let a = Email::derive_message_key("acct-1", emailid(), "INBOX", "42", "5");
|
||||
let b = Email::derive_message_key("acct-2", emailid(), "INBOX", "42", "5");
|
||||
assert_ne!(
|
||||
a.key, b.key,
|
||||
"the same EMAILID in two accounts must not collide"
|
||||
);
|
||||
|
||||
let gmsgid = || RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: Some("1234567890"),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
let a = Email::derive_message_key("acct-1", gmsgid(), "INBOX", "42", "5");
|
||||
let b = Email::derive_message_key("acct-2", gmsgid(), "INBOX", "42", "5");
|
||||
assert_ne!(
|
||||
a.key, b.key,
|
||||
"the same X-GM-MSGID in two accounts must not collide"
|
||||
);
|
||||
|
||||
let msgid_content = || RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: Some("a@b.example"),
|
||||
content_hash: Some("deadbeef"),
|
||||
};
|
||||
let a = Email::derive_message_key("acct-1", msgid_content(), "INBOX", "42", "5");
|
||||
let b = Email::derive_message_key("acct-2", msgid_content(), "INBOX", "42", "5");
|
||||
assert_ne!(
|
||||
a.key, b.key,
|
||||
"the same Message-ID and content hash in two accounts must not collide"
|
||||
);
|
||||
|
||||
let surrogate = RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
let a = Email::derive_message_key("acct-1", surrogate, "INBOX", "42", "5");
|
||||
let b = Email::derive_message_key("acct-2", surrogate, "INBOX", "42", "5");
|
||||
assert_ne!(
|
||||
a.key, b.key,
|
||||
"the same folder/uidvalidity/remote-id in two accounts must not collide"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_message_id_without_a_content_hash_never_reaches_the_content_tier() {
|
||||
// The non-negotiable: a Message-ID is public and forgeable, so it must
|
||||
// never key storage on its own. Missing content hash falls through to the
|
||||
// folder-scoped surrogate rather than trusting the header alone.
|
||||
let id = Email::derive_message_key(
|
||||
"acct-1",
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: Some("a@b.example"),
|
||||
content_hash: None,
|
||||
},
|
||||
"INBOX",
|
||||
"42",
|
||||
"5",
|
||||
);
|
||||
assert_eq!(id.source, IdentitySource::Surrogate);
|
||||
|
||||
let surrogate_only = Email::derive_message_key(
|
||||
"acct-1",
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
"INBOX",
|
||||
"42",
|
||||
"5",
|
||||
);
|
||||
assert_eq!(id.key, surrogate_only.key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_message_id_with_different_content_is_two_messages() {
|
||||
let mk = |hash: &str| {
|
||||
Email::derive_message_key(
|
||||
"acct-1",
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: Some("a@b.example"),
|
||||
content_hash: Some(hash),
|
||||
},
|
||||
"INBOX",
|
||||
"42",
|
||||
"5",
|
||||
)
|
||||
};
|
||||
let list_copy = mk("1111");
|
||||
let direct_copy = mk("2222");
|
||||
assert_eq!(list_copy.source, IdentitySource::MessageIdContent);
|
||||
assert_ne!(list_copy.key, direct_copy.key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tiers_are_domain_separated_so_a_shared_value_cannot_collide_across_them() {
|
||||
let as_emailid = Email::derive_message_key(
|
||||
"acct-1",
|
||||
RemoteIdentity {
|
||||
email_id: Some("X"),
|
||||
gm_msgid: None,
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
"INBOX",
|
||||
"42",
|
||||
"5",
|
||||
);
|
||||
let as_gmsgid = Email::derive_message_key(
|
||||
"acct-1",
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: Some("X"),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
"INBOX",
|
||||
"42",
|
||||
"5",
|
||||
);
|
||||
assert_ne!(as_emailid.key, as_gmsgid.key);
|
||||
}
|
||||
|
||||
fn sample_placement() -> Placement {
|
||||
Placement {
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "INBOX".to_string(),
|
||||
uid_validity: "42".to_string(),
|
||||
remote_id: "5".to_string(),
|
||||
read: true,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placement_key_carries_every_addressing_coordinate_and_drops_the_read_flag() {
|
||||
// The key must reproduce all four coordinates exactly. A key that
|
||||
// dropped or reordered one would address a different occupancy, and a
|
||||
// sync diffing reported-against-stored would delete live mail.
|
||||
let placement = sample_placement();
|
||||
let key = placement.key();
|
||||
assert_eq!(key.account_id, "acct-1");
|
||||
assert_eq!(key.folder_id, "INBOX");
|
||||
assert_eq!(key.uid_validity, "42");
|
||||
assert_eq!(key.remote_id, "5");
|
||||
|
||||
// The read flag is the one field a key must NOT carry: it describes an
|
||||
// occupancy rather than naming it, so including it would make the same
|
||||
// occupancy compare unequal to itself after the user opened the message.
|
||||
let mut unread = placement.clone();
|
||||
unread.read = false;
|
||||
assert_ne!(placement, unread);
|
||||
assert_eq!(placement.key(), unread.key());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placement_keys_distinguish_every_coordinate_that_can_differ() {
|
||||
let base = sample_placement().key();
|
||||
for mutated in [
|
||||
PlacementKey {
|
||||
account_id: "acct-2".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
PlacementKey {
|
||||
folder_id: "Archive".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
PlacementKey {
|
||||
uid_validity: "43".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
PlacementKey {
|
||||
remote_id: "6".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
] {
|
||||
assert_ne!(
|
||||
base, mutated,
|
||||
"changing one coordinate must yield a different occupancy"
|
||||
);
|
||||
}
|
||||
|
||||
// Usable as a set key and sortable, which is what the sync path needs
|
||||
// to diff a reported UID set against what the store holds.
|
||||
let mut set = std::collections::HashSet::new();
|
||||
assert!(set.insert(base.clone()));
|
||||
assert!(!set.insert(base.clone()), "an equal key must not re-insert");
|
||||
let mut keys = [
|
||||
PlacementKey {
|
||||
remote_id: "6".to_string(),
|
||||
..base.clone()
|
||||
},
|
||||
base.clone(),
|
||||
];
|
||||
keys.sort();
|
||||
assert_eq!(keys[0], base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placement_key_round_trips_through_serde() {
|
||||
let key = sample_placement().key();
|
||||
let json = serde_json::to_string(&key).unwrap();
|
||||
let parsed: PlacementKey = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed, key);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn placement_round_trips_through_serde_including_its_read_flag() {
|
||||
let placement = sample_placement();
|
||||
let json = serde_json::to_string(&placement).unwrap();
|
||||
let parsed: Placement = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(parsed, placement);
|
||||
assert!(parsed.read);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identity_source_round_trips_through_its_stored_string() {
|
||||
for source in [
|
||||
IdentitySource::EmailId,
|
||||
IdentitySource::GmailMsgId,
|
||||
IdentitySource::MessageIdContent,
|
||||
IdentitySource::Surrogate,
|
||||
] {
|
||||
assert_eq!(IdentitySource::from_token(source.as_str()), Some(source));
|
||||
}
|
||||
assert_eq!(
|
||||
IdentitySource::from_token("not_a_tier"),
|
||||
None,
|
||||
"an unrecognised token must fail loudly, never default to a tier"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -6,13 +6,27 @@ use crate::ast::{
|
||||
};
|
||||
use arc_swap::ArcSwap;
|
||||
use hmac::{Hmac, Mac};
|
||||
use nuncio_core::model::Email;
|
||||
use nuncio_core::model::{Email, Placement};
|
||||
use regex::Regex;
|
||||
use sha2::Sha256;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// A message together with the one mailbox occupancy being evaluated.
|
||||
///
|
||||
/// `WHERE FOLDER = 'INBOX'` has no answer for a message that sits in three
|
||||
/// folders at once, so evaluation is defined against a single placement and the
|
||||
/// caller fans out over however many the message has. Borrowing both halves
|
||||
/// keeps that fan-out free of clones.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct PlacedEmail<'a> {
|
||||
/// Identity and content -- the same for every placement.
|
||||
pub email: &'a Email,
|
||||
/// The occupancy this evaluation is about.
|
||||
pub placement: &'a Placement,
|
||||
}
|
||||
|
||||
/// Pre-compiled single filter rule optimizing regex and predicate execution.
|
||||
#[derive(Clone)]
|
||||
pub struct CompiledFilter {
|
||||
@@ -55,31 +69,40 @@ impl CompiledFilter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Evaluate email message against compiled rule condition tree.
|
||||
/// Evaluate one placement against the compiled rule condition tree.
|
||||
///
|
||||
/// This checks only the WHERE-clause conditions; it does NOT apply the
|
||||
/// rule's account scope. Callers must gate on
|
||||
/// [`FilterRule::matches_account`] first (as every `FilterEngine` entry
|
||||
/// point does), or a rule scoped to one account will fire on messages of
|
||||
/// another.
|
||||
pub fn evaluate_condition(&self, email: &Email) -> bool {
|
||||
Self::eval_node(&self.rule.conditions, email, &self.compiled_regexes)
|
||||
pub fn evaluate_condition(&self, placed: PlacedEmail<'_>) -> bool {
|
||||
Self::eval_node(&self.rule.conditions, placed, &self.compiled_regexes)
|
||||
}
|
||||
|
||||
fn eval_node(node: &ConditionNode, email: &Email, regexes: &[(String, Regex)]) -> bool {
|
||||
fn eval_node(
|
||||
node: &ConditionNode,
|
||||
placed: PlacedEmail<'_>,
|
||||
regexes: &[(String, Regex)],
|
||||
) -> bool {
|
||||
match node {
|
||||
ConditionNode::Leaf(leaf) => Self::eval_leaf(leaf, email, regexes),
|
||||
ConditionNode::Leaf(leaf) => Self::eval_leaf(leaf, placed, regexes),
|
||||
ConditionNode::And(children) => {
|
||||
children.iter().all(|c| Self::eval_node(c, email, regexes))
|
||||
children.iter().all(|c| Self::eval_node(c, placed, regexes))
|
||||
}
|
||||
ConditionNode::Or(children) => {
|
||||
children.iter().any(|c| Self::eval_node(c, email, regexes))
|
||||
children.iter().any(|c| Self::eval_node(c, placed, regexes))
|
||||
}
|
||||
ConditionNode::Not(inner) => !Self::eval_node(inner, email, regexes),
|
||||
ConditionNode::Not(inner) => !Self::eval_node(inner, placed, regexes),
|
||||
}
|
||||
}
|
||||
|
||||
fn eval_leaf(leaf: &ConditionLeaf, email: &Email, regexes: &[(String, Regex)]) -> bool {
|
||||
fn eval_leaf(
|
||||
leaf: &ConditionLeaf,
|
||||
placed: PlacedEmail<'_>,
|
||||
regexes: &[(String, Regex)],
|
||||
) -> bool {
|
||||
let email = placed.email;
|
||||
match &leaf.field {
|
||||
FilterField::Subject => {
|
||||
Self::eval_string_op(&email.subject, &leaf.operator, &leaf.value, regexes)
|
||||
@@ -98,12 +121,18 @@ impl CompiledFilter {
|
||||
.unwrap_or("");
|
||||
Self::eval_string_op(body, &leaf.operator, &leaf.value, regexes)
|
||||
}
|
||||
FilterField::Folder => {
|
||||
Self::eval_string_op(&email.folder_id, &leaf.operator, &leaf.value, regexes)
|
||||
}
|
||||
FilterField::Account => {
|
||||
Self::eval_string_op(&email.account_id, &leaf.operator, &leaf.value, regexes)
|
||||
}
|
||||
FilterField::Folder => Self::eval_string_op(
|
||||
&placed.placement.folder_id,
|
||||
&leaf.operator,
|
||||
&leaf.value,
|
||||
regexes,
|
||||
),
|
||||
FilterField::Account => Self::eval_string_op(
|
||||
&placed.placement.account_id,
|
||||
&leaf.operator,
|
||||
&leaf.value,
|
||||
regexes,
|
||||
),
|
||||
FilterField::HasAttachment => {
|
||||
let has = !email.attachments.is_empty();
|
||||
if let FilterValue::Boolean(b) = leaf.value {
|
||||
@@ -277,18 +306,26 @@ impl FilterEngine {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Evaluate email message returning matching rule actions.
|
||||
/// Evaluate one placement of a message, returning the matching rules' actions.
|
||||
///
|
||||
/// Defined per placement, not per message: `FOLDER` and `ACCOUNT` are
|
||||
/// properties of where the message sits, and a message can sit in several
|
||||
/// places at once. Callers holding a multi-placement message evaluate each
|
||||
/// placement and are responsible for not acting twice on one message -- see
|
||||
/// the fire-once claim in the store.
|
||||
///
|
||||
/// A rule only ever fires for the account it was created against
|
||||
/// (`FilterRule::target_account`, set via `ON ACCOUNT` or `*` for all
|
||||
/// accounts) — condition matching alone is not account-scoped, so this
|
||||
/// check must happen before `evaluate_condition` runs.
|
||||
pub fn evaluate(&self, email: &Email) -> Vec<(FilterRule, Vec<RuleAction>)> {
|
||||
pub fn evaluate(&self, placed: PlacedEmail<'_>) -> Vec<(FilterRule, Vec<RuleAction>)> {
|
||||
let guard = self.cache.load();
|
||||
let mut results = Vec::new();
|
||||
|
||||
for filter in &guard.filters {
|
||||
if filter.rule.matches_account(&email.account_id) && filter.evaluate_condition(email) {
|
||||
if filter.rule.matches_account(&placed.placement.account_id)
|
||||
&& filter.evaluate_condition(placed)
|
||||
{
|
||||
debug!(
|
||||
rule_id = %filter.rule.id,
|
||||
rule_name = %filter.rule.name,
|
||||
@@ -315,21 +352,25 @@ impl FilterEngine {
|
||||
/// the remaining rules rather than aborting the whole batch.
|
||||
pub async fn evaluate_with_timeout(
|
||||
&self,
|
||||
email: &Email,
|
||||
placed: PlacedEmail<'_>,
|
||||
timeout_duration: Duration,
|
||||
) -> Vec<(FilterRule, Vec<RuleAction>)> {
|
||||
let guard = self.cache.load();
|
||||
let mut results = Vec::new();
|
||||
|
||||
for filter in &guard.filters {
|
||||
if !filter.rule.matches_account(&email.account_id) {
|
||||
if !filter.rule.matches_account(&placed.placement.account_id) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let filter_for_task = filter.clone();
|
||||
let email_for_task = email.clone();
|
||||
let email_for_task = placed.email.clone();
|
||||
let placement_for_task = placed.placement.clone();
|
||||
let eval_task = tokio::task::spawn_blocking(move || {
|
||||
filter_for_task.evaluate_condition(&email_for_task)
|
||||
filter_for_task.evaluate_condition(PlacedEmail {
|
||||
email: &email_for_task,
|
||||
placement: &placement_for_task,
|
||||
})
|
||||
});
|
||||
|
||||
match tokio::time::timeout(timeout_duration, eval_task).await {
|
||||
@@ -370,7 +411,7 @@ impl FilterEngine {
|
||||
/// Rules whose account scope does not match the message are reported as
|
||||
/// `SKIPPED (account scope mismatch)` rather than omitted, so the trace
|
||||
/// distinguishes "condition did not match" from "rule does not apply here".
|
||||
pub fn preview(&self, email: &Email) -> FilterPreviewResult {
|
||||
pub fn preview(&self, placed: PlacedEmail<'_>) -> FilterPreviewResult {
|
||||
let start = Instant::now();
|
||||
let guard = self.cache.load();
|
||||
let mut matched_rule_id = None;
|
||||
@@ -380,7 +421,7 @@ impl FilterEngine {
|
||||
let mut matched = false;
|
||||
|
||||
for filter in &guard.filters {
|
||||
if !filter.rule.matches_account(&email.account_id) {
|
||||
if !filter.rule.matches_account(&placed.placement.account_id) {
|
||||
traces.push(format!(
|
||||
"Rule '{}' (priority {}): SKIPPED (account scope mismatch)",
|
||||
filter.rule.name, filter.rule.priority
|
||||
@@ -388,7 +429,7 @@ impl FilterEngine {
|
||||
continue;
|
||||
}
|
||||
|
||||
let is_match = filter.evaluate_condition(email);
|
||||
let is_match = filter.evaluate_condition(placed);
|
||||
traces.push(format!(
|
||||
"Rule '{}' (priority {}): {}",
|
||||
filter.rule.name,
|
||||
@@ -406,7 +447,7 @@ impl FilterEngine {
|
||||
let elapsed = start.elapsed().as_micros() as u64;
|
||||
|
||||
FilterPreviewResult {
|
||||
message_id: email.id.clone(),
|
||||
message_id: placed.email.id.clone(),
|
||||
matched,
|
||||
matched_rule_id,
|
||||
matched_rule_name,
|
||||
@@ -435,22 +476,31 @@ mod tests {
|
||||
use super::*;
|
||||
use nuncio_core::model::Email;
|
||||
|
||||
fn test_email(account_id: &str, subject: &str, folder_id: &str) -> Email {
|
||||
Email {
|
||||
/// Build an `Email`/`Placement` pair that shares one account between the
|
||||
/// message and the mailbox it is placed in, matching how a message and
|
||||
/// its own placement always agree on account.
|
||||
fn test_placed_email(account_id: &str, subject: &str, folder_id: &str) -> (Email, Placement) {
|
||||
let email = Email {
|
||||
id: "msg-1".to_string(),
|
||||
account_id: account_id.to_string(),
|
||||
folder_id: folder_id.to_string(),
|
||||
remote_id: "1".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: subject.to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1700000000,
|
||||
read: false,
|
||||
body_plain: Some("Hello".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
let placement = Placement {
|
||||
account_id: account_id.to_string(),
|
||||
folder_id: folder_id.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: "1".to_string(),
|
||||
read: false,
|
||||
};
|
||||
(email, placement)
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -459,27 +509,19 @@ mod tests {
|
||||
let rule = crate::parser::NsqlParser::parse_rule("Urgent", 1, nsql).unwrap();
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
let email = Email {
|
||||
id: "msg-1".to_string(),
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
remote_id: "1".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: "Urgent Meeting".to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1700000000,
|
||||
read: false,
|
||||
body_plain: Some("Hello".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
let (email, placement) = test_placed_email("acct-1", "Urgent Meeting", "inbox");
|
||||
|
||||
let results = engine.evaluate(&email);
|
||||
let results = engine.evaluate(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &placement,
|
||||
});
|
||||
assert_eq!(results.len(), 1);
|
||||
assert_eq!(results[0].1.len(), 2);
|
||||
|
||||
let preview = engine.preview(&email);
|
||||
let preview = engine.preview(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &placement,
|
||||
});
|
||||
assert!(preview.matched);
|
||||
assert_eq!(preview.matched_rule_name, Some("Urgent".to_string()));
|
||||
}
|
||||
@@ -499,14 +541,27 @@ mod tests {
|
||||
|
||||
// Condition matches the subject, but the message belongs to a
|
||||
// different account than the rule was created for.
|
||||
let email_b = test_email("acct-b", "Weekly Report", "inbox");
|
||||
let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox");
|
||||
assert!(
|
||||
engine.evaluate(&email_b).is_empty(),
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email_b,
|
||||
placement: &placement_b
|
||||
})
|
||||
.is_empty(),
|
||||
"rule scoped to acct-a must not match a message from acct-b"
|
||||
);
|
||||
|
||||
let email_a = test_email("acct-a", "Weekly Report", "inbox");
|
||||
assert_eq!(engine.evaluate(&email_a).len(), 1);
|
||||
let (email_a, placement_a) = test_placed_email("acct-a", "Weekly Report", "inbox");
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email_a,
|
||||
placement: &placement_a
|
||||
})
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -515,15 +570,27 @@ mod tests {
|
||||
let rule = crate::parser::NsqlParser::parse_rule("Scoped To A", 1, nsql).unwrap();
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
let email_b = test_email("acct-b", "Weekly Report", "inbox");
|
||||
let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox");
|
||||
let results = engine
|
||||
.evaluate_with_timeout(&email_b, Duration::from_secs(1))
|
||||
.evaluate_with_timeout(
|
||||
PlacedEmail {
|
||||
email: &email_b,
|
||||
placement: &placement_b,
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
assert!(results.is_empty());
|
||||
|
||||
let email_a = test_email("acct-a", "Weekly Report", "inbox");
|
||||
let (email_a, placement_a) = test_placed_email("acct-a", "Weekly Report", "inbox");
|
||||
let results = engine
|
||||
.evaluate_with_timeout(&email_a, Duration::from_secs(1))
|
||||
.evaluate_with_timeout(
|
||||
PlacedEmail {
|
||||
email: &email_a,
|
||||
placement: &placement_a,
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(results.len(), 1);
|
||||
}
|
||||
@@ -534,8 +601,11 @@ mod tests {
|
||||
let rule = crate::parser::NsqlParser::parse_rule("Scoped To A", 1, nsql).unwrap();
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
let email_b = test_email("acct-b", "Weekly Report", "inbox");
|
||||
let preview = engine.preview(&email_b);
|
||||
let (email_b, placement_b) = test_placed_email("acct-b", "Weekly Report", "inbox");
|
||||
let preview = engine.preview(PlacedEmail {
|
||||
email: &email_b,
|
||||
placement: &placement_b,
|
||||
});
|
||||
assert!(!preview.matched);
|
||||
assert!(preview
|
||||
.condition_traces
|
||||
@@ -549,8 +619,16 @@ mod tests {
|
||||
let rule = crate::parser::NsqlParser::parse_rule("Global", 1, nsql).unwrap();
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
let email = test_email("any-account", "Weekly Report", "inbox");
|
||||
assert_eq!(engine.evaluate(&email).len(), 1);
|
||||
let (email, placement) = test_placed_email("any-account", "Weekly Report", "inbox");
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &placement
|
||||
})
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -560,20 +638,30 @@ mod tests {
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
// Same folder_id, different account_id: a rule matching on the
|
||||
// `account` condition field must key off email.account_id, never
|
||||
// fall through to comparing folder_id.
|
||||
let email_a = test_email("acct-a", "Anything", "shared-folder");
|
||||
// `account` condition field must key off the placement's account_id,
|
||||
// never fall through to comparing folder_id.
|
||||
let (email_a, placement_a) = test_placed_email("acct-a", "Anything", "shared-folder");
|
||||
assert_eq!(
|
||||
engine.evaluate(&email_a).len(),
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email_a,
|
||||
placement: &placement_a
|
||||
})
|
||||
.len(),
|
||||
1,
|
||||
"rule on account = 'acct-a' must match a message with account_id 'acct-a'"
|
||||
"rule on account = 'acct-a' must match a placement with account_id 'acct-a'"
|
||||
);
|
||||
|
||||
let email_b = test_email("acct-b", "Anything", "shared-folder");
|
||||
let (email_b, placement_b) = test_placed_email("acct-b", "Anything", "shared-folder");
|
||||
assert!(
|
||||
engine.evaluate(&email_b).is_empty(),
|
||||
"rule on account = 'acct-a' must not match a message with account_id 'acct-b', \
|
||||
even when it shares the same folder_id as an acct-a message"
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email_b,
|
||||
placement: &placement_b
|
||||
})
|
||||
.is_empty(),
|
||||
"rule on account = 'acct-a' must not match a placement with account_id 'acct-b', \
|
||||
even when it shares the same folder_id as an acct-a placement"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -583,11 +671,24 @@ mod tests {
|
||||
let rule = crate::parser::NsqlParser::parse_rule("Not In Spam Or Trash", 1, nsql).unwrap();
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
let inbox_email = test_email("acct-1", "Anything", "inbox");
|
||||
assert_eq!(engine.evaluate(&inbox_email).len(), 1);
|
||||
let (inbox_email, inbox_placement) = test_placed_email("acct-1", "Anything", "inbox");
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &inbox_email,
|
||||
placement: &inbox_placement
|
||||
})
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
|
||||
let spam_email = test_email("acct-1", "Anything", "spam");
|
||||
assert!(engine.evaluate(&spam_email).is_empty());
|
||||
let (spam_email, spam_placement) = test_placed_email("acct-1", "Anything", "spam");
|
||||
assert!(engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &spam_email,
|
||||
placement: &spam_placement
|
||||
})
|
||||
.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -612,8 +713,16 @@ mod tests {
|
||||
};
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
let email = test_email("acct-1", "Anything", "inbox");
|
||||
assert_eq!(engine.evaluate(&email).len(), 1);
|
||||
let (email, placement) = test_placed_email("acct-1", "Anything", "inbox");
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &placement
|
||||
})
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -640,8 +749,16 @@ mod tests {
|
||||
};
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
let email = test_email("acct-1", "Anything", "inbox");
|
||||
assert_eq!(engine.evaluate(&email).len(), 1);
|
||||
let (email, placement) = test_placed_email("acct-1", "Anything", "inbox");
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &placement
|
||||
})
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -672,11 +789,17 @@ mod tests {
|
||||
};
|
||||
let engine = FilterEngine::new(vec![rule]).unwrap();
|
||||
|
||||
let mut email = test_email("acct-1", "Anything", "inbox");
|
||||
let (mut email, placement) = test_placed_email("acct-1", "Anything", "inbox");
|
||||
email.body_plain = Some("A".repeat(60_000_000));
|
||||
|
||||
let results = engine
|
||||
.evaluate_with_timeout(&email, Duration::from_millis(1))
|
||||
.evaluate_with_timeout(
|
||||
PlacedEmail {
|
||||
email: &email,
|
||||
placement: &placement,
|
||||
},
|
||||
Duration::from_millis(1),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
@@ -693,4 +816,99 @@ mod tests {
|
||||
"expected a WARN log carrying the rule id for the timed-out evaluation, got: {captured:?}"
|
||||
);
|
||||
}
|
||||
|
||||
fn sample_email(key: &str) -> Email {
|
||||
Email {
|
||||
id: key.to_string(),
|
||||
account_id: "acct-1".to_string(),
|
||||
subject: "Anything".to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1700000000,
|
||||
body_plain: Some("Hello".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_placement(folder_id: &str, remote_id: &str) -> Placement {
|
||||
Placement {
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: folder_id.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: remote_id.to_string(),
|
||||
read: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn rule_where_folder_is(folder: &str) -> FilterRule {
|
||||
let nsql = format!("WHERE folder = '{folder}' ACTION MARK READ");
|
||||
crate::parser::NsqlParser::parse_rule("Folder Filter", 1, &nsql).unwrap()
|
||||
}
|
||||
|
||||
fn rule_where_account_is(account: &str) -> FilterRule {
|
||||
let nsql = format!("WHERE account = '{account}' ACTION MARK READ");
|
||||
crate::parser::NsqlParser::parse_rule("Account Filter", 1, &nsql).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_folder_condition_matches_only_the_placement_it_is_evaluated_against() {
|
||||
let engine = FilterEngine::new(vec![rule_where_folder_is("INBOX")]).unwrap();
|
||||
let email = sample_email("key-1");
|
||||
let inbox = sample_placement("INBOX", "5");
|
||||
let archive = sample_placement("Archive", "9");
|
||||
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &inbox
|
||||
})
|
||||
.len(),
|
||||
1,
|
||||
"the INBOX placement matches"
|
||||
);
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &archive
|
||||
})
|
||||
.len(),
|
||||
0,
|
||||
"the same message's Archive placement does not"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_account_condition_reads_the_placement_account() {
|
||||
let engine = FilterEngine::new(vec![rule_where_account_is("acct-1")]).unwrap();
|
||||
let email = sample_email("key-1");
|
||||
let mine = sample_placement("INBOX", "5");
|
||||
let theirs = Placement {
|
||||
account_id: "acct-2".into(),
|
||||
..sample_placement("INBOX", "5")
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &mine
|
||||
})
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
engine
|
||||
.evaluate(PlacedEmail {
|
||||
email: &email,
|
||||
placement: &theirs
|
||||
})
|
||||
.len(),
|
||||
0
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -92,6 +92,10 @@ impl NsqlValidator {
|
||||
"header field conditions are not yet supported: message headers are not indexed"
|
||||
.to_string(),
|
||||
)),
|
||||
// `Folder` and `Account` are validated the same as the
|
||||
// content fields here: only the value's shape (string or
|
||||
// string list) matters at this pass. At evaluation time they
|
||||
// read a placement's coordinates, not the message's content.
|
||||
FilterField::Subject
|
||||
| FilterField::From
|
||||
| FilterField::To
|
||||
|
||||
@@ -1,10 +1,28 @@
|
||||
//! Protocol-agnostic async mail backend trait definitions.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nuncio_core::model::{Attachment, Email, Folder};
|
||||
use nuncio_core::model::{Attachment, Email, Folder, IdentitySource, Placement, PlacementKey};
|
||||
|
||||
use crate::parser::MailError;
|
||||
|
||||
/// One message as a backend surfaced it: identity, content, and the mailbox
|
||||
/// occupancy it was found in.
|
||||
///
|
||||
/// Backends emit this rather than a bare [`Email`] because a fetch observes two
|
||||
/// separate facts at once: *which message this is* -- folder-independent, and
|
||||
/// the same in every mailbox that holds a copy -- and *where this pass found
|
||||
/// it*, which is per-mailbox and changes under a move. Collapsing them into one
|
||||
/// record is what made a moved message read as a new one.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PlacedMessage {
|
||||
/// Identity and content. `email.id` is the derived message key.
|
||||
pub email: Email,
|
||||
/// Which precedence tier produced `email.id`.
|
||||
pub source: IdentitySource,
|
||||
/// The occupancy this fetch found it in.
|
||||
pub placement: Placement,
|
||||
}
|
||||
|
||||
/// The remote change a [`RemoteMutationSpec`] applies to one addressed message.
|
||||
///
|
||||
/// This is the closed vocabulary of genuinely-remote filter actions that the
|
||||
@@ -72,21 +90,28 @@ pub struct RemoteMutationSpec {
|
||||
/// or deleted by any other client -- another Nuncio daemon, a phone, webmail --
|
||||
/// simply stays in the local store forever. Expressing absence is the whole
|
||||
/// point of the type.
|
||||
///
|
||||
/// Absence is reported as [`PlacementKey`]s, not message keys, because what a
|
||||
/// folder stops mentioning is an *occupancy*. The same message may still sit in
|
||||
/// another mailbox, and deleting the message on the strength of one folder's
|
||||
/// silence would destroy a copy the user still has. A message goes only when
|
||||
/// its last placement does.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct FolderChanges {
|
||||
/// Messages to store, whether newly arrived or changed.
|
||||
pub upserts: Vec<Email>,
|
||||
/// Surrogate ids the server **explicitly reported as gone**, via QRESYNC
|
||||
/// Messages to store, whether newly arrived or changed, each with the
|
||||
/// occupancy this pass found it in.
|
||||
pub upserts: Vec<PlacedMessage>,
|
||||
/// Occupancies the server **explicitly reported as gone**, via QRESYNC
|
||||
/// `VANISHED`. Always safe to delete.
|
||||
pub removals: Vec<String>,
|
||||
/// Surrogate ids of everything the folder currently holds, when this pass
|
||||
/// enumerated the complete UID set. The caller may delete anything it
|
||||
/// stores for this folder that is absent from this list.
|
||||
pub removals: Vec<PlacementKey>,
|
||||
/// Every occupancy the folder currently holds, when this pass enumerated
|
||||
/// the complete UID set. The caller may delete any placement it stores for
|
||||
/// this folder that is absent from this list.
|
||||
///
|
||||
/// `None` and `Some(vec![])` mean different things, and conflating them
|
||||
/// deletes a mailbox. `None` is "this pass was incremental and cannot
|
||||
/// speak to absence"; `Some(vec![])` is "the folder is genuinely empty".
|
||||
pub present: Option<Vec<String>>,
|
||||
pub present: Option<Vec<PlacementKey>>,
|
||||
/// The checkpoint to resume from next time.
|
||||
pub next_state: String,
|
||||
}
|
||||
@@ -158,11 +183,16 @@ pub trait MailBackend: Send + Sync {
|
||||
///
|
||||
/// Provided rather than required so no implementation can satisfy the
|
||||
/// trait by supplying this and quietly never reporting a removal.
|
||||
///
|
||||
/// Every message comes back as a [`PlacedMessage`] so the caller can tell
|
||||
/// a message it already stores in another folder from one it has never
|
||||
/// seen: the derived key is the same in both folders, only the placement
|
||||
/// differs.
|
||||
async fn sync_messages(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
since_state: Option<&str>,
|
||||
) -> Result<(Vec<Email>, String), MailError> {
|
||||
) -> Result<(Vec<PlacedMessage>, String), MailError> {
|
||||
let changes = self.sync_changes(folder_id, since_state).await?;
|
||||
Ok((changes.upserts, changes.next_state))
|
||||
}
|
||||
|
||||
+579
-102
@@ -1,7 +1,7 @@
|
||||
//! IMAP4rev1 dual-socket connection engine and IDLE push stream manager.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nuncio_core::model::{Email, Folder};
|
||||
use nuncio_core::model::{Email, Folder, MessageIdentity, Placement, PlacementKey, RemoteIdentity};
|
||||
use nuncio_core::TlsMode;
|
||||
use std::fmt::Debug;
|
||||
use std::pin::Pin;
|
||||
@@ -52,7 +52,10 @@ const MAX_MESSAGES_PER_FETCH_BATCH: usize = 200;
|
||||
/// would ever accept is pointless to transfer only to reject.
|
||||
const MAX_MESSAGE_BODY_BYTES: u32 = 25 * 1024 * 1024;
|
||||
|
||||
use crate::backend::{MailBackend, MutationOutcome, RemoteMutationKind, RemoteMutationSpec};
|
||||
use crate::backend::{
|
||||
FolderChanges, MailBackend, MutationOutcome, PlacedMessage, RemoteMutationKind,
|
||||
RemoteMutationSpec,
|
||||
};
|
||||
use crate::parser::{MailError, MimeParserAdapter};
|
||||
|
||||
/// Recover a raw IMAP UID from a [`RemoteMutationSpec::remote_id`] (a decimal
|
||||
@@ -783,7 +786,8 @@ impl ImapEngine {
|
||||
}
|
||||
|
||||
/// Fetch and parse the messages in a folder over an active IMAP session,
|
||||
/// returning the fetched messages and the new sync checkpoint.
|
||||
/// returning each message with the occupancy it was found in, plus the new
|
||||
/// sync checkpoint.
|
||||
///
|
||||
/// `since_state` is the checkpoint returned by a previous sync of this
|
||||
/// folder, encoded as `"{uidvalidity}:{uidnext}"` (see [`CHECKPOINT_DELIM`]).
|
||||
@@ -799,7 +803,7 @@ impl ImapEngine {
|
||||
folder_id: &str,
|
||||
since_state: Option<&str>,
|
||||
session: &mut async_imap::Session<S>,
|
||||
) -> Result<crate::backend::FolderChanges, MailError>
|
||||
) -> Result<FolderChanges, MailError>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static,
|
||||
{
|
||||
@@ -818,15 +822,21 @@ impl ImapEngine {
|
||||
/// exercise the multi-batch path with a small window instead of a huge
|
||||
/// fixture.
|
||||
///
|
||||
/// The sync runs in two phases so the transient working set stays bounded
|
||||
/// regardless of mailbox size:
|
||||
/// The sync runs in two phases:
|
||||
/// 1. A cheap enumeration ([`build_enumerate_command_query`]) reads every
|
||||
/// in-range message's UID and `RFC822.SIZE` with NO body. Messages at or
|
||||
/// below [`MAX_MESSAGE_BODY_BYTES`] are queued for a full-body fetch;
|
||||
/// oversized ones are queued for a headers-only fetch and WARN-logged.
|
||||
/// 2. Each queue is drained in explicit UID-set batches of at most
|
||||
/// `batch_size`, so the full bodies held in memory at once never exceed
|
||||
/// one batch.
|
||||
/// `batch_size`, so a single `UID FETCH` request never asks the server
|
||||
/// for more than that many bodies at a time.
|
||||
///
|
||||
/// Batching bounds the size of each *request*, not the memory this function
|
||||
/// holds: the parsed messages accumulate into one `Vec` across every batch,
|
||||
/// so the returned [`FolderChanges`] carries the whole in-range set and the
|
||||
/// peak working set still scales with the mailbox. That is the shape a
|
||||
/// caller receiving `FolderChanges` depends on, so anything downstream must
|
||||
/// be sized for a whole folder rather than a batch.
|
||||
#[tracing::instrument(
|
||||
skip(self, session),
|
||||
fields(account_id = %self.account_id, folder_id = %folder_id),
|
||||
@@ -838,7 +848,7 @@ impl ImapEngine {
|
||||
since_state: Option<&str>,
|
||||
session: &mut async_imap::Session<S>,
|
||||
batch_size: usize,
|
||||
) -> Result<crate::backend::FolderChanges, MailError>
|
||||
) -> Result<FolderChanges, MailError>
|
||||
where
|
||||
S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static,
|
||||
{
|
||||
@@ -968,16 +978,44 @@ impl ImapEngine {
|
||||
.unwrap_or_else(|| "full-resync-required".to_string()),
|
||||
};
|
||||
|
||||
let removals: Vec<String> = plan
|
||||
.vanished_uids
|
||||
.iter()
|
||||
.map(|uid| self.surrogate_for(folder_id, server_uid_validity, *uid))
|
||||
.collect();
|
||||
let present = plan.present_uids.as_ref().map(|uids| {
|
||||
uids.iter()
|
||||
.map(|uid| self.surrogate_for(folder_id, server_uid_validity, *uid))
|
||||
.collect()
|
||||
});
|
||||
// Absence can only be reported against a known UIDVALIDITY generation.
|
||||
//
|
||||
// `[UIDVALIDITY]` is REQUIRED on a successful SELECT (RFC 3501 section
|
||||
// 6.3.1), so this should not happen -- but a server that omits it would
|
||||
// otherwise produce keys carrying an empty generation, and NONE of them
|
||||
// would match placements stored under a real one. Under the documented
|
||||
// contract ("delete any placement absent from `present`") that reads as
|
||||
// "the folder holds nothing", and the caller deletes the whole mailbox.
|
||||
// Saying nothing is the only safe answer, and it mirrors the checkpoint
|
||||
// above, which likewise refuses to write a validity-less value.
|
||||
let (removals, present) = match server_uid_validity {
|
||||
Some(_) => {
|
||||
// Both are occupancies, not messages: a UID that vanished from
|
||||
// this folder says nothing about copies of the same message
|
||||
// elsewhere.
|
||||
let removals: Vec<PlacementKey> = plan
|
||||
.vanished_uids
|
||||
.iter()
|
||||
.map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid))
|
||||
.collect();
|
||||
let present = plan.present_uids.as_ref().map(|uids| {
|
||||
uids.iter()
|
||||
.map(|uid| self.placement_key_for(folder_id, server_uid_validity, *uid))
|
||||
.collect()
|
||||
});
|
||||
(removals, present)
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
folder_id,
|
||||
vanished = plan.vanished_uids.len(),
|
||||
enumerated = plan.present_uids.as_ref().map_or(0, Vec::len),
|
||||
"mailbox reported no UIDVALIDITY, so this pass cannot key an occupancy and \
|
||||
reports no absence; upserts are still surfaced"
|
||||
);
|
||||
(Vec::new(), None)
|
||||
}
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
account_id = %self.account_id,
|
||||
@@ -985,13 +1023,13 @@ impl ImapEngine {
|
||||
rung = ?plan.rung,
|
||||
fetched = emails.len(),
|
||||
removed = removals.len(),
|
||||
enumerated = plan.present_uids.as_ref().map_or(0, Vec::len),
|
||||
enumerated = present.as_ref().map_or(0, Vec::len),
|
||||
oversized = oversized_uids.len(),
|
||||
skipped = skipped_no_uid,
|
||||
"imap folder sync complete"
|
||||
);
|
||||
|
||||
Ok(crate::backend::FolderChanges {
|
||||
Ok(FolderChanges {
|
||||
upserts: emails,
|
||||
removals,
|
||||
present,
|
||||
@@ -1179,45 +1217,126 @@ impl ImapEngine {
|
||||
Ok(uids)
|
||||
}
|
||||
|
||||
/// Surrogate id for a UID in this account/folder/UIDVALIDITY scope, so
|
||||
/// removals name messages the same way stored rows do.
|
||||
fn surrogate_for(&self, folder_id: &str, uid_validity: Option<u32>, uid: u32) -> String {
|
||||
let validity = uid_validity.map(|v| v.to_string()).unwrap_or_default();
|
||||
Email::surrogate_id(&self.account_id, folder_id, &validity, &uid.to_string())
|
||||
/// The placement key a UID addresses in this account/folder/UIDVALIDITY
|
||||
/// scope, so absence reports name occupancies the same way stored rows do.
|
||||
///
|
||||
/// Deliberately NOT a message key. A UID is only ever evidence about one
|
||||
/// mailbox: `VANISHED` says this folder no longer holds this UID, not that
|
||||
/// the message is gone -- another mailbox may still hold a copy. It also
|
||||
/// could not be a message key even in principle, because the key a message
|
||||
/// is stored under depends on which identity tier the server supported,
|
||||
/// and a UID cannot reconstruct that.
|
||||
fn placement_key_for(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
uid_validity: Option<u32>,
|
||||
uid: u32,
|
||||
) -> PlacementKey {
|
||||
PlacementKey {
|
||||
account_id: self.account_id.clone(),
|
||||
folder_id: folder_id.to_string(),
|
||||
uid_validity: uid_validity.map(|v| v.to_string()).unwrap_or_default(),
|
||||
remote_id: uid.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build an [`Email`] from a full-body (or envelope-only) FETCH item. When
|
||||
/// the item carries a `BODY[]` it is parsed as a full MIME message; when it
|
||||
/// does not (a server that returned only the ENVELOPE), a minimal record is
|
||||
/// synthesised from the envelope so the message is still surfaced honestly.
|
||||
/// Build the identity and placement for one fetched message.
|
||||
///
|
||||
/// `EMAILID` and `X-GM-MSGID` are only present when the server advertised
|
||||
/// `OBJECTID` / `X-GM-EXT-1` and the FETCH asked for them; when neither is
|
||||
/// available the pair of `Message-ID` and content hash carries the identity,
|
||||
/// and only if both are missing does this fall back to the folder-scoped
|
||||
/// surrogate -- which is the one tier that does not survive a move.
|
||||
///
|
||||
/// The server-supplied hints arrive as one [`RemoteIdentity`] rather than as
|
||||
/// four loose arguments: they are already a single value at every call site,
|
||||
/// and spreading them out buys no clarity while pushing this function past
|
||||
/// the argument-count lint.
|
||||
///
|
||||
/// In practice every IMAP call site passes `email_id: None` and
|
||||
/// `gm_msgid: None` today, because the FETCH never asks for them, so this
|
||||
/// engine reaches only the `Message-ID`+content tier or the surrogate. The
|
||||
/// JMAP engine enters at the top tier, which means an IMAP-derived key and
|
||||
/// a JMAP-derived key for the *same* message will not match. Syncing one
|
||||
/// account over both engines does not converge on a shared identity yet.
|
||||
fn place(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
uid_validity: &str,
|
||||
remote_id: &str,
|
||||
remote: RemoteIdentity<'_>,
|
||||
read: bool,
|
||||
) -> (MessageIdentity, Placement) {
|
||||
let identity =
|
||||
Email::derive_message_key(&self.account_id, remote, folder_id, uid_validity, remote_id);
|
||||
let placement = Placement {
|
||||
account_id: self.account_id.clone(),
|
||||
folder_id: folder_id.to_string(),
|
||||
uid_validity: uid_validity.to_string(),
|
||||
remote_id: remote_id.to_string(),
|
||||
read,
|
||||
};
|
||||
(identity, placement)
|
||||
}
|
||||
|
||||
/// Build a [`PlacedMessage`] from a full-body (or envelope-only) FETCH item.
|
||||
/// When the item carries a `BODY[]` it is parsed as a full MIME message;
|
||||
/// when it does not (a server that returned only the ENVELOPE), a minimal
|
||||
/// record is synthesised from the envelope so the message is still surfaced
|
||||
/// honestly.
|
||||
fn build_email_from_fetch(
|
||||
&self,
|
||||
folder_id: &str,
|
||||
server_uid_validity: Option<u32>,
|
||||
fetch_data: &async_imap::types::Fetch,
|
||||
) -> Result<Email, MailError> {
|
||||
) -> Result<PlacedMessage, MailError> {
|
||||
let uid_num = fetch_data.uid.unwrap_or(0);
|
||||
// The message is addressed on the wire by its UID within the folder's
|
||||
// UIDVALIDITY scope; the persisted id is an opaque surrogate hashed over
|
||||
// both plus the account and folder, so the same UID in another
|
||||
// folder/account can never collide.
|
||||
// UIDVALIDITY scope. That pair, with the folder and account, is the
|
||||
// *placement*; the message's own identity is derived separately so that
|
||||
// moving it between folders does not make it a different message.
|
||||
let remote_id = uid_num.to_string();
|
||||
let uid_validity = server_uid_validity
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_default();
|
||||
let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id);
|
||||
|
||||
let is_read = fetch_data
|
||||
.flags()
|
||||
.any(|flag| matches!(flag, async_imap::types::Flag::Seen));
|
||||
|
||||
// EMAILID (RFC 8474) and X-GM-MSGID are not requested by the current
|
||||
// FETCH, so identity rests on the Message-ID/content pair where the
|
||||
// message carried a Message-ID and on the folder-scoped surrogate
|
||||
// otherwise.
|
||||
//
|
||||
// Adding those FETCH items is not a local change. Every already-stored
|
||||
// message would derive a DIFFERENT key on its next resync, so each
|
||||
// re-reported placement repoints to the new key -- and the old message
|
||||
// row, now referenced by nothing, is never reaped. Its decrypted body
|
||||
// stays in the plaintext FTS index forever, which is exactly the leak
|
||||
// the reap on placement removal exists to prevent. The FETCH items can
|
||||
// only be requested once the repoint path reaps the message it
|
||||
// abandons; until then, requesting them leaks plaintext.
|
||||
if let Some(raw_bytes) = fetch_data.body() {
|
||||
let mut email =
|
||||
MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, raw_bytes)?;
|
||||
email.remote_id = remote_id;
|
||||
email.uid_validity = uid_validity;
|
||||
email.read = is_read;
|
||||
return Ok(email);
|
||||
let mut email = MimeParserAdapter::parse_mime(&self.account_id, raw_bytes)?;
|
||||
let (identity, placement) = self.place(
|
||||
folder_id,
|
||||
&uid_validity,
|
||||
&remote_id,
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: email.message_id.as_deref(),
|
||||
content_hash: email.content_hash.as_deref(),
|
||||
},
|
||||
is_read,
|
||||
);
|
||||
email.id = identity.key;
|
||||
return Ok(PlacedMessage {
|
||||
email,
|
||||
source: identity.source,
|
||||
placement,
|
||||
});
|
||||
}
|
||||
|
||||
let subject = fetch_data
|
||||
@@ -1241,24 +1360,51 @@ impl ImapEngine {
|
||||
|
||||
let received_at = fetch_data.internal_date().map_or(0, |dt| dt.timestamp());
|
||||
|
||||
Ok(Email {
|
||||
id: email_id,
|
||||
account_id: self.account_id.clone(),
|
||||
folder_id: folder_id.to_string(),
|
||||
remote_id,
|
||||
uid_validity,
|
||||
subject,
|
||||
sender,
|
||||
recipient,
|
||||
received_at,
|
||||
read: is_read,
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
// No body was returned, so there are no octets to hash -- but the
|
||||
// ENVELOPE carries `message-id` (RFC 3501 section 7.4.2), so identity
|
||||
// capture does not have to be skipped on this path.
|
||||
let message_id = fetch_data
|
||||
.envelope()
|
||||
.and_then(|env| env.message_id.as_ref())
|
||||
.and_then(|raw| Email::normalize_message_id(&String::from_utf8_lossy(raw)));
|
||||
|
||||
// With no octets there is no content hash, and the `Message-ID` alone
|
||||
// is never enough to key on, so this path lands on the surrogate tier.
|
||||
// That is the honest answer: nothing was fetched that could prove two
|
||||
// folders hold the same message.
|
||||
let (identity, placement) = self.place(
|
||||
folder_id,
|
||||
&uid_validity,
|
||||
&remote_id,
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: message_id.as_deref(),
|
||||
content_hash: None,
|
||||
},
|
||||
is_read,
|
||||
);
|
||||
|
||||
Ok(PlacedMessage {
|
||||
email: Email {
|
||||
id: identity.key,
|
||||
account_id: self.account_id.clone(),
|
||||
subject,
|
||||
sender,
|
||||
recipient,
|
||||
received_at,
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id,
|
||||
content_hash: None,
|
||||
},
|
||||
source: identity.source,
|
||||
placement,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build an [`Email`] for a message whose body exceeded
|
||||
/// Build a [`PlacedMessage`] for a message whose body exceeded
|
||||
/// [`MAX_MESSAGE_BODY_BYTES`]: its headers (`BODY[HEADER]`) are parsed for
|
||||
/// honest metadata (subject/sender/date) but the oversized body is left
|
||||
/// unfetched, so the record carries an EMPTY body rather than a fabricated
|
||||
@@ -1269,30 +1415,50 @@ impl ImapEngine {
|
||||
folder_id: &str,
|
||||
server_uid_validity: Option<u32>,
|
||||
fetch_data: &async_imap::types::Fetch,
|
||||
) -> Email {
|
||||
) -> PlacedMessage {
|
||||
let uid_num = fetch_data.uid.unwrap_or(0);
|
||||
let remote_id = uid_num.to_string();
|
||||
let uid_validity = server_uid_validity
|
||||
.map(|v| v.to_string())
|
||||
.unwrap_or_default();
|
||||
let email_id = Email::surrogate_id(&self.account_id, folder_id, &uid_validity, &remote_id);
|
||||
let is_read = fetch_data
|
||||
.flags()
|
||||
.any(|flag| matches!(flag, async_imap::types::Flag::Seen));
|
||||
|
||||
if let Some(header_bytes) = fetch_data.header() {
|
||||
if let Ok(mut email) =
|
||||
MimeParserAdapter::parse_mime(&email_id, &self.account_id, folder_id, header_bytes)
|
||||
{
|
||||
if let Ok(mut email) = MimeParserAdapter::parse_mime(&self.account_id, header_bytes) {
|
||||
// A headers-only parse yields no body, but be explicit that the
|
||||
// oversized body was intentionally not fetched.
|
||||
email.remote_id = remote_id;
|
||||
email.uid_validity = uid_validity;
|
||||
email.read = is_read;
|
||||
email.body_plain = None;
|
||||
email.body_html = None;
|
||||
email.attachments = Vec::new();
|
||||
return email;
|
||||
// `parse_mime` hashed the header section it was handed, but
|
||||
// `content_hash` is defined over a message's FULL octets.
|
||||
// Keeping that value would publish a hash that silently means
|
||||
// something else, so drop it; the `Message-ID` it recovered
|
||||
// from those headers is still correct and is kept. Dropping it
|
||||
// also costs this message the content tier -- a `Message-ID`
|
||||
// never keys on its own -- so an oversized message identifies
|
||||
// by its folder-scoped surrogate until the octets are fetched.
|
||||
email.content_hash = None;
|
||||
let (identity, placement) = self.place(
|
||||
folder_id,
|
||||
&uid_validity,
|
||||
&remote_id,
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: email.message_id.as_deref(),
|
||||
content_hash: None,
|
||||
},
|
||||
is_read,
|
||||
);
|
||||
email.id = identity.key;
|
||||
return PlacedMessage {
|
||||
email,
|
||||
source: identity.source,
|
||||
placement,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1300,20 +1466,31 @@ impl ImapEngine {
|
||||
// `build_email_from_fetch` cannot error on this path (no body to parse),
|
||||
// but if it ever did, surface a minimal honest record rather than panic.
|
||||
self.build_email_from_fetch(folder_id, server_uid_validity, fetch_data)
|
||||
.unwrap_or_else(|_| Email {
|
||||
id: email_id,
|
||||
account_id: self.account_id.clone(),
|
||||
folder_id: folder_id.to_string(),
|
||||
remote_id,
|
||||
uid_validity,
|
||||
subject: "No Subject".to_string(),
|
||||
sender: "unknown@nuncio.mx".to_string(),
|
||||
recipient: "me@nuncio.mx".to_string(),
|
||||
received_at: 0,
|
||||
read: is_read,
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
.unwrap_or_else(|_| {
|
||||
let (identity, placement) = self.place(
|
||||
folder_id,
|
||||
&uid_validity,
|
||||
&remote_id,
|
||||
RemoteIdentity::default(),
|
||||
is_read,
|
||||
);
|
||||
PlacedMessage {
|
||||
email: Email {
|
||||
id: identity.key,
|
||||
account_id: self.account_id.clone(),
|
||||
subject: "No Subject".to_string(),
|
||||
sender: "unknown@nuncio.mx".to_string(),
|
||||
recipient: "me@nuncio.mx".to_string(),
|
||||
received_at: 0,
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
source: identity.source,
|
||||
placement,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1325,7 +1502,7 @@ impl ImapEngine {
|
||||
&self,
|
||||
folder_id: &str,
|
||||
since_state: Option<&str>,
|
||||
) -> Result<crate::backend::FolderChanges, MailError> {
|
||||
) -> Result<FolderChanges, MailError> {
|
||||
let (username, password) = match (&self.username, &self.password) {
|
||||
(Some(u), Some(p)) => (u.as_str(), p.as_str()),
|
||||
_ => {
|
||||
@@ -1793,7 +1970,7 @@ impl MailBackend for ImapEngine {
|
||||
&self,
|
||||
folder_id: &str,
|
||||
since_state: Option<&str>,
|
||||
) -> Result<crate::backend::FolderChanges, MailError> {
|
||||
) -> Result<FolderChanges, MailError> {
|
||||
self.sync_folder_messages(folder_id, since_state).await
|
||||
}
|
||||
|
||||
@@ -1808,6 +1985,7 @@ impl MailBackend for ImapEngine {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use nuncio_core::model::IdentitySource;
|
||||
use tracing_test::traced_test;
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2193,17 +2371,25 @@ mod tests {
|
||||
}
|
||||
|
||||
/// The RFC822 body of a scripted fixture message.
|
||||
///
|
||||
/// Carries a `Message-ID` so the full-body path reaches the content tier.
|
||||
/// That is what makes the headers-only path's tier meaningful to assert:
|
||||
/// the same message identifies differently there, and only because the
|
||||
/// header-section hash is deliberately discarded.
|
||||
fn fixture_body(uid: u32) -> String {
|
||||
format!(
|
||||
"Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\
|
||||
Message-ID: <msg{uid}@example.test>\r\n\
|
||||
To: me@example.test\r\n\r\nBody of message {uid}.\r\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// The header block (no body) of a scripted fixture message.
|
||||
/// The header block (no body) of a scripted fixture message. Same headers
|
||||
/// as [`fixture_body`], including the `Message-ID`.
|
||||
fn fixture_header(uid: u32) -> String {
|
||||
format!(
|
||||
"Subject: Message {uid}\r\nFrom: sender{uid}@example.test\r\n\
|
||||
Message-ID: <msg{uid}@example.test>\r\n\
|
||||
To: me@example.test\r\n\r\n"
|
||||
)
|
||||
}
|
||||
@@ -2237,13 +2423,17 @@ mod tests {
|
||||
/// and records every `UID FETCH` command line the client issued so a test
|
||||
/// can assert on the exact batching. Returns the synced emails, the new
|
||||
/// checkpoint, and the captured `UID FETCH` command lines.
|
||||
///
|
||||
/// `uidvalidity` is an `Option` so a test can script the one thing RFC 3501
|
||||
/// section 6.3.1 says a SELECT must never do -- omit `[UIDVALIDITY]` -- and
|
||||
/// check the sync stays safe when a server does it anyway.
|
||||
async fn run_batched_sync(
|
||||
messages: Vec<(u32, u32)>,
|
||||
uidvalidity: u32,
|
||||
uidvalidity: Option<u32>,
|
||||
uidnext: u32,
|
||||
since_state: Option<String>,
|
||||
batch_size: usize,
|
||||
) -> (Vec<Email>, String, Vec<String>) {
|
||||
) -> (FolderChanges, Vec<String>) {
|
||||
let (client_io, server_io) = tokio::io::duplex(65536);
|
||||
let fetch_cmds = Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
|
||||
let server_cmds = fetch_cmds.clone();
|
||||
@@ -2305,9 +2495,13 @@ mod tests {
|
||||
.write_all(format!("{tag} OK Logged in\r\n").as_bytes())
|
||||
.await;
|
||||
} else if upper.contains("SELECT") {
|
||||
let validity_line = match uidvalidity {
|
||||
Some(v) => format!("* OK [UIDVALIDITY {v}] ok\r\n"),
|
||||
None => String::new(),
|
||||
};
|
||||
let resp = format!(
|
||||
"* FLAGS (\\Seen)\r\n* {exists} EXISTS\r\n* 0 RECENT\r\n\
|
||||
* OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\
|
||||
{validity_line}* OK [UIDNEXT {uidnext}] ok\r\n\
|
||||
{tag} OK [READ-WRITE] SELECT done\r\n"
|
||||
);
|
||||
let _ = io.write_all(resp.as_bytes()).await;
|
||||
@@ -2331,12 +2525,11 @@ mod tests {
|
||||
.sync_folder_messages_batched("INBOX", since_state.as_deref(), &mut session, batch_size)
|
||||
.await
|
||||
.expect("batched sync succeeds");
|
||||
let (emails, checkpoint) = (changes.upserts, changes.next_state);
|
||||
drop(session);
|
||||
let _ = server.await;
|
||||
|
||||
let cmds = fetch_cmds.lock().expect("lock captured commands").clone();
|
||||
(emails, checkpoint, cmds)
|
||||
(changes, cmds)
|
||||
}
|
||||
|
||||
/// The captured `UID FETCH` command lines that pulled FULL bodies (a
|
||||
@@ -2358,10 +2551,11 @@ mod tests {
|
||||
// `1:*` body fetch. The enumeration discovers the UIDs; the bodies are
|
||||
// then fetched in explicit UID-set chunks of at most the batch size.
|
||||
let messages: Vec<(u32, u32)> = (1..=5).map(|u| (u, 100)).collect();
|
||||
let (emails, checkpoint, cmds) = run_batched_sync(messages, 1, 6, None, 2).await;
|
||||
let (changes, cmds) = run_batched_sync(messages, Some(1), 6, None, 2).await;
|
||||
let emails = &changes.upserts;
|
||||
|
||||
assert_eq!(emails.len(), 5, "every message must be synced");
|
||||
assert_eq!(checkpoint, "v1:uidnext:1:6");
|
||||
assert_eq!(changes.next_state, "v1:uidnext:1:6");
|
||||
|
||||
let body_cmds = body_fetch_commands(&cmds);
|
||||
assert_eq!(
|
||||
@@ -2391,7 +2585,8 @@ mod tests {
|
||||
// normal messages get full bodies.
|
||||
let huge = MAX_MESSAGE_BODY_BYTES + 1;
|
||||
let messages: Vec<(u32, u32)> = vec![(1, 100), (2, huge), (3, 100)];
|
||||
let (emails, _checkpoint, cmds) = run_batched_sync(messages, 1, 4, None, 200).await;
|
||||
let (changes, cmds) = run_batched_sync(messages, Some(1), 4, None, 200).await;
|
||||
let emails = &changes.upserts;
|
||||
|
||||
assert_eq!(emails.len(), 3, "no message may be silently dropped");
|
||||
|
||||
@@ -2416,27 +2611,54 @@ mod tests {
|
||||
// body -- never a fabricated one.
|
||||
let oversized = emails
|
||||
.iter()
|
||||
.find(|e| e.remote_id == "2")
|
||||
.find(|e| e.placement.remote_id == "2")
|
||||
.expect("oversized message must be present");
|
||||
assert!(
|
||||
oversized.body_plain.is_none() && oversized.body_html.is_none(),
|
||||
oversized.email.body_plain.is_none() && oversized.email.body_html.is_none(),
|
||||
"oversized message must carry no body, got: {oversized:?}"
|
||||
);
|
||||
assert!(
|
||||
oversized.subject.contains('2'),
|
||||
oversized.email.subject.contains('2'),
|
||||
"oversized message must keep honest header metadata, got: {oversized:?}"
|
||||
);
|
||||
|
||||
// The header section it WAS given hashes to something, but that hash
|
||||
// covers only the headers, not the message. Publishing it would key the
|
||||
// message on the wrong octets -- and then the same message would key
|
||||
// differently once its body was finally fetched, resurrecting it as a
|
||||
// duplicate. So the hash is dropped, which costs this message the
|
||||
// content tier even though it carried a Message-ID.
|
||||
assert!(
|
||||
oversized.email.message_id.is_some(),
|
||||
"the headers-only parse must still recover the Message-ID"
|
||||
);
|
||||
assert_eq!(
|
||||
oversized.email.content_hash, None,
|
||||
"a hash over only the header section must never be published"
|
||||
);
|
||||
assert_eq!(
|
||||
oversized.source,
|
||||
IdentitySource::Surrogate,
|
||||
"without octets to hash, a Message-ID alone must not reach the content tier"
|
||||
);
|
||||
|
||||
// The normal messages DID get full bodies.
|
||||
for uid in ["1", "3"] {
|
||||
let email = emails
|
||||
.iter()
|
||||
.find(|e| e.remote_id == uid)
|
||||
.find(|e| e.placement.remote_id == uid)
|
||||
.expect("normal message present");
|
||||
assert!(
|
||||
email.body_plain.is_some(),
|
||||
email.email.body_plain.is_some(),
|
||||
"normal message {uid} must have a body, got: {email:?}"
|
||||
);
|
||||
// The contrast that gives the assertion above its force: the same
|
||||
// headers, plus the octets, DO reach the content tier.
|
||||
assert_eq!(
|
||||
email.source,
|
||||
IdentitySource::MessageIdContent,
|
||||
"a full-body fetch with a Message-ID must reach the content tier"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2447,24 +2669,24 @@ mod tests {
|
||||
// checkpoint enumerates only the narrowed `{uidnext}:*` range (so a
|
||||
// fire-once-per-new-message consumer never re-sees old mail).
|
||||
let messages: Vec<(u32, u32)> = (1..=3).map(|u| (u, 100)).collect();
|
||||
let (first, checkpoint, first_cmds) =
|
||||
run_batched_sync(messages.clone(), 1, 4, None, 2).await;
|
||||
assert_eq!(first.len(), 3);
|
||||
let (first, first_cmds) = run_batched_sync(messages.clone(), Some(1), 4, None, 2).await;
|
||||
let checkpoint = first.next_state.clone();
|
||||
assert_eq!(first.upserts.len(), 3);
|
||||
assert_eq!(checkpoint, "v1:uidnext:1:4");
|
||||
assert!(
|
||||
first_cmds.iter().any(|c| c.contains("1:*")),
|
||||
"first sync enumerates the full range, got: {first_cmds:?}"
|
||||
);
|
||||
|
||||
let (second, checkpoint2, second_cmds) =
|
||||
run_batched_sync(messages, 1, 4, Some(checkpoint.clone()), 2).await;
|
||||
let (second, second_cmds) =
|
||||
run_batched_sync(messages, Some(1), 4, Some(checkpoint.clone()), 2).await;
|
||||
assert_eq!(
|
||||
second.len(),
|
||||
second.upserts.len(),
|
||||
0,
|
||||
"no new mail arrived, so an incremental sync returns nothing"
|
||||
);
|
||||
assert_eq!(
|
||||
checkpoint2, "v1:uidnext:1:4",
|
||||
second.next_state, "v1:uidnext:1:4",
|
||||
"checkpoint stays stable across an empty pass"
|
||||
);
|
||||
let second_bodies = body_fetch_ranges(&second_cmds);
|
||||
@@ -3258,4 +3480,259 @@ mod tests {
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drive a folder sync over a scripted server that serves exactly ONE
|
||||
/// message: `uid`, with `raw` as its full RFC822 octets and `seen`
|
||||
/// controlling its `\Seen` flag. Holding the octets fixed while varying the
|
||||
/// folder, UIDVALIDITY and UID is the shape of "the same message, found in
|
||||
/// two mailboxes" -- which is what the identity/placement split exists for.
|
||||
///
|
||||
/// Passing `raw = None` scripts the envelope-only path instead: the server
|
||||
/// answers the body fetch with an `ENVELOPE` and no `BODY[]`, which is what
|
||||
/// a server that will not hand over octets looks like.
|
||||
async fn run_single_message_sync(
|
||||
folder_id: &str,
|
||||
uidvalidity: u32,
|
||||
uid: u32,
|
||||
seen: bool,
|
||||
raw: Option<&str>,
|
||||
) -> PlacedMessage {
|
||||
let (client_io, server_io) = tokio::io::duplex(65536);
|
||||
let raw = raw.map(str::to_string);
|
||||
let uidnext = uid.saturating_add(1);
|
||||
let flags = if seen { "\\Seen" } else { "" };
|
||||
|
||||
let server = tokio::spawn(async move {
|
||||
let mut io = server_io;
|
||||
while let Some(line) = read_scripted_line(&mut io).await {
|
||||
let tag = line.split_whitespace().next().unwrap_or("").to_string();
|
||||
let upper = line.to_ascii_uppercase();
|
||||
if upper.contains("UID FETCH") {
|
||||
let size = raw.as_ref().map_or(64, String::len);
|
||||
let resp = if !upper.contains("BODY.PEEK[]") {
|
||||
// Enumeration pass: UID + size only, no body.
|
||||
format!(
|
||||
"* 1 FETCH (UID {uid} RFC822.SIZE {size})\r\n\
|
||||
{tag} OK FETCH completed\r\n"
|
||||
)
|
||||
} else if let Some(raw) = raw.as_ref() {
|
||||
format!(
|
||||
"* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \
|
||||
BODY[] {{{size}}}\r\n{raw})\r\n{tag} OK FETCH completed\r\n"
|
||||
)
|
||||
} else {
|
||||
// A server that answers with metadata but no octets.
|
||||
// ENVELOPE still carries `message-id` (RFC 3501
|
||||
// section 7.4.2), so identity capture is not skipped --
|
||||
// but there is nothing to hash.
|
||||
format!(
|
||||
"* 1 FETCH (UID {uid} RFC822.SIZE {size} FLAGS ({flags}) \
|
||||
ENVELOPE (\"Mon, 1 Jan 2024 00:00:00 +0000\" \"Env subject\" \
|
||||
((\"A\" NIL \"alice\" \"b.example\")) \
|
||||
((\"A\" NIL \"alice\" \"b.example\")) \
|
||||
((\"A\" NIL \"alice\" \"b.example\")) \
|
||||
((\"B\" NIL \"bob\" \"b.example\")) \
|
||||
NIL NIL NIL \"<env@b.example>\"))\r\n\
|
||||
{tag} OK FETCH completed\r\n"
|
||||
)
|
||||
};
|
||||
let _ = io.write_all(resp.as_bytes()).await;
|
||||
} else if upper.contains("CAPABILITY") {
|
||||
// The planner negotiates capabilities before selecting; a
|
||||
// script that never answers this blocks the client forever.
|
||||
let _ = io
|
||||
.write_all(
|
||||
format!(
|
||||
"* CAPABILITY IMAP4rev1 UIDPLUS MOVE\r\n{tag} OK CAPABILITY\r\n"
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.await;
|
||||
} else if upper.contains("LOGIN") {
|
||||
let _ = io
|
||||
.write_all(format!("{tag} OK Logged in\r\n").as_bytes())
|
||||
.await;
|
||||
} else if upper.contains("SELECT") {
|
||||
let resp = format!(
|
||||
"* FLAGS (\\Seen)\r\n* 1 EXISTS\r\n* 0 RECENT\r\n\
|
||||
* OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT {uidnext}] ok\r\n\
|
||||
{tag} OK [READ-WRITE] SELECT done\r\n"
|
||||
);
|
||||
let _ = io.write_all(resp.as_bytes()).await;
|
||||
} else if upper.contains("LOGOUT") {
|
||||
let _ = io
|
||||
.write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes())
|
||||
.await;
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let client = async_imap::Client::new(client_io);
|
||||
let mut session = client
|
||||
.login("user", "pass")
|
||||
.await
|
||||
.map_err(|(e, _)| e)
|
||||
.expect("scripted login succeeds");
|
||||
let engine = ImapEngine::new("acct-1", "example.test", 993);
|
||||
let changes = engine
|
||||
.sync_folder_messages_batched(folder_id, None, &mut session, 8)
|
||||
.await
|
||||
.expect("scripted sync succeeds");
|
||||
drop(session);
|
||||
let _ = server.await;
|
||||
|
||||
changes
|
||||
.upserts
|
||||
.into_iter()
|
||||
.next()
|
||||
.expect("the scripted message must be surfaced")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_imap_backend_gives_one_identity_to_a_message_found_in_two_folders() {
|
||||
// Byte-identical octets under two different UIDs in two different
|
||||
// mailboxes -- an IMAP COPY, or a Gmail label. Identity comes from the
|
||||
// Message-ID/content pair, so the derived key is ONE value while the
|
||||
// placements stay distinct. Before the split this was two messages.
|
||||
let raw = "Message-ID: <a@b.example>\r\nSubject: Hi\r\n\
|
||||
From: alice@b.example\r\n\r\nbody\r\n";
|
||||
let inbox = run_single_message_sync("INBOX", 42, 5, true, Some(raw)).await;
|
||||
let archive = run_single_message_sync("Archive", 77, 9, false, Some(raw)).await;
|
||||
|
||||
assert_eq!(inbox.source, IdentitySource::MessageIdContent);
|
||||
assert_eq!(archive.source, IdentitySource::MessageIdContent);
|
||||
assert_eq!(
|
||||
inbox.email.id, archive.email.id,
|
||||
"the same octets in two folders must derive one message key"
|
||||
);
|
||||
|
||||
assert_eq!(inbox.placement.account_id, "acct-1");
|
||||
assert_eq!(inbox.placement.folder_id, "INBOX");
|
||||
assert_eq!(inbox.placement.uid_validity, "42");
|
||||
assert_eq!(inbox.placement.remote_id, "5");
|
||||
assert_eq!(archive.placement.folder_id, "Archive");
|
||||
assert_eq!(archive.placement.uid_validity, "77");
|
||||
assert_eq!(archive.placement.remote_id, "9");
|
||||
|
||||
// IMAP `\Seen` is per-mailbox (RFC 3501 section 2.3.2), so read state
|
||||
// belongs to the occupancy, not to the message.
|
||||
assert!(inbox.placement.read);
|
||||
assert!(!archive.placement.read);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_message_with_no_message_id_stays_on_the_folder_scoped_surrogate() {
|
||||
// RFC 5322 section 3.6.4 makes Message-ID a SHOULD, so a message may
|
||||
// carry none. Without it there is no content tier, and the backend must
|
||||
// report two honest folder-scoped keys rather than claim a stability
|
||||
// the server gave it no basis for.
|
||||
let raw = "Subject: No id\r\nFrom: alice@b.example\r\n\r\nbody\r\n";
|
||||
let inbox = run_single_message_sync("INBOX", 42, 5, false, Some(raw)).await;
|
||||
let archive = run_single_message_sync("Archive", 77, 9, false, Some(raw)).await;
|
||||
|
||||
assert_eq!(inbox.source, IdentitySource::Surrogate);
|
||||
assert_ne!(
|
||||
inbox.email.id, archive.email.id,
|
||||
"a surrogate is folder-scoped and must not be claimed to survive a move"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_envelope_only_fetch_keeps_its_message_id_but_stays_on_the_surrogate() {
|
||||
// A server that returns metadata and no octets. ENVELOPE carries
|
||||
// `message-id` (RFC 3501 section 7.4.2) so the header is still
|
||||
// captured, but there is nothing to hash -- and a Message-ID alone is
|
||||
// public and forgeable, so it must never key storage by itself.
|
||||
let placed = run_single_message_sync("INBOX", 42, 5, false, None).await;
|
||||
|
||||
assert_eq!(placed.email.subject, "Env subject");
|
||||
assert_eq!(placed.email.message_id.as_deref(), Some("env@b.example"));
|
||||
assert_eq!(
|
||||
placed.email.content_hash, None,
|
||||
"no octets were fetched, so there is nothing honest to hash"
|
||||
);
|
||||
assert_eq!(
|
||||
placed.source,
|
||||
IdentitySource::Surrogate,
|
||||
"an envelope-only fetch must not reach the content tier"
|
||||
);
|
||||
assert_eq!(placed.placement.remote_id, "5");
|
||||
assert_eq!(placed.placement.uid_validity, "42");
|
||||
}
|
||||
|
||||
#[traced_test]
|
||||
#[tokio::test]
|
||||
async fn a_select_without_uidvalidity_reports_no_absence_at_all() {
|
||||
// RFC 3501 section 6.3.1 makes `[UIDVALIDITY]` REQUIRED on SELECT, so a
|
||||
// server omitting it is broken -- but the failure mode is a wiped
|
||||
// mailbox, not a bad sync. Every placement key this pass could build
|
||||
// would carry an empty generation and match nothing the store holds,
|
||||
// and under the `present` contract that reads as "the folder is empty".
|
||||
// The pass must therefore decline to speak about absence, while still
|
||||
// surfacing what it fetched.
|
||||
let messages: Vec<(u32, u32)> = (1..=2).map(|u| (u, 100)).collect();
|
||||
let (changes, _cmds) = run_batched_sync(messages, None, 3, None, 200).await;
|
||||
|
||||
assert_eq!(
|
||||
changes.upserts.len(),
|
||||
2,
|
||||
"the messages themselves are still surfaced"
|
||||
);
|
||||
assert_eq!(
|
||||
changes.present, None,
|
||||
"a pass that cannot key an occupancy must not claim to know what is present"
|
||||
);
|
||||
assert!(
|
||||
changes.removals.is_empty(),
|
||||
"nor may it report a removal, got: {:?}",
|
||||
changes.removals
|
||||
);
|
||||
// And it must say why, rather than failing silently.
|
||||
assert!(logs_contain("mailbox reported no UIDVALIDITY"));
|
||||
|
||||
// The same pass against a compliant server DOES report presence, so the
|
||||
// assertions above are about the missing UIDVALIDITY and nothing else.
|
||||
let messages: Vec<(u32, u32)> = (1..=2).map(|u| (u, 100)).collect();
|
||||
let (compliant, _cmds) = run_batched_sync(messages, Some(7), 3, None, 200).await;
|
||||
let present = compliant
|
||||
.present
|
||||
.expect("a compliant SELECT must let the pass report presence");
|
||||
assert_eq!(present.len(), 2);
|
||||
assert!(present.iter().all(|k| k.uid_validity == "7"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_message_fetched_from_two_folders_keeps_one_identity() {
|
||||
// Both folders return the same Message-ID and the same octets, so the
|
||||
// content tier must produce one key from two different UIDs.
|
||||
let raw = b"Message-ID: <a@b.example>\r\nSubject: Hi\r\n\r\nbody";
|
||||
let hash = Email::content_hash_of(raw);
|
||||
let inbox = Email::derive_message_key(
|
||||
"acct-1",
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: Some("a@b.example"),
|
||||
content_hash: Some(&hash),
|
||||
},
|
||||
"INBOX",
|
||||
"42",
|
||||
"5",
|
||||
);
|
||||
let archive = Email::derive_message_key(
|
||||
"acct-1",
|
||||
RemoteIdentity {
|
||||
email_id: None,
|
||||
gm_msgid: None,
|
||||
message_id: Some("a@b.example"),
|
||||
content_hash: Some(&hash),
|
||||
},
|
||||
"Archive",
|
||||
"77",
|
||||
"9",
|
||||
);
|
||||
assert_eq!(inbox.key, archive.key);
|
||||
assert_eq!(inbox.source, IdentitySource::MessageIdContent);
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,22 @@
|
||||
//! JMAP (RFC 8620 / RFC 8621) protocol engine, session discovery, and differential update parser.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nuncio_core::model::{Email, Folder};
|
||||
use nuncio_core::model::{Email, Folder, Placement, RemoteIdentity};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::backend::{MailBackend, MutationOutcome, RemoteMutationKind, RemoteMutationSpec};
|
||||
use crate::backend::{
|
||||
FolderChanges, MailBackend, MutationOutcome, PlacedMessage, RemoteMutationKind,
|
||||
RemoteMutationSpec,
|
||||
};
|
||||
use crate::parser::MailError;
|
||||
|
||||
/// Stand-in for the UIDVALIDITY scope on JMAP, which has no such concept: a
|
||||
/// JMAP Email object id is already globally stable within its account (RFC
|
||||
/// 8621), so a fixed sentinel keeps the surrogate-id computation uniform with
|
||||
/// IMAP without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats
|
||||
/// this non-numeric value as "no validity to compare" and is never engaged for
|
||||
/// a JMAP mutation, which addresses messages by object id.
|
||||
/// JMAP Email object id is already stable within its account (RFC 8621), so a
|
||||
/// fixed sentinel keeps a placement key the same shape on both protocols
|
||||
/// without implying a UIDVALIDITY guard. The IMAP UIDVALIDITY guard treats this
|
||||
/// non-numeric value as "no validity to compare" and is never engaged for a
|
||||
/// JMAP mutation, which addresses messages by object id.
|
||||
const JMAP_UID_VALIDITY_SENTINEL: &str = "jmap";
|
||||
|
||||
/// JMAP Session Object (RFC 8620 Section 2).
|
||||
@@ -92,6 +95,10 @@ pub struct JmapEmail {
|
||||
pub received_at: Option<i64>,
|
||||
pub is_unread: Option<bool>,
|
||||
pub body_snippet: Option<String>,
|
||||
/// RFC 8621 models `messageId` as a list, because a message may carry more
|
||||
/// than one `Message-ID` header. The first entry is the one used.
|
||||
#[serde(default)]
|
||||
pub message_id: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
/// JMAP email address object.
|
||||
@@ -184,7 +191,7 @@ impl JmapEngine {
|
||||
{
|
||||
"accountId": account_id,
|
||||
"ids": ids,
|
||||
"properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet"]
|
||||
"properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet", "messageId"]
|
||||
},
|
||||
"c1"
|
||||
]
|
||||
@@ -364,11 +371,12 @@ impl JmapEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse raw JMAP `Email/get` JSON response payload into domain [`Email`] list and new state string.
|
||||
/// Parse a raw JMAP `Email/get` JSON response payload into a list of
|
||||
/// [`PlacedMessage`] and the new state string.
|
||||
pub fn parse_email_get_response(
|
||||
&self,
|
||||
raw_json: &str,
|
||||
) -> Result<(Vec<Email>, String), MailError> {
|
||||
) -> Result<(Vec<PlacedMessage>, String), MailError> {
|
||||
let val: Value = serde_json::from_str(raw_json)
|
||||
.map_err(|e| MailError::ParseFailed(format!("invalid JMAP JSON response: {e}")))?;
|
||||
let payload = Self::extract_method_response_payload(&val)?;
|
||||
@@ -394,30 +402,58 @@ impl JmapEngine {
|
||||
.unwrap_or_else(|| "me@nuncio.mx".to_string());
|
||||
|
||||
let folder_id = "inbox".to_string();
|
||||
// The JMAP object id is the protocol-native id; the persisted
|
||||
// id is an opaque surrogate hashed over it plus the account,
|
||||
// folder, and (sentinel) scope so it stays uniform with IMAP.
|
||||
let remote_id = item.id;
|
||||
let id = Email::surrogate_id(
|
||||
let message_id = item
|
||||
.message_id
|
||||
.as_ref()
|
||||
.and_then(|ids| ids.first())
|
||||
.and_then(|raw| Email::normalize_message_id(raw));
|
||||
|
||||
// A JMAP Email id is server-assigned and stable across mailboxes
|
||||
// within the account, which is the same guarantee RFC 8474
|
||||
// EMAILID gives, so it enters at the top tier rather than
|
||||
// through the surrogate.
|
||||
let identity = Email::derive_message_key(
|
||||
&self.account_id,
|
||||
RemoteIdentity {
|
||||
email_id: Some(&remote_id),
|
||||
gm_msgid: None,
|
||||
message_id: message_id.as_deref(),
|
||||
// `Email/get` returns a `bodySnippet`, not the
|
||||
// message's octets, so there is nothing to hash here
|
||||
// without an extra download per message.
|
||||
content_hash: None,
|
||||
},
|
||||
&folder_id,
|
||||
JMAP_UID_VALIDITY_SENTINEL,
|
||||
&remote_id,
|
||||
);
|
||||
Email {
|
||||
id,
|
||||
let placement = Placement {
|
||||
account_id: self.account_id.clone(),
|
||||
folder_id,
|
||||
remote_id,
|
||||
// JMAP has no UIDVALIDITY; the sentinel keeps the placement
|
||||
// key the same shape as IMAP's without implying a guard.
|
||||
uid_validity: JMAP_UID_VALIDITY_SENTINEL.to_string(),
|
||||
subject: item.subject.unwrap_or_else(|| "No Subject".to_string()),
|
||||
sender,
|
||||
recipient,
|
||||
received_at: item.received_at.unwrap_or(0),
|
||||
remote_id: remote_id.clone(),
|
||||
read: !item.is_unread.unwrap_or(false),
|
||||
body_plain: item.body_snippet,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
};
|
||||
|
||||
PlacedMessage {
|
||||
email: Email {
|
||||
id: identity.key,
|
||||
account_id: self.account_id.clone(),
|
||||
subject: item.subject.unwrap_or_else(|| "No Subject".to_string()),
|
||||
sender,
|
||||
recipient,
|
||||
received_at: item.received_at.unwrap_or(0),
|
||||
body_plain: item.body_snippet,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id,
|
||||
content_hash: None,
|
||||
},
|
||||
source: identity.source,
|
||||
placement,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
@@ -565,7 +601,7 @@ impl MailBackend for JmapEngine {
|
||||
&self,
|
||||
folder_id: &str,
|
||||
_since_state: Option<&str>,
|
||||
) -> Result<crate::backend::FolderChanges, MailError> {
|
||||
) -> Result<FolderChanges, MailError> {
|
||||
if !self.has_credentials() {
|
||||
return Err(MailError::AuthError(
|
||||
"JMAP message sync requires credentials".to_string(),
|
||||
@@ -589,7 +625,7 @@ impl MailBackend for JmapEngine {
|
||||
"jmap message sync complete"
|
||||
);
|
||||
|
||||
Ok(crate::backend::FolderChanges {
|
||||
Ok(FolderChanges {
|
||||
upserts: emails,
|
||||
removals: Vec::new(),
|
||||
// `Email/query` was filtered to one mailbox, but this engine does
|
||||
@@ -722,6 +758,35 @@ mod tests {
|
||||
assert_eq!(new_state, "s-2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_email_get_response_captures_normalized_message_id() {
|
||||
// RFC 8621 models `messageId` as a list; the first entry is taken and
|
||||
// normalized to the same canonical form the IMAP paths produce, so two
|
||||
// engines syncing one account via different protocols agree.
|
||||
let raw = r#"{
|
||||
"methodResponses": [
|
||||
["Email/get", {"state": "s-1", "list": [
|
||||
{"id": "m-1", "subject": "Hi", "messageId": ["<Abc.123@mail.nuncio.mx>"]},
|
||||
{"id": "m-2", "subject": "No id"}
|
||||
]}, "c1"]
|
||||
]
|
||||
}"#;
|
||||
let engine = JmapEngine::new("acct-1");
|
||||
let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get");
|
||||
|
||||
assert_eq!(
|
||||
emails[0].email.message_id.as_deref(),
|
||||
Some("Abc.123@mail.nuncio.mx")
|
||||
);
|
||||
assert_eq!(
|
||||
emails[1].email.message_id, None,
|
||||
"an absent messageId must stay None, not become empty"
|
||||
);
|
||||
// `Email/get` returns a snippet, never the octets, so there is nothing
|
||||
// honest to hash on this path.
|
||||
assert_eq!(emails[0].email.content_hash, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_invalid_jmap_json_fails() {
|
||||
let engine = JmapEngine::new("acct-1");
|
||||
|
||||
@@ -18,7 +18,7 @@ pub mod smtp;
|
||||
pub mod test_server;
|
||||
|
||||
pub use backend::{
|
||||
FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage,
|
||||
FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage, PlacedMessage,
|
||||
RemoteMutationKind, RemoteMutationSpec,
|
||||
};
|
||||
pub use imap::{IdleSocketState, ImapDualSocketManager, ImapEngine};
|
||||
|
||||
+170
-28
@@ -1,11 +1,12 @@
|
||||
//! Deterministic mock mail backend for offline testing and integration verification.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nuncio_core::model::{Email, Folder};
|
||||
use nuncio_core::model::{Email, Folder, IdentitySource, Placement, PlacementKey};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use crate::backend::{
|
||||
MailBackend, MessageSender, MutationOutcome, OutboundMessage, RemoteMutationSpec,
|
||||
FolderChanges, MailBackend, MessageSender, MutationOutcome, OutboundMessage, PlacedMessage,
|
||||
RemoteMutationSpec,
|
||||
};
|
||||
use crate::parser::MailError;
|
||||
|
||||
@@ -13,7 +14,7 @@ use crate::parser::MailError;
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct MockMailBackend {
|
||||
folders: Arc<Mutex<Vec<Folder>>>,
|
||||
messages: Arc<Mutex<Vec<Email>>>,
|
||||
messages: Arc<Mutex<Vec<PlacedMessage>>>,
|
||||
should_fail: Arc<Mutex<bool>>,
|
||||
/// Every `since_state` argument this mock's `sync_messages` was called
|
||||
/// with, in call order, so tests can prove a caller threads the returned
|
||||
@@ -25,6 +26,10 @@ pub struct MockMailBackend {
|
||||
/// so a daemon E2E can prove the outbox executor genuinely invoked the
|
||||
/// backend op (not a fabricated completion).
|
||||
applied_mutations: Arc<Mutex<Vec<RemoteMutationSpec>>>,
|
||||
/// Occupancies this mock reports as explicitly gone, the way QRESYNC
|
||||
/// `VANISHED` does. Handed back on the pass for the folder each key names,
|
||||
/// so a test can prove a caller acts on a server-confirmed removal.
|
||||
removals: Arc<Mutex<Vec<PlacementKey>>>,
|
||||
}
|
||||
|
||||
impl MockMailBackend {
|
||||
@@ -56,6 +61,15 @@ impl MockMailBackend {
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Stage occupancies this mock reports as explicitly gone, as a QRESYNC
|
||||
/// `VANISHED` would. Each key is handed back on the pass for the folder it
|
||||
/// names, so staging a removal for `Archive` does not leak into `INBOX`.
|
||||
pub fn set_removals(&self, removals: Vec<PlacementKey>) {
|
||||
if let Ok(mut guard) = self.removals.lock() {
|
||||
*guard = removals;
|
||||
}
|
||||
}
|
||||
|
||||
/// Configure the mock to simulate network failure errors.
|
||||
pub fn set_should_fail(&self, fail: bool) {
|
||||
if let Ok(mut flag) = self.should_fail.lock() {
|
||||
@@ -70,13 +84,81 @@ impl MockMailBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/// Add a mock email message to the storage.
|
||||
pub fn add_message(&self, email: Email) {
|
||||
/// Add a mock message to the storage, as the occupancy a backend would
|
||||
/// have found it in. Taking a [`PlacedMessage`] rather than a bare `Email`
|
||||
/// lets a test stage the same message key in two folders and prove the
|
||||
/// caller treats it as one message in two places.
|
||||
pub fn add_message(&self, message: PlacedMessage) {
|
||||
if let Ok(mut guard) = self.messages.lock() {
|
||||
guard.push(email);
|
||||
guard.push(message);
|
||||
}
|
||||
}
|
||||
|
||||
/// Account every message staged by [`Self::shared_message_in`] and
|
||||
/// [`Self::with_same_message_in`] belongs to.
|
||||
pub const SHARED_ACCOUNT_ID: &'static str = "acct-1";
|
||||
|
||||
/// The one derived message key every occupancy staged by
|
||||
/// [`Self::shared_message_in`] carries, whatever folder it sits in.
|
||||
pub const SHARED_MESSAGE_KEY: &'static str = "msg-shared-identity";
|
||||
|
||||
/// One occupancy of the single shared message, in `folder_id` under
|
||||
/// `remote_id`.
|
||||
///
|
||||
/// Identity is byte-identical across every folder -- that is the whole
|
||||
/// point: a caller cannot tell a first arrival in a second mailbox from a
|
||||
/// message it already stores unless it compares occupancies rather than
|
||||
/// message keys.
|
||||
pub fn shared_message_in(folder_id: &str, remote_id: &str) -> PlacedMessage {
|
||||
PlacedMessage {
|
||||
email: Email {
|
||||
id: Self::SHARED_MESSAGE_KEY.to_string(),
|
||||
account_id: Self::SHARED_ACCOUNT_ID.to_string(),
|
||||
subject: "Shared across folders".to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
body_plain: Some("one message, several mailboxes".to_string()),
|
||||
body_html: None,
|
||||
message_id: Some("shared@nuncio.mx".to_string()),
|
||||
content_hash: None,
|
||||
attachments: Vec::new(),
|
||||
},
|
||||
// EmailId, not Surrogate: a folder-independent key is exactly what
|
||||
// a server-assigned identity buys, and it is the tier under which
|
||||
// one message legitimately reports from several mailboxes.
|
||||
source: IdentitySource::EmailId,
|
||||
placement: Placement {
|
||||
account_id: Self::SHARED_ACCOUNT_ID.to_string(),
|
||||
folder_id: folder_id.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: remote_id.to_string(),
|
||||
read: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// A backend serving `folders`, each holding the same message under its own
|
||||
/// `remote_id` -- one identity, one occupancy per folder.
|
||||
///
|
||||
/// Stages the folders as well as the messages, so a caller that enumerates
|
||||
/// folders and then syncs each one sees every occupancy.
|
||||
pub fn with_same_message_in(folders: &[&str]) -> Self {
|
||||
let backend = Self::new();
|
||||
for (index, folder_id) in folders.iter().enumerate() {
|
||||
backend.add_folder(Folder {
|
||||
id: (*folder_id).to_string(),
|
||||
name: (*folder_id).to_string(),
|
||||
total_messages: 1,
|
||||
unread_messages: 1,
|
||||
});
|
||||
// A distinct UID per folder: UIDs are folder-scoped, so the same
|
||||
// mail in two mailboxes is addressed by two different ones.
|
||||
backend.add_message(Self::shared_message_in(folder_id, &(index + 1).to_string()));
|
||||
}
|
||||
backend
|
||||
}
|
||||
|
||||
/// Every [`RemoteMutationSpec`] this mock's `apply_mutation` was called
|
||||
/// with, in call order. A failing configuration (`set_should_fail(true)`)
|
||||
/// records nothing, so a test can prove a failed op is never counted as
|
||||
@@ -112,7 +194,7 @@ impl MailBackend for MockMailBackend {
|
||||
&self,
|
||||
folder_id: &str,
|
||||
since_state: Option<&str>,
|
||||
) -> Result<crate::backend::FolderChanges, MailError> {
|
||||
) -> Result<FolderChanges, MailError> {
|
||||
// Record the checkpoint arg before any early return, so even a failing
|
||||
// call is visible to a test asserting on how the caller resumes.
|
||||
if let Ok(mut calls) = self.since_state_calls.lock() {
|
||||
@@ -134,11 +216,25 @@ impl MailBackend for MockMailBackend {
|
||||
.map_err(|e| MailError::ParseFailed(e.to_string()))?
|
||||
.clone();
|
||||
|
||||
// Staged removals are scoped to the folder each key names, and are
|
||||
// reported on both the full and the incremental pass -- a `VANISHED`
|
||||
// report is exactly the kind of thing an incremental resync carries.
|
||||
let removals: Vec<PlacementKey> = self
|
||||
.removals
|
||||
.lock()
|
||||
.map_err(|e| MailError::ParseFailed(e.to_string()))?
|
||||
.iter()
|
||||
.filter(|key| key.folder_id == folder_id)
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
// Simulate a since_state-aware backend: with a prior checkpoint there
|
||||
// is nothing new to hand back, so the incremental fetch is genuinely
|
||||
// narrower (empty) rather than a full re-report of every message.
|
||||
// `present` stays `None`: an incremental pass cannot speak to absence.
|
||||
if since_state.is_some() {
|
||||
return Ok(crate::backend::FolderChanges {
|
||||
return Ok(FolderChanges {
|
||||
removals,
|
||||
next_state: returned_state,
|
||||
..Default::default()
|
||||
});
|
||||
@@ -150,15 +246,16 @@ impl MailBackend for MockMailBackend {
|
||||
.map_err(|e| MailError::ParseFailed(e.to_string()))?;
|
||||
let matches: Vec<_> = messages
|
||||
.iter()
|
||||
.filter(|m| m.folder_id == folder_id)
|
||||
.filter(|m| m.placement.folder_id == folder_id)
|
||||
.cloned()
|
||||
.collect();
|
||||
// A full pass over the mock's store did see everything in the folder,
|
||||
// so it can honestly report what is present.
|
||||
let present = matches.iter().map(|m| m.id.clone()).collect();
|
||||
Ok(crate::backend::FolderChanges {
|
||||
// so it can honestly report what is present -- as occupancies, since
|
||||
// that is what a folder can speak to.
|
||||
let present = matches.iter().map(|m| m.placement.key()).collect();
|
||||
Ok(FolderChanges {
|
||||
upserts: matches,
|
||||
removals: Vec::new(),
|
||||
removals,
|
||||
present: Some(present),
|
||||
next_state: returned_state,
|
||||
})
|
||||
@@ -256,22 +353,30 @@ mod tests {
|
||||
unread_messages: 1,
|
||||
});
|
||||
|
||||
let email = Email {
|
||||
id: "msg-mock-1".to_string(),
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
remote_id: "1".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: "Mock Test".to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1700000000,
|
||||
read: false,
|
||||
body_plain: Some("Mock body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
let placed = PlacedMessage {
|
||||
email: Email {
|
||||
id: "msg-mock-1".to_string(),
|
||||
account_id: "acct-1".to_string(),
|
||||
subject: "Mock Test".to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1700000000,
|
||||
body_plain: Some("Mock body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
source: IdentitySource::Surrogate,
|
||||
placement: Placement {
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: "1".to_string(),
|
||||
read: false,
|
||||
},
|
||||
};
|
||||
mock.add_message(email.clone());
|
||||
mock.add_message(placed.clone());
|
||||
|
||||
let folders = mock.sync_folders().await.expect("sync folders succeeds");
|
||||
assert_eq!(folders.len(), 1);
|
||||
@@ -281,6 +386,10 @@ mod tests {
|
||||
.await
|
||||
.expect("sync messages succeeds");
|
||||
assert_eq!(msgs.len(), 1);
|
||||
assert_eq!(
|
||||
msgs[0], placed,
|
||||
"the mock must hand back exactly the placement it was staged with"
|
||||
);
|
||||
assert_eq!(state, "mock-state-token-100");
|
||||
|
||||
// Test error simulation
|
||||
@@ -289,6 +398,39 @@ mod tests {
|
||||
assert!(mock.sync_messages("inbox", None).await.is_err());
|
||||
}
|
||||
|
||||
/// The staging helper is only useful if it really produces one identity in
|
||||
/// two places: same message key, different occupancies. If it ever drifted
|
||||
/// into two keys, every test built on it would pass for the wrong reason.
|
||||
#[tokio::test]
|
||||
async fn with_same_message_in_stages_one_identity_across_every_folder() {
|
||||
let mock = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]);
|
||||
|
||||
let folders = mock.sync_folders().await.expect("sync folders succeeds");
|
||||
let folder_ids: Vec<&str> = folders.iter().map(|f| f.id.as_str()).collect();
|
||||
assert_eq!(folder_ids, vec!["INBOX", "Archive"]);
|
||||
|
||||
let inbox = mock
|
||||
.sync_changes("INBOX", None)
|
||||
.await
|
||||
.expect("inbox pass succeeds");
|
||||
let archive = mock
|
||||
.sync_changes("Archive", None)
|
||||
.await
|
||||
.expect("archive pass succeeds");
|
||||
assert_eq!(inbox.upserts.len(), 1);
|
||||
assert_eq!(archive.upserts.len(), 1);
|
||||
|
||||
assert_eq!(
|
||||
inbox.upserts[0].email.id, archive.upserts[0].email.id,
|
||||
"both occupancies must carry the same derived message key"
|
||||
);
|
||||
assert_ne!(
|
||||
inbox.upserts[0].placement.key(),
|
||||
archive.upserts[0].placement.key(),
|
||||
"the occupancies themselves must differ, or there is nothing to classify"
|
||||
);
|
||||
}
|
||||
|
||||
fn sample_outbound_message() -> OutboundMessage {
|
||||
OutboundMessage {
|
||||
from: "alice@nuncio.mx".to_string(),
|
||||
|
||||
@@ -61,12 +61,15 @@ impl MimeParserAdapter {
|
||||
pub const MAX_PAYLOAD_BYTES: usize = 25 * 1024 * 1024;
|
||||
|
||||
/// Parse a raw RFC 5322 byte slice into an [`Email`] domain entity.
|
||||
pub fn parse_mime(
|
||||
id: &str,
|
||||
account_id: &str,
|
||||
folder_id: &str,
|
||||
raw_bytes: &[u8],
|
||||
) -> Result<Email, MailError> {
|
||||
///
|
||||
/// The returned `Email` carries an **empty** `id`, and it is the caller's
|
||||
/// job to fill it from [`Email::derive_message_key`]. The derivation needs
|
||||
/// inputs this parser cannot see -- the transport's addressing coordinates
|
||||
/// and any server-assigned identity (RFC 8474 `EMAILID`, `X-GM-MSGID`) --
|
||||
/// alongside the `Message-ID` and content hash recovered here, so the id
|
||||
/// can only be settled once both halves are in hand. Minting a provisional
|
||||
/// value here would risk one reaching the store as though it were derived.
|
||||
pub fn parse_mime(account_id: &str, raw_bytes: &[u8]) -> Result<Email, MailError> {
|
||||
if raw_bytes.len() > Self::MAX_PAYLOAD_BYTES {
|
||||
return Err(MailError::ParseFailed(
|
||||
"MIME payload exceeds maximum allowed limit of 25MB".to_string(),
|
||||
@@ -95,6 +98,9 @@ impl MimeParserAdapter {
|
||||
|
||||
let received_at = msg.date().map_or(0, |d| d.to_timestamp());
|
||||
|
||||
let message_id = msg.message_id().and_then(Email::normalize_message_id);
|
||||
let content_hash = Some(Email::content_hash_of(raw_bytes));
|
||||
|
||||
let body_plain = msg.body_text(0).map(|b| b.to_string());
|
||||
let body_html = msg.body_html(0).map(|b| b.to_string());
|
||||
|
||||
@@ -126,21 +132,19 @@ impl MimeParserAdapter {
|
||||
}
|
||||
|
||||
Ok(Email {
|
||||
id: id.to_string(),
|
||||
// Assigned by the caller from the derived message key; see the
|
||||
// method doc for why it cannot be settled here.
|
||||
id: String::new(),
|
||||
account_id: account_id.to_string(),
|
||||
folder_id: folder_id.to_string(),
|
||||
// The MIME body carries no protocol addressing; the caller (the
|
||||
// IMAP sync) fills these from the transport before persisting.
|
||||
remote_id: String::new(),
|
||||
uid_validity: String::new(),
|
||||
subject,
|
||||
sender,
|
||||
recipient,
|
||||
received_at,
|
||||
read: false,
|
||||
body_plain,
|
||||
body_html,
|
||||
attachments,
|
||||
message_id,
|
||||
content_hash,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -158,9 +162,12 @@ mod tests {
|
||||
\r\n\
|
||||
Hello Bob, this is a plain text email.";
|
||||
|
||||
let email =
|
||||
MimeParserAdapter::parse_mime("msg-1", "acct-1", "inbox", raw).expect("parse succeeds");
|
||||
assert_eq!(email.id, "msg-1");
|
||||
let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds");
|
||||
assert_eq!(
|
||||
email.id, "",
|
||||
"the parser must not mint an id; the caller derives it"
|
||||
);
|
||||
assert_eq!(email.account_id, "acct-1");
|
||||
assert_eq!(email.subject, "Test Subject");
|
||||
assert_eq!(email.sender, "alice@nuncio.mx");
|
||||
assert_eq!(email.recipient, "bob@nuncio.mx");
|
||||
@@ -170,11 +177,41 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_captures_normalized_message_id_and_content_hash() {
|
||||
let raw = b"From: Alice <alice@nuncio.mx>\r\n\
|
||||
To: Bob <bob@nuncio.mx>\r\n\
|
||||
Message-ID: <Abc.123@mail.nuncio.mx>\r\n\
|
||||
Subject: Test\r\n\
|
||||
\r\n\
|
||||
body";
|
||||
|
||||
let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds");
|
||||
assert_eq!(email.message_id.as_deref(), Some("Abc.123@mail.nuncio.mx"));
|
||||
assert_eq!(
|
||||
email.content_hash,
|
||||
Some(nuncio_core::model::Email::content_hash_of(raw)),
|
||||
"the hash must cover the octets actually parsed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_reports_no_message_id_when_the_header_is_absent() {
|
||||
// RFC 5322 3.6.4 makes Message-ID a SHOULD. An absent header must read
|
||||
// as "none present", never as an invented or empty value.
|
||||
let raw = b"From: Alice <alice@nuncio.mx>\r\n\
|
||||
Subject: No id\r\n\
|
||||
\r\n\
|
||||
body";
|
||||
let email = MimeParserAdapter::parse_mime("acct-1", raw).expect("parse succeeds");
|
||||
assert_eq!(email.message_id, None);
|
||||
assert!(email.content_hash.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_invalid_mime_returns_error() {
|
||||
let raw = b"";
|
||||
let err = MimeParserAdapter::parse_mime("msg-2", "acct-1", "inbox", raw)
|
||||
.expect_err("should fail");
|
||||
let err = MimeParserAdapter::parse_mime("acct-1", raw).expect_err("should fail");
|
||||
assert_eq!(
|
||||
err,
|
||||
MailError::ParseFailed("invalid RFC 5322 MIME structure".to_string())
|
||||
@@ -188,7 +225,7 @@ mod tests {
|
||||
#[test]
|
||||
fn parse_oversized_mime_returns_error() {
|
||||
let oversized = vec![0u8; MimeParserAdapter::MAX_PAYLOAD_BYTES + 1];
|
||||
let err = MimeParserAdapter::parse_mime("msg-huge", "acct-1", "inbox", &oversized)
|
||||
let err = MimeParserAdapter::parse_mime("acct-1", &oversized)
|
||||
.expect_err("should fail for oversized payload");
|
||||
assert!(err
|
||||
.to_string()
|
||||
|
||||
@@ -357,17 +357,15 @@ mod tests {
|
||||
Email {
|
||||
id: "msg-1".to_string(),
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "outbox".to_string(),
|
||||
remote_id: "1".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: "Status Update".to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1700000000,
|
||||
read: true,
|
||||
body_plain: Some("Plaintext status update".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
|
||||
use nuncio_core::model::PlacementKey;
|
||||
use nuncio_mail::imap_raw;
|
||||
use nuncio_mail::test_server::{MockImapServer, ServerProfile};
|
||||
|
||||
@@ -345,14 +346,17 @@ async fn every_rung_reports_a_message_that_disappeared() {
|
||||
let second = sync_folder(&server, Some(&first.next_state)).await;
|
||||
|
||||
// However the rung learned it, the caller must end up able to identify
|
||||
// the removed message and only that one.
|
||||
let mut reported: Vec<String> = second.removals.clone();
|
||||
// the occupancy that went and only that one. Absence is reported per
|
||||
// mailbox, so what is compared is the placement each upsert was found
|
||||
// under, not the message key -- the same message may still sit
|
||||
// elsewhere and must not be swept up by this folder's silence.
|
||||
let mut reported: Vec<PlacementKey> = second.removals.clone();
|
||||
if let Some(present) = &second.present {
|
||||
let stored_gone = first
|
||||
.upserts
|
||||
.iter()
|
||||
.map(|e| e.id.clone())
|
||||
.filter(|id| !present.contains(id));
|
||||
.map(|e| e.placement.key())
|
||||
.filter(|key| !present.contains(key));
|
||||
reported.extend(stored_gone);
|
||||
}
|
||||
reported.sort();
|
||||
@@ -366,12 +370,17 @@ async fn every_rung_reports_a_message_that_disappeared() {
|
||||
let surviving = first
|
||||
.upserts
|
||||
.iter()
|
||||
.find(|e| e.remote_id == keep.to_string())
|
||||
.find(|e| e.placement.remote_id == keep.to_string())
|
||||
.expect("the kept message was in the first pass");
|
||||
assert!(
|
||||
!reported.contains(&surviving.id),
|
||||
!reported.contains(&surviving.placement.key()),
|
||||
"{profile:?}: the surviving message must not be reported gone"
|
||||
);
|
||||
assert_eq!(
|
||||
reported[0].remote_id,
|
||||
gone.to_string(),
|
||||
"{profile:?}: the reported occupancy must be the expunged UID"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -474,9 +483,17 @@ async fn a_uidvalidity_change_refetches_rather_than_deleting_the_folder() {
|
||||
second.removals.is_empty(),
|
||||
"a renumbering is not a removal report"
|
||||
);
|
||||
// The re-fetched message has a different surrogate, because UIDVALIDITY is
|
||||
// part of identity -- so the caller replaces rather than merges.
|
||||
assert_ne!(second.upserts[0].id, first.upserts[0].id);
|
||||
// The re-fetched message lands under a new occupancy, because UIDVALIDITY
|
||||
// is part of a placement key -- so the caller writes a new placement rather
|
||||
// than updating the old one, and the stale UID never addresses live mail.
|
||||
assert_ne!(
|
||||
second.upserts[0].placement.uid_validity,
|
||||
first.upserts[0].placement.uid_validity
|
||||
);
|
||||
assert_ne!(second.upserts[0].placement, first.upserts[0].placement);
|
||||
// These two are also genuinely different messages, so their derived keys
|
||||
// differ as well.
|
||||
assert_ne!(second.upserts[0].email.id, first.upserts[0].email.id);
|
||||
}
|
||||
|
||||
/// Build an engine pointed at the mock and apply one mutation through the
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
|
||||
use nuncio_core::model::IdentitySource;
|
||||
use nuncio_mail::{JmapEngine, MailBackend, RemoteMutationKind, RemoteMutationSpec};
|
||||
use serde_json::json;
|
||||
use wiremock::matchers::{body_partial_json, method, path};
|
||||
@@ -84,13 +85,16 @@ async fn wiremock_jmap_session_discovery_and_email_get_sync() {
|
||||
.expect("parse email get");
|
||||
|
||||
assert_eq!(emails.len(), 1);
|
||||
// The JMAP object id is preserved as the protocol-native `remote_id`, while
|
||||
// the persisted id is an opaque 64-char surrogate hash (never the raw id).
|
||||
assert_eq!(emails[0].remote_id, "msg-wm-100");
|
||||
assert_ne!(emails[0].id, "msg-wm-100");
|
||||
assert_eq!(emails[0].id.len(), 64);
|
||||
assert_eq!(emails[0].subject, "WireMock JMAP E2E Test");
|
||||
assert_eq!(emails[0].sender, "sender@nuncio.mx");
|
||||
// The JMAP object id is preserved as the placement's protocol-native
|
||||
// `remote_id`, while the message's own id is an opaque 64-char derived key
|
||||
// (never the raw id). The object id is account-stable, so it enters the
|
||||
// precedence at the EMAILID tier rather than as a folder-scoped surrogate.
|
||||
assert_eq!(emails[0].placement.remote_id, "msg-wm-100");
|
||||
assert_eq!(emails[0].source, IdentitySource::EmailId);
|
||||
assert_ne!(emails[0].email.id, "msg-wm-100");
|
||||
assert_eq!(emails[0].email.id.len(), 64);
|
||||
assert_eq!(emails[0].email.subject, "WireMock JMAP E2E Test");
|
||||
assert_eq!(emails[0].email.sender, "sender@nuncio.mx");
|
||||
assert_eq!(new_state, "sync-state-500");
|
||||
}
|
||||
|
||||
@@ -204,15 +208,18 @@ async fn jmap_engine_sync_folders_and_messages_through_mail_backend_trait() {
|
||||
.await
|
||||
.expect("sync_messages succeeds");
|
||||
assert_eq!(emails.len(), 1);
|
||||
assert_eq!(emails[0].remote_id, "msg-wm-200");
|
||||
assert_ne!(emails[0].id, "msg-wm-200");
|
||||
assert_eq!(emails[0].id.len(), 64);
|
||||
assert_eq!(emails[0].subject, "Trait-Level JMAP Sync");
|
||||
assert_eq!(emails[0].sender, "sender@nuncio.mx");
|
||||
assert_eq!(emails[0].received_at, 1700002000);
|
||||
assert!(!emails[0].read);
|
||||
assert_eq!(emails[0].placement.remote_id, "msg-wm-200");
|
||||
assert_eq!(emails[0].placement.folder_id, "inbox");
|
||||
assert_eq!(emails[0].source, IdentitySource::EmailId);
|
||||
assert_ne!(emails[0].email.id, "msg-wm-200");
|
||||
assert_eq!(emails[0].email.id.len(), 64);
|
||||
assert_eq!(emails[0].email.subject, "Trait-Level JMAP Sync");
|
||||
assert_eq!(emails[0].email.sender, "sender@nuncio.mx");
|
||||
assert_eq!(emails[0].email.received_at, 1700002000);
|
||||
// Read state is a property of the occupancy, not of the message.
|
||||
assert!(!emails[0].placement.read);
|
||||
assert_eq!(
|
||||
emails[0].body_plain,
|
||||
emails[0].email.body_plain,
|
||||
Some("Fetched through the MailBackend trait.".to_string())
|
||||
);
|
||||
assert_eq!(new_state, "sync-state-900");
|
||||
|
||||
@@ -519,10 +519,19 @@ message Attachment {
|
||||
// (rather than `Email`) to avoid colliding with common generated-code
|
||||
// identifiers in hand-written Rust using this module.
|
||||
message Message {
|
||||
// Unique message identifier. OPAQUE and server-assigned: a deterministic
|
||||
// surrogate derived from the message's stable protocol identity, never a
|
||||
// raw upstream protocol id (e.g. an IMAP UID, which is only stable within
|
||||
// a single IMAP session and is NOT a durable cross-session identifier).
|
||||
// Unique message identifier. OPAQUE and server-assigned: content-addressed
|
||||
// from the strongest identity the message's server offered. Never a raw
|
||||
// upstream protocol id (e.g. an IMAP UID, which is only stable within a
|
||||
// single IMAP session and is NOT a durable cross-session identifier).
|
||||
//
|
||||
// Stability across a move depends on what the server offered. Where a
|
||||
// server-assigned object id or a usable Message-ID was available, the id is
|
||||
// folder-independent and survives a move between folders. Where none was --
|
||||
// a message carrying no Message-ID, or one fetched headers-only -- the id
|
||||
// falls back to a derivation that includes the folder, and moving such a
|
||||
// message yields a different id. Clients MUST NOT assume an id outlives a
|
||||
// move.
|
||||
//
|
||||
// Clients MUST treat this as an opaque token -- pass it back verbatim in
|
||||
// later requests, but never parse, construct, or assume any structure in
|
||||
// its value.
|
||||
@@ -530,6 +539,11 @@ message Message {
|
||||
// Account identifier owning the message.
|
||||
string account_id = 2;
|
||||
// Mailbox folder identifier (e.g. "inbox").
|
||||
//
|
||||
// A message can occupy several folders at once. This names the one the
|
||||
// message was reached through: the requested folder for a listing, and the
|
||||
// lowest-sorting folder id for a lookup by id. A field carrying the complete
|
||||
// placement set is not yet part of this contract.
|
||||
string folder_id = 3;
|
||||
// Subject line.
|
||||
string subject = 4;
|
||||
@@ -777,6 +791,13 @@ service Filters {
|
||||
// applied retroactively to mail that was synced before those rules
|
||||
// existed. Streams cumulative progress as it works, ending with a final
|
||||
// `done: true` message once the whole store has been scanned.
|
||||
//
|
||||
// Each (rule, message) pair fires at most once, ever -- the same claim
|
||||
// ledger live sync uses, keyed by rule id and message id. This is not a
|
||||
// repeatable rescan: running Triage again with the same rules finds
|
||||
// nothing left to claim and reports `matched_count: 0`. Only messages
|
||||
// stored since the last run, or rules created since then, produce further
|
||||
// matches; editing an existing rule's contents does not reset its claims.
|
||||
rpc Triage(TriageRequest) returns (stream TriageProgress);
|
||||
}
|
||||
|
||||
|
||||
+1942
-592
File diff suppressed because it is too large.
Load diff
@@ -6,7 +6,10 @@ use serde::{Deserialize, Serialize};
|
||||
/// Search hit result for full-text query matches.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SearchHit {
|
||||
/// Entity ID matching the query.
|
||||
/// Entity ID matching the query. For a message hit this is the
|
||||
/// folder-independent message key: the index holds one row per message, so
|
||||
/// a hit names a message and never one of its mailbox placements. Resolve
|
||||
/// it through [`DatabaseEngine::get_message`].
|
||||
pub id: String,
|
||||
/// Matching title or subject.
|
||||
pub title: String,
|
||||
@@ -50,7 +53,7 @@ impl<'a> SearchEngine<'a> {
|
||||
|
||||
let rows: Vec<(String, String, String)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT id, subject, snippet(messages_fts, 3, '<b>', '</b>', '...', 10) as snippet
|
||||
SELECT message_key, subject, snippet(messages_fts, 3, '<b>', '</b>', '...', 10) as snippet
|
||||
FROM messages_fts
|
||||
WHERE messages_fts MATCH ?
|
||||
ORDER BY rank
|
||||
@@ -94,7 +97,7 @@ impl<'a> SearchEngine<'a> {
|
||||
|
||||
let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new(
|
||||
"SELECT id, title, snippet, rk FROM ( \
|
||||
SELECT id, subject AS title, \
|
||||
SELECT message_key AS id, subject AS title, \
|
||||
snippet(messages_fts, 3, '<b>', '</b>', '...', 10) AS snippet, rank AS rk \
|
||||
FROM messages_fts WHERE messages_fts MATCH ",
|
||||
);
|
||||
@@ -255,21 +258,39 @@ mod tests {
|
||||
nuncio_core::model::Email {
|
||||
id: id.to_string(),
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
remote_id: id.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: subject.to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
read: false,
|
||||
body_plain: Some(body_plain.to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The crux test: proves body search works over the REAL `save_email` write path (no
|
||||
fn sample_placement(uid: &str) -> nuncio_core::model::Placement {
|
||||
nuncio_core::model::Placement {
|
||||
account_id: "acct-1".to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: uid.to_string(),
|
||||
read: false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn save(db: &DatabaseEngine, email: &nuncio_core::model::Email) {
|
||||
db.save_email_at(
|
||||
email,
|
||||
nuncio_core::model::IdentitySource::Surrogate,
|
||||
&sample_placement(&email.id),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// The crux test: proves body search works over the REAL `save_email_at` write path (no
|
||||
/// raw-SQL plaintext injection bypassing encryption), while the stored body column stays
|
||||
/// ciphertext. This is what the old `fts5_message_search_and_triggers` test failed to
|
||||
/// prove -- it inserted plaintext directly via raw SQL, which trivially "worked" but never
|
||||
@@ -288,7 +309,7 @@ mod tests {
|
||||
"Quarterly Financial Meeting",
|
||||
"Let's discuss the annual budget revenue forecast",
|
||||
);
|
||||
db.save_email(&email).await.unwrap();
|
||||
save(&db, &email).await;
|
||||
|
||||
// Body search matches a plaintext body term -- proves the FTS index holds real,
|
||||
// decrypted-at-write-time plaintext trigrams, not the AES-256-GCM ciphertext that is
|
||||
@@ -305,39 +326,44 @@ mod tests {
|
||||
// The body column at rest must remain encrypted ciphertext -- never the plaintext
|
||||
// search term, and never equal to the original plaintext body.
|
||||
let (stored_body,): (String,) =
|
||||
sqlx::query_as("SELECT body_plain FROM messages WHERE id = 'msg-1'")
|
||||
sqlx::query_as("SELECT body_plain FROM messages WHERE message_key = 'msg-1'")
|
||||
.fetch_one(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_ne!(stored_body, email.body_plain.clone().unwrap());
|
||||
assert!(!stored_body.to_lowercase().contains("revenue"));
|
||||
|
||||
// Re-saving via save_email (an update, not a fresh insert) replaces the FTS row rather
|
||||
// than duplicating or leaving a stale entry: the old term no longer matches, the new
|
||||
// one does.
|
||||
let mut updated = email.clone();
|
||||
updated.body_plain = Some("Updated strategy review notes".to_string());
|
||||
db.save_email(&updated).await.unwrap();
|
||||
// Re-saving the same key with a different body must leave the index alone: the message
|
||||
// row is first-write-wins, so a second write that cannot change the stored body must
|
||||
// not change what search returns for it either. Anything else would have search
|
||||
// disagree with the store -- either finding a body that was never persisted, or holding
|
||||
// two rows for one message.
|
||||
let mut rewritten = email.clone();
|
||||
rewritten.body_plain = Some("Updated strategy review notes".to_string());
|
||||
save(&db, &rewritten).await;
|
||||
|
||||
assert!(search.search_messages("revenue").await.unwrap().is_empty());
|
||||
let new_hits = search.search_messages("strategy").await.unwrap();
|
||||
assert_eq!(new_hits.len(), 1);
|
||||
assert_eq!(new_hits[0].id, "msg-1");
|
||||
let unchanged = search.search_messages("revenue").await.unwrap();
|
||||
assert_eq!(unchanged.len(), 1, "the stored body is still searchable");
|
||||
assert_eq!(unchanged[0].id, "msg-1");
|
||||
assert!(
|
||||
search.search_messages("strategy").await.unwrap().is_empty(),
|
||||
"a body the store rejected must never become searchable"
|
||||
);
|
||||
|
||||
// Delete message trigger still cleans up the FTS row (subject/sender/id only -- no
|
||||
// Delete message trigger still cleans up the FTS row (subject/sender/key only -- no
|
||||
// ciphertext involved).
|
||||
sqlx::query("DELETE FROM messages WHERE id = 'msg-1'")
|
||||
sqlx::query("DELETE FROM messages WHERE message_key = 'msg-1'")
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let deleted_hits = search.search_messages("strategy").await.unwrap();
|
||||
let deleted_hits = search.search_messages("revenue").await.unwrap();
|
||||
assert!(deleted_hits.is_empty());
|
||||
}
|
||||
|
||||
/// Proves the backfill half of the fix: a message saved through the real write path,
|
||||
/// whose FTS entry is then lost (simulating either a row written before the FTS5 index
|
||||
/// existed, or one written by a process that bypassed `save_email`'s explicit indexing),
|
||||
/// existed, or one written by a process that bypassed `save_email_at`'s explicit indexing),
|
||||
/// is NOT searchable until a migration pass runs -- at which point `backfill_message_fts`
|
||||
/// decrypts the ciphertext body column with this engine's real storage key and repopulates
|
||||
/// the plaintext-derived trigram index, with no search call ever required to trigger it.
|
||||
@@ -349,11 +375,11 @@ mod tests {
|
||||
"Legacy Roadmap Notes",
|
||||
"Confidential quarterly roadmap details",
|
||||
);
|
||||
db.save_email(&email).await.unwrap();
|
||||
save(&db, &email).await;
|
||||
|
||||
// Simulate pre-existing data whose FTS entry is missing (e.g. written before the FTS5
|
||||
// index existed) by dropping its messages_fts row directly.
|
||||
sqlx::query("DELETE FROM messages_fts WHERE id = 'msg-legacy-1'")
|
||||
sqlx::query("DELETE FROM messages_fts WHERE message_key = 'msg-legacy-1'")
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -383,13 +409,15 @@ mod tests {
|
||||
async fn search_messages_page_returns_every_hit_once_no_gaps() {
|
||||
let (db, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
|
||||
for i in 0..5 {
|
||||
db.save_email(&sample_email(
|
||||
&format!("msg-{i}"),
|
||||
"Budget Planning Meeting",
|
||||
"annual budget revenue forecast discussion",
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
save(
|
||||
&db,
|
||||
&sample_email(
|
||||
&format!("msg-{i}"),
|
||||
"Budget Planning Meeting",
|
||||
"annual budget revenue forecast discussion",
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let search = SearchEngine::new(&db);
|
||||
|
||||
+753
-164
File diff suppressed because it is too large.
Load diff
+191
-18
@@ -289,7 +289,7 @@ async fn execute_one(
|
||||
|
||||
match mutation.mutation_type.as_str() {
|
||||
"MOVE" | "COPY" | "FLAG" | "UNFLAG" | "DELETE" => {
|
||||
dispatch_mailbox_mutation(env, mutation, &email, &payload).await
|
||||
dispatch_mailbox_mutation(db, env, mutation, &email, &payload).await
|
||||
}
|
||||
"FORWARD" => dispatch_forward(db, env, &email, &payload).await,
|
||||
"WEBHOOK" => dispatch_webhook(env, mutation, &email, &payload).await,
|
||||
@@ -298,10 +298,30 @@ async fn execute_one(
|
||||
}
|
||||
|
||||
/// Execute a mailbox-affecting mutation (move/copy/flag/unflag/delete) against
|
||||
/// the account's mail backend, recovering the remote addressing (protocol id +
|
||||
/// UIDVALIDITY scope) from the stored message row so the backend can enforce
|
||||
/// its UIDVALIDITY guard.
|
||||
/// the account's mail backend, recovering the remote addressing (folder,
|
||||
/// protocol id, UIDVALIDITY scope) from the message's stored placement so the
|
||||
/// backend can enforce its UIDVALIDITY guard.
|
||||
///
|
||||
/// The addressing comes from a placement rather than the message row, because
|
||||
/// those coordinates are properties of a mailbox occupancy and a message can
|
||||
/// hold several. The outbox row records only the message key, so which occupancy
|
||||
/// the rule matched is not recoverable here -- and when the message occupies
|
||||
/// more than one mailbox, this **refuses** rather than picking one.
|
||||
///
|
||||
/// Guessing is not the conservative option. A rule that matched the `INBOX`
|
||||
/// copy would `MOVE` or `DELETE` whichever copy happened to sort first, against
|
||||
/// mail the user never targeted and with no undo; Gmail labels make
|
||||
/// multi-placement ordinary rather than exotic. Unresolvable *addressing* is no
|
||||
/// more a licence to act than an unverifiable claim is in
|
||||
/// [`crate::sync::apply_filter_actions`]: refusing is inert and visible,
|
||||
/// guessing is destructive and irreversible.
|
||||
///
|
||||
/// The refusal is permanent, not retryable: nothing about a later attempt makes
|
||||
/// the outbox row remember a placement it never recorded. Carrying the
|
||||
/// originating placement on the row is the real fix, and belongs with the
|
||||
/// placement-aware wire surface.
|
||||
async fn dispatch_mailbox_mutation(
|
||||
db: &DatabaseEngine,
|
||||
env: &dyn RemoteExecutionEnv,
|
||||
mutation: &PendingRemoteMutation,
|
||||
email: &nuncio_core::model::Email,
|
||||
@@ -330,20 +350,56 @@ async fn dispatch_mailbox_mutation(
|
||||
other => return Disposition::Permanent(format!("unexpected mailbox mutation '{other}'")),
|
||||
};
|
||||
|
||||
let backend = match env.mail_backend(&email.account_id).await {
|
||||
// Recover the remote addressing from a stored placement rather than
|
||||
// parsing it out of the opaque message key. The backend still
|
||||
// SELECT-verifies the UIDVALIDITY before mutating.
|
||||
let placements = match db.placements_of(&email.id).await {
|
||||
Ok(placements) => placements,
|
||||
Err(e) => {
|
||||
return Disposition::Retry(format!("failed to resolve the message's placements: {e}"))
|
||||
}
|
||||
};
|
||||
let placement = match placements.as_slice() {
|
||||
[only] => only,
|
||||
[] => {
|
||||
// No occupancy means no way to address the message on any server,
|
||||
// and no retry can conjure one back.
|
||||
return Disposition::Permanent(format!(
|
||||
"message '{}' occupies no mailbox to address",
|
||||
email.id
|
||||
));
|
||||
}
|
||||
several => {
|
||||
let folders: Vec<&str> = several.iter().map(|p| p.folder_id.as_str()).collect();
|
||||
tracing::warn!(
|
||||
mutation_id = %mutation.id,
|
||||
op = %mutation.mutation_type,
|
||||
message_id = %email.id,
|
||||
placement_count = several.len(),
|
||||
"outbox: refusing an ambiguously-addressed mutation"
|
||||
);
|
||||
return Disposition::Permanent(format!(
|
||||
"message '{}' occupies {} mailboxes ({}) and the outbox row does not record \
|
||||
which one rule '{}' matched; refusing to guess a {} target",
|
||||
email.id,
|
||||
several.len(),
|
||||
folders.join(", "),
|
||||
mutation.rule_id,
|
||||
mutation.mutation_type
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
let backend = match env.mail_backend(&placement.account_id).await {
|
||||
Ok(backend) => backend,
|
||||
Err(e) => return Disposition::Retry(format!("failed to build mail backend: {e}")),
|
||||
};
|
||||
|
||||
// Recover the remote addressing from the stored message row -- the
|
||||
// protocol-native id and the UIDVALIDITY scope it was captured under --
|
||||
// rather than parsing it out of the opaque surrogate id. The backend still
|
||||
// SELECT-verifies the UIDVALIDITY before mutating.
|
||||
let spec = RemoteMutationSpec {
|
||||
message_id: email.id.clone(),
|
||||
remote_id: email.remote_id.clone(),
|
||||
folder_id: email.folder_id.clone(),
|
||||
uid_validity: email.uid_validity.clone(),
|
||||
remote_id: placement.remote_id.clone(),
|
||||
folder_id: placement.folder_id.clone(),
|
||||
uid_validity: placement.uid_validity.clone(),
|
||||
kind,
|
||||
};
|
||||
|
||||
@@ -580,7 +636,7 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::lifecycle::{ShutdownController, ShutdownSignal};
|
||||
use crate::test_tracing::with_recorder;
|
||||
use nuncio_core::model::Email;
|
||||
use nuncio_core::model::{Email, IdentitySource, Placement};
|
||||
use nuncio_core::EventBus;
|
||||
use nuncio_filter::MutationPayload;
|
||||
use nuncio_mail::MockMailBackend;
|
||||
@@ -634,17 +690,27 @@ mod tests {
|
||||
Email {
|
||||
id: "msg-outbox-log".to_string(),
|
||||
account_id: "acct-outbox-log".to_string(),
|
||||
folder_id: "INBOX".to_string(),
|
||||
remote_id: "77".to_string(),
|
||||
uid_validity: "9".to_string(),
|
||||
subject: "Confidential outbox subject".to_string(),
|
||||
sender: "alice@example.com".to_string(),
|
||||
recipient: "owner@nuncio.mx".to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
read: false,
|
||||
body_plain: Some("secret outbox body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The one mailbox occupancy `sample_email` is stored in -- the addressing
|
||||
/// the outbox recovers to build its `RemoteMutationSpec`.
|
||||
fn sample_placement() -> Placement {
|
||||
Placement {
|
||||
account_id: "acct-outbox-log".to_string(),
|
||||
folder_id: "INBOX".to_string(),
|
||||
uid_validity: "9".to_string(),
|
||||
remote_id: "77".to_string(),
|
||||
read: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -683,7 +749,13 @@ mod tests {
|
||||
db.save_folder_sync_state("acct-outbox-log", "INBOX", "9:100")
|
||||
.await
|
||||
.expect("save folder checkpoint");
|
||||
db.save_email(&sample_email()).await.expect("save email");
|
||||
db.save_email_at(
|
||||
&sample_email(),
|
||||
IdentitySource::Surrogate,
|
||||
&sample_placement(),
|
||||
)
|
||||
.await
|
||||
.expect("save email");
|
||||
db.save_pending_mutation(&flag_mutation())
|
||||
.await
|
||||
.expect("save pending mutation");
|
||||
@@ -727,4 +799,105 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A second occupancy of `sample_email`, in a different mailbox -- an
|
||||
/// ordinary Gmail label, or a `COPY` another client made.
|
||||
fn second_placement() -> Placement {
|
||||
Placement {
|
||||
account_id: "acct-outbox-log".to_string(),
|
||||
folder_id: "Archive".to_string(),
|
||||
uid_validity: "9".to_string(),
|
||||
remote_id: "78".to_string(),
|
||||
read: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed the message into every given occupancy and dispatch one FLAG
|
||||
/// mutation against it. Returns the honest disposition and every spec the
|
||||
/// backend was actually asked to apply.
|
||||
///
|
||||
/// Calls [`dispatch_mailbox_mutation`] rather than draining through
|
||||
/// [`execute_pending_mutations`] deliberately. The drain's own
|
||||
/// "attempting"/"completed" `tracing` callsites are what
|
||||
/// `drain_logs_attempt_and_completion_without_leaking_message_content`
|
||||
/// asserts on, and `tracing` caches callsite interest globally: a
|
||||
/// concurrent test hitting those callsites with no recorder installed
|
||||
/// caches them as uninteresting and makes that test fail. Addressing is
|
||||
/// decided here anyway, so the narrower call is also the tighter assertion.
|
||||
async fn dispatch_with_placements(
|
||||
placements: &[Placement],
|
||||
) -> (Disposition, Vec<RemoteMutationSpec>) {
|
||||
let (db, _dir) = DatabaseEngine::connect_ephemeral()
|
||||
.await
|
||||
.expect("ephemeral db");
|
||||
for placement in placements {
|
||||
db.save_email_at(&sample_email(), IdentitySource::Surrogate, placement)
|
||||
.await
|
||||
.expect("store the occupancy");
|
||||
}
|
||||
|
||||
let backend = MockMailBackend::new();
|
||||
let env = SucceedingEnv {
|
||||
backend: backend.clone(),
|
||||
};
|
||||
let mutation = flag_mutation();
|
||||
let payload =
|
||||
serde_json::from_str::<MutationPayload>(&mutation.payload).expect("parse payload");
|
||||
let disposition =
|
||||
dispatch_mailbox_mutation(&db, &env, &mutation, &sample_email(), &payload).await;
|
||||
(disposition, backend.applied_mutations())
|
||||
}
|
||||
|
||||
/// The unambiguous case still works: exactly one occupancy, so the row's
|
||||
/// target is fully determined and the mutation is addressed from it.
|
||||
#[tokio::test]
|
||||
async fn a_single_placement_mutation_is_addressed_from_that_placement() {
|
||||
let (disposition, applied) = dispatch_with_placements(&[sample_placement()]).await;
|
||||
|
||||
assert!(
|
||||
matches!(disposition, Disposition::Completed),
|
||||
"a fully-addressed mutation must go through"
|
||||
);
|
||||
assert_eq!(applied.len(), 1, "the backend really was asked to act");
|
||||
assert_eq!(applied[0].folder_id, "INBOX");
|
||||
assert_eq!(applied[0].remote_id, "77");
|
||||
assert_eq!(applied[0].uid_validity, "9");
|
||||
assert_eq!(applied[0].message_id, "msg-outbox-log");
|
||||
}
|
||||
|
||||
/// The ambiguous case refuses. The outbox row names only a message, so for
|
||||
/// a message in two mailboxes there is no way to know which copy the rule
|
||||
/// matched -- and a guessed MOVE or DELETE lands on mail the user never
|
||||
/// targeted, with no undo.
|
||||
#[tokio::test]
|
||||
async fn a_multi_placement_mutation_is_refused_rather_than_guessed() {
|
||||
let (disposition, applied) =
|
||||
dispatch_with_placements(&[sample_placement(), second_placement()]).await;
|
||||
|
||||
assert!(
|
||||
applied.is_empty(),
|
||||
"an ambiguously-addressed mutation must never reach the server"
|
||||
);
|
||||
match disposition {
|
||||
// Permanent, not Retry: no later attempt makes the outbox row
|
||||
// remember a placement it never recorded.
|
||||
Disposition::Permanent(reason) => {
|
||||
assert!(
|
||||
reason.contains("INBOX") && reason.contains("Archive"),
|
||||
"the refusal must name the competing mailboxes: {reason}"
|
||||
);
|
||||
assert!(
|
||||
reason.contains("refusing"),
|
||||
"and say plainly that it refused: {reason}"
|
||||
);
|
||||
}
|
||||
Disposition::Completed => panic!("expected a permanent refusal, got Completed"),
|
||||
Disposition::Conflict(observed) => {
|
||||
panic!("expected a permanent refusal, got Conflict({observed})")
|
||||
}
|
||||
Disposition::Retry(reason) => {
|
||||
panic!("expected a permanent refusal, got Retry({reason})")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+489
-174
@@ -7,7 +7,8 @@
|
||||
//! account it connects a real [`MailBackend`] (a real [`ImapEngine`] for an
|
||||
//! IMAP/SMTP transport, built from [`AccountConfig`] plus the
|
||||
//! account's keyring-stored password), fetches folders and messages, and
|
||||
//! persists every message through [`DatabaseEngine::save_email`].
|
||||
//! persists every message, together with the mailbox occupancy it was found
|
||||
//! in, through [`DatabaseEngine::save_email_at`].
|
||||
//!
|
||||
//! # Testability
|
||||
//!
|
||||
@@ -20,6 +21,7 @@
|
||||
//! all-accounts sync fans these per-account runs out through
|
||||
//! [`crate::sync_dispatcher::sync_all_configured`], so concurrency and the
|
||||
//! per-account timeout are bounded in one place rather than serialized here.
|
||||
use nuncio_core::model::PlacementKey;
|
||||
use nuncio_core::{AccountConfig, CoreCommand, CoreEvent, EventBus, Transport};
|
||||
use nuncio_filter::{FilterEngine, OutboxManager, RuleAction};
|
||||
use nuncio_mail::{ImapEngine, JmapEngine, MailBackend, MailError};
|
||||
@@ -68,24 +70,39 @@ fn remote_action_tag(action: &RuleAction) -> Option<(&'static str, Option<String
|
||||
}
|
||||
}
|
||||
|
||||
/// Evaluate `email` against `filter_engine` and route every matched rule's
|
||||
/// actions to their real, persisted effect.
|
||||
/// Evaluate `email` **as it sits in `placement`** against `filter_engine` and
|
||||
/// route every matched rule's actions to their real, persisted effect.
|
||||
///
|
||||
/// Filters fire EXACTLY ONCE per newly-arrived message: callers (see
|
||||
/// [`fetch_and_persist`]) must only invoke this for a message that was
|
||||
/// genuinely new to the store on this sync pass, never for one that already
|
||||
/// existed. Sync is not yet incremental -- every sync re-fetches and
|
||||
/// re-persists messages the backend still reports -- so calling this
|
||||
/// unconditionally on every persisted message would re-fire a rule's actions
|
||||
/// on every single re-sync of the same mailbox: a fresh
|
||||
/// `PendingRemoteMutation` and a fresh `FilterExecutionLog` entry per cycle
|
||||
/// for a message that arrived once, growing the outbox and the audit ledger
|
||||
/// without bound and (once a real mutation transport lands) duplicating the
|
||||
/// real remote operation. This function itself has no way to tell "new" from
|
||||
/// "already seen" -- that determination is the caller's responsibility.
|
||||
/// Evaluation is per placement because the rule language can ask where a
|
||||
/// message is: `WHERE FOLDER = 'INBOX'` has no answer for a message that
|
||||
/// occupies three mailboxes, so the caller fans out over the occupancies and
|
||||
/// each call answers for exactly one.
|
||||
///
|
||||
/// `MarkRead`/`MarkUnread` apply immediately to the stored message via
|
||||
/// [`DatabaseEngine::set_message_read`]. Every other action enqueues a real
|
||||
/// Acting, in contrast, is once per (rule, message). Every matched rule must
|
||||
/// win [`DatabaseEngine::claim_filter_fire`] before any of its actions run, and
|
||||
/// the claim is keyed on message identity, not on the placement. `MOVE` and
|
||||
/// `FLAG` converge under repetition -- ten moves leave one net effect -- but
|
||||
/// `FORWARD` and `CALL WEBHOOK` land on third parties and have no shared state
|
||||
/// to converge against: one message in ten folders would page an on-call
|
||||
/// engineer ten times. The claim fails closed, so an unverifiable claim skips
|
||||
/// the actions rather than licensing them.
|
||||
///
|
||||
/// The claim is spent the moment it is won, before the actions are known to
|
||||
/// have succeeded. A rule whose enqueue then fails is therefore not retried on
|
||||
/// a later pass. That is the deliberate trade: re-firing FORWARD and CALL
|
||||
/// WEBHOOK on a partial failure would re-deliver to a third party, and a
|
||||
/// duplicate mail cannot be un-received, while a dropped one is visible in the
|
||||
/// warning below and recoverable by hand.
|
||||
///
|
||||
/// The claim also subsumes the older "only call me for a new message" contract
|
||||
/// for re-arrivals of the same message. Callers should still gate on a genuinely
|
||||
/// new occupancy (see [`fetch_and_persist`]) -- the claim stops repeat *actions*,
|
||||
/// not the wasted evaluation of a mailbox nothing changed in.
|
||||
///
|
||||
/// `MarkRead`/`MarkUnread` apply immediately to the occupancy that was
|
||||
/// evaluated, via [`DatabaseEngine::set_placement_read`]: IMAP `\Seen` is
|
||||
/// per-mailbox, so "mark this read" is only well-defined once it names a
|
||||
/// mailbox. Every other action enqueues a real
|
||||
/// [`nuncio_filter::PendingRemoteMutation`] through the existing outbox
|
||||
/// (`OutboxManager::create_mutation` + `DatabaseEngine::save_pending_mutation`)
|
||||
/// for the background outbox worker to pick up -- this function never
|
||||
@@ -104,40 +121,69 @@ fn remote_action_tag(action: &RuleAction) -> Option<(&'static str, Option<String
|
||||
///
|
||||
/// Standalone and reusable by design: this is the exact evaluate-and-route
|
||||
/// logic other sync paths (e.g. a bulk retroactive triage over the whole
|
||||
/// store) need, so it takes only a `DatabaseEngine`/`FilterEngine`/`Email`
|
||||
/// and has no dependency on the live inbound-sync call chain.
|
||||
/// store) need, so it takes only a `DatabaseEngine`/`FilterEngine`/`Email`/
|
||||
/// `Placement` and has no dependency on the live inbound-sync call chain.
|
||||
pub async fn apply_filter_actions(
|
||||
db: &nuncio_store::db::DatabaseEngine,
|
||||
filter_engine: &FilterEngine,
|
||||
email: &nuncio_core::model::Email,
|
||||
placement: &nuncio_core::model::Placement,
|
||||
) -> usize {
|
||||
let mut applied = 0usize;
|
||||
let placed = nuncio_filter::PlacedEmail { email, placement };
|
||||
|
||||
for (rule, actions) in filter_engine.evaluate(email) {
|
||||
for (rule, actions) in filter_engine.evaluate(placed) {
|
||||
// Claim before acting, never after. The claim is keyed on message
|
||||
// identity, so the same mail reaching a second mailbox -- an ordinary
|
||||
// MOVE, or a Gmail label -- evaluates again but acts only once.
|
||||
match db.claim_filter_fire(&rule.id, &email.id).await {
|
||||
Ok(true) => {}
|
||||
Ok(false) => {
|
||||
tracing::debug!(
|
||||
rule_id = %rule.id,
|
||||
message_key = %email.id,
|
||||
folder_id = %placement.folder_id,
|
||||
"rule already fired for this message; skipping actions for this placement"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
// Fail closed. An unverifiable claim must not become a licence
|
||||
// to act: a lost claim check on a FORWARD is a duplicate mail
|
||||
// the recipient cannot un-receive.
|
||||
tracing::error!(
|
||||
rule_id = %rule.id,
|
||||
message_key = %email.id,
|
||||
error = %e,
|
||||
"could not claim the filter fire; skipping actions"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let placement_key = placement.key();
|
||||
for action in actions {
|
||||
let immediate_ok = match &action {
|
||||
RuleAction::MarkRead => match db.set_message_read(&email.id, true).await {
|
||||
Ok(()) => true,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"filter rule '{}' MARK READ failed for message '{}': {e}",
|
||||
rule.id,
|
||||
email.id
|
||||
);
|
||||
false
|
||||
// One write with a boolean rather than two near-identical
|
||||
// arms. Both name the occupancy that was evaluated, because
|
||||
// `\Seen` is per-mailbox: a message read in INBOX is not
|
||||
// thereby read in Archive.
|
||||
RuleAction::MarkRead | RuleAction::MarkUnread => {
|
||||
let read = matches!(action, RuleAction::MarkRead);
|
||||
let label = if read { "MARK READ" } else { "MARK UNREAD" };
|
||||
match db.set_placement_read(&placement_key, read).await {
|
||||
Ok(()) => true,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"filter rule '{}' {label} failed for message '{}' in folder '{}': {e}",
|
||||
rule.id,
|
||||
email.id,
|
||||
placement.folder_id
|
||||
);
|
||||
false
|
||||
}
|
||||
}
|
||||
},
|
||||
RuleAction::MarkUnread => match db.set_message_read(&email.id, false).await {
|
||||
Ok(()) => true,
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"filter rule '{}' MARK UNREAD failed for message '{}': {e}",
|
||||
rule.id,
|
||||
email.id
|
||||
);
|
||||
false
|
||||
}
|
||||
},
|
||||
}
|
||||
RuleAction::MoveTo(_)
|
||||
| RuleAction::CopyTo(_)
|
||||
| RuleAction::Flag
|
||||
@@ -197,31 +243,42 @@ pub async fn apply_filter_actions(
|
||||
applied
|
||||
}
|
||||
|
||||
/// Fetch every folder and every message in every folder from `backend`,
|
||||
/// persisting each message via [`DatabaseEngine::save_email`] and, for
|
||||
/// messages genuinely new to the store, evaluating it against
|
||||
/// `filter_engine` via [`apply_filter_actions`].
|
||||
/// Fetch every folder from `backend`, persist each reported occupancy via
|
||||
/// [`DatabaseEngine::save_email_at`], evaluate the genuinely-new occupancies
|
||||
/// against `filter_engine` through [`apply_filter_actions`], and delete the
|
||||
/// occupancies the server no longer reports.
|
||||
///
|
||||
/// Sync is not yet incremental (a re-sync re-fetches every message the
|
||||
/// backend still reports, not just genuinely new ones), so new-vs-seen must
|
||||
/// be determined BEFORE `save_email` persists (or re-persists) anything --
|
||||
/// `save_email` is `INSERT OR REPLACE` and would otherwise erase the
|
||||
/// distinction between "arriving for the first time" and "seen again". Doing
|
||||
/// that with one existence lookup per message (`DatabaseEngine::get_message`)
|
||||
/// is an N+1 query pattern over a fetched chunk; instead, this classifies an
|
||||
/// entire folder's chunk with a single batched
|
||||
/// `DatabaseEngine::existing_message_ids` lookup up front, then folds in a
|
||||
/// per-batch "already seen this pass" set while iterating so a chunk that
|
||||
/// itself contains a repeated id (e.g. a backend surfacing the same message
|
||||
/// twice in one fetch) still fires at most once -- exactly as if each
|
||||
/// message's existence had been checked one at a time immediately before its
|
||||
/// own `save_email`.
|
||||
/// # Why classification is per placement, not per message
|
||||
///
|
||||
/// Returns the total number of messages processed (a message id "processed"
|
||||
/// more than once, e.g. by a backend that returns the same id from multiple
|
||||
/// folders, is counted once per occurrence -- `save_email` itself is
|
||||
/// `INSERT OR REPLACE`, so storage stays deduplicated by message id
|
||||
/// regardless).
|
||||
/// Message identity is folder-independent: a message already stored from
|
||||
/// `INBOX` has the same key when it turns up in `Archive`. Asking "have I seen
|
||||
/// this *message*?" therefore answers `yes` for a mailbox this daemon has never
|
||||
/// filtered, and the arrival silently fires nothing -- no error, no log. The
|
||||
/// question that matches the intent is "have I seen this *occupancy*?", so this
|
||||
/// classifies on [`PlacementKey`] via
|
||||
/// [`DatabaseEngine::existing_placements`].
|
||||
///
|
||||
/// Classification happens BEFORE the write, because `save_email_at` upserts and
|
||||
/// would otherwise erase the distinction between a first arrival and a
|
||||
/// re-report. One batched lookup covers the whole chunk rather than an N+1 of
|
||||
/// per-row existence checks, and a per-pass "already seen" set folds in on top
|
||||
/// so a backend that surfaces the same occupancy twice in one fetch still
|
||||
/// classifies it as new exactly once.
|
||||
///
|
||||
/// Firing once per *arrival* is not the same as acting once per *message*; the
|
||||
/// second guarantee lives in the claim inside [`apply_filter_actions`].
|
||||
///
|
||||
/// # Absence
|
||||
///
|
||||
/// [`nuncio_mail::FolderChanges::present`] is honoured as written: `None` means
|
||||
/// the pass was incremental and cannot speak to absence, and treating it as
|
||||
/// "nothing is present" would delete the whole folder. Only `Some(list)` --
|
||||
/// which `nuncio-mail` withholds when a SELECT omitted UIDVALIDITY, since the
|
||||
/// keys it could build would name no stored row -- licenses deleting the stored
|
||||
/// placements missing from it.
|
||||
///
|
||||
/// Returns the total number of occupancies processed: one message reported from
|
||||
/// two folders counts twice, because two mailboxes really did report it.
|
||||
///
|
||||
/// Contains no event-bus or credential logic -- pure fetch-and-persist, so
|
||||
/// it is the smallest unit tests can exercise directly with a
|
||||
@@ -248,25 +305,29 @@ async fn fetch_and_persist(
|
||||
let changes = backend
|
||||
.sync_changes(&folder.id, last_state.as_deref())
|
||||
.await?;
|
||||
let emails = changes.upserts;
|
||||
let fetched = emails.len();
|
||||
let placed = changes.upserts;
|
||||
let fetched = placed.len();
|
||||
let new_state = changes.next_state;
|
||||
|
||||
// One round trip classifies the whole chunk instead of one lookup per
|
||||
// message. A lookup failure that is not "not found" would be a
|
||||
// genuine DB error; `existing_message_ids` surfaces those as `Err`
|
||||
// rather than silently treating the batch as "not new", so callers
|
||||
// never persist and skip-filter on an unverified assumption.
|
||||
let chunk_ids: Vec<String> = emails.iter().map(|e| e.id.clone()).collect();
|
||||
let already_present = db.existing_message_ids(&chunk_ids).await?;
|
||||
// occupancy. This asks about *placements*, not messages: identity is
|
||||
// folder-independent, so a message already stored from another folder
|
||||
// is still a first arrival here, and testing the message key instead
|
||||
// would classify it as seen and skip filtering it entirely. A lookup
|
||||
// failure that is not "not found" surfaces as `Err` rather than
|
||||
// silently reading as "not new", so nothing is persisted and
|
||||
// skip-filtered on an unverified assumption.
|
||||
let chunk_keys: Vec<PlacementKey> = placed.iter().map(|p| p.placement.key()).collect();
|
||||
let already_present = db.existing_placements(&chunk_keys).await?;
|
||||
let mut seen_this_pass = std::collections::HashSet::new();
|
||||
|
||||
for email in emails {
|
||||
let is_new =
|
||||
!already_present.contains(&email.id) && seen_this_pass.insert(email.id.clone());
|
||||
db.save_email(&email).await?;
|
||||
for message in placed {
|
||||
let key = message.placement.key();
|
||||
let is_new = !already_present.contains(&key) && seen_this_pass.insert(key);
|
||||
db.save_email_at(&message.email, message.source, &message.placement)
|
||||
.await?;
|
||||
if is_new {
|
||||
apply_filter_actions(db, filter_engine, &email).await;
|
||||
apply_filter_actions(db, filter_engine, &message.email, &message.placement).await;
|
||||
}
|
||||
synced += 1;
|
||||
}
|
||||
@@ -274,31 +335,39 @@ async fn fetch_and_persist(
|
||||
// could only ever add: a message moved or deleted by any other client
|
||||
// -- another daemon, a phone, webmail -- stayed in the local store
|
||||
// forever, and every account accumulated ghosts.
|
||||
if let Some(acct) = account_id {
|
||||
let mut gone: Vec<String> = changes.removals;
|
||||
//
|
||||
// What a folder stops mentioning is an occupancy, never a message: the
|
||||
// same mail may still sit in another mailbox, and it goes only when its
|
||||
// last placement does -- which `delete_placements` decides, not this.
|
||||
//
|
||||
// A `PlacementKey` names its own account, so an explicitly-reported
|
||||
// removal is actionable with no ambient account context at all. Only
|
||||
// the `present` diff needs one, because its other half is a lookup
|
||||
// keyed by (account, folder).
|
||||
let mut gone: Vec<PlacementKey> = changes.removals;
|
||||
|
||||
// `present` is the folder's complete contents when the pass was
|
||||
// able to enumerate them. `None` and `Some(vec![])` are NOT the
|
||||
// same: `None` means the pass was incremental and cannot speak to
|
||||
// absence, and treating it as "nothing is present" would delete
|
||||
// the folder.
|
||||
if let Some(present) = changes.present {
|
||||
let present: std::collections::HashSet<String> = present.into_iter().collect();
|
||||
let stored = db.message_ids_in_folder(acct, &folder.id).await?;
|
||||
gone.extend(stored.into_iter().filter(|id| !present.contains(id)));
|
||||
}
|
||||
// `present` is the folder's complete contents when the pass was
|
||||
// able to enumerate them. `None` and `Some(vec![])` are NOT the
|
||||
// same: `None` means the pass was incremental and cannot speak to
|
||||
// absence, and treating it as "nothing is present" would delete
|
||||
// the folder.
|
||||
if let (Some(acct), Some(present)) = (account_id, changes.present) {
|
||||
let present: std::collections::HashSet<PlacementKey> = present.into_iter().collect();
|
||||
let stored = db.placements_in_folder(acct, &folder.id).await?;
|
||||
gone.extend(stored.into_iter().filter(|key| !present.contains(key)));
|
||||
}
|
||||
|
||||
gone.sort_unstable();
|
||||
gone.dedup();
|
||||
if !gone.is_empty() {
|
||||
let deleted = db.delete_messages(&gone).await?;
|
||||
tracing::info!(
|
||||
account_id = %acct,
|
||||
folder_id = %folder.id,
|
||||
deleted,
|
||||
"removed messages that are no longer on the server"
|
||||
);
|
||||
}
|
||||
gone.sort_unstable();
|
||||
gone.dedup();
|
||||
if !gone.is_empty() {
|
||||
let outcome = db.delete_placements(&gone).await?;
|
||||
tracing::info!(
|
||||
account_id = ?account_id,
|
||||
folder_id = %folder.id,
|
||||
placements_removed = outcome.placements_removed,
|
||||
messages_reaped = outcome.messages_reaped,
|
||||
"removed placements that are no longer on the server"
|
||||
);
|
||||
}
|
||||
|
||||
// Persist the returned checkpoint only AFTER this folder's messages
|
||||
@@ -446,9 +515,9 @@ pub async fn run_account_sync(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use nuncio_core::model::{Email, Folder};
|
||||
use nuncio_core::model::{Email, Folder, IdentitySource, Placement};
|
||||
use nuncio_core::{CoreEvent, EngineStatus, TlsMode};
|
||||
use nuncio_mail::MockMailBackend;
|
||||
use nuncio_mail::{MockMailBackend, PlacedMessage};
|
||||
use nuncio_store::db::DatabaseEngine;
|
||||
use serde_json::json;
|
||||
use wiremock::matchers::{body_partial_json, method, path};
|
||||
@@ -568,24 +637,58 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn mock_email(id: &str, folder_id: &str, subject: &str) -> Email {
|
||||
Email {
|
||||
id: id.to_string(),
|
||||
account_id: "acct-mock-1".to_string(),
|
||||
folder_id: folder_id.to_string(),
|
||||
remote_id: id.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: subject.to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
read: false,
|
||||
body_plain: Some(format!("body for {id}")),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
/// One mock message as a backend would surface it: identity plus the
|
||||
/// occupancy it was found in. `id` doubles as the folder-scoped UID, which
|
||||
/// keeps distinct ids in one folder distinct occupancies.
|
||||
fn mock_placed(id: &str, folder_id: &str, subject: &str) -> PlacedMessage {
|
||||
PlacedMessage {
|
||||
email: Email {
|
||||
id: id.to_string(),
|
||||
account_id: "acct-mock-1".to_string(),
|
||||
subject: subject.to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "bob@nuncio.mx".to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
body_plain: Some(format!("body for {id}")),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
source: IdentitySource::Surrogate,
|
||||
placement: Placement {
|
||||
account_id: "acct-mock-1".to_string(),
|
||||
folder_id: folder_id.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: id.to_string(),
|
||||
read: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// The read flag of a message's one occupancy, for the single-placement
|
||||
/// mock fixtures. Fails loudly rather than defaulting if the message turns
|
||||
/// out to sit in no mailbox at all -- "unread" and "not there" are
|
||||
/// different answers and must not be conflated by a test helper.
|
||||
async fn read_flag_of(db: &DatabaseEngine, message_key: &str) -> bool {
|
||||
let placements = db
|
||||
.placements_of(message_key)
|
||||
.await
|
||||
.expect("read the message's placements");
|
||||
assert_eq!(
|
||||
placements.len(),
|
||||
1,
|
||||
"fixture expects exactly one occupancy of '{message_key}'"
|
||||
);
|
||||
placements[0].read
|
||||
}
|
||||
|
||||
/// The message key every [`MockMailBackend::with_same_message_in`]
|
||||
/// occupancy carries.
|
||||
fn shared_key() -> &'static str {
|
||||
MockMailBackend::SHARED_MESSAGE_KEY
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sync_with_backend_persists_mock_messages_and_emits_events() {
|
||||
let (db, _dir) = DatabaseEngine::connect_ephemeral()
|
||||
@@ -601,8 +704,8 @@ mod tests {
|
||||
total_messages: 2,
|
||||
unread_messages: 2,
|
||||
});
|
||||
mock.add_message(mock_email("m1", "inbox", "Hello"));
|
||||
mock.add_message(mock_email("m2", "inbox", "World"));
|
||||
mock.add_message(mock_placed("m1", "inbox", "Hello"));
|
||||
mock.add_message(mock_placed("m2", "inbox", "World"));
|
||||
|
||||
let filter_engine = empty_filter_engine();
|
||||
let synced = sync_with_backend(
|
||||
@@ -617,11 +720,14 @@ mod tests {
|
||||
assert_eq!(synced, 2);
|
||||
|
||||
let persisted = db
|
||||
.list_messages("inbox", 10)
|
||||
.list_messages("acct-mock-1", "inbox", 10)
|
||||
.await
|
||||
.expect("list persisted messages");
|
||||
assert_eq!(persisted.len(), 2);
|
||||
let subjects: Vec<String> = persisted.iter().map(|e| e.subject.clone()).collect();
|
||||
let subjects: Vec<String> = persisted
|
||||
.iter()
|
||||
.map(|(email, _placement)| email.subject.clone())
|
||||
.collect();
|
||||
assert!(subjects.contains(&"Hello".to_string()));
|
||||
assert!(subjects.contains(&"World".to_string()));
|
||||
|
||||
@@ -815,12 +921,13 @@ mod tests {
|
||||
// folder, and its protocol-native JMAP object id round-trips in
|
||||
// `remote_id`.
|
||||
let persisted = db
|
||||
.list_messages("inbox", 10)
|
||||
.list_messages("acct-jmap-real-1", "inbox", 10)
|
||||
.await
|
||||
.expect("jmap message persisted");
|
||||
assert_eq!(persisted.len(), 1);
|
||||
assert_eq!(persisted[0].subject, "Welcome to JMAP Sync");
|
||||
assert_eq!(persisted[0].remote_id, "jmap-msg-1");
|
||||
let (email, placement) = &persisted[0];
|
||||
assert_eq!(email.subject, "Welcome to JMAP Sync");
|
||||
assert_eq!(placement.remote_id, "jmap-msg-1");
|
||||
|
||||
assert_eq!(
|
||||
events.recv().await.expect("start event"),
|
||||
@@ -927,17 +1034,16 @@ mod tests {
|
||||
total_messages: 1,
|
||||
unread_messages: 1,
|
||||
});
|
||||
mock.add_message(mock_email("m-urgent", "inbox", "Urgent: server down"));
|
||||
mock.add_message(mock_placed("m-urgent", "inbox", "Urgent: server down"));
|
||||
|
||||
let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
|
||||
.await
|
||||
.expect("sync succeeds");
|
||||
assert_eq!(synced, 1);
|
||||
|
||||
let stored = db.get_message("m-urgent").await.expect("message persisted");
|
||||
assert!(
|
||||
stored.read,
|
||||
"MARK READ action must genuinely flip the stored read flag"
|
||||
read_flag_of(&db, "m-urgent").await,
|
||||
"MARK READ action must genuinely flip the occupancy's read flag"
|
||||
);
|
||||
|
||||
let logs = db
|
||||
@@ -979,7 +1085,7 @@ mod tests {
|
||||
total_messages: 1,
|
||||
unread_messages: 1,
|
||||
});
|
||||
mock.add_message(mock_email("m-archive", "inbox", "Please Archive Me"));
|
||||
mock.add_message(mock_placed("m-archive", "inbox", "Please Archive Me"));
|
||||
|
||||
let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
|
||||
.await
|
||||
@@ -1004,11 +1110,7 @@ mod tests {
|
||||
assert_eq!(logs[0].message_id, "m-archive");
|
||||
|
||||
// The remote action must not have applied a local read-flag change.
|
||||
let stored = db
|
||||
.get_message("m-archive")
|
||||
.await
|
||||
.expect("message persisted");
|
||||
assert!(!stored.read);
|
||||
assert!(!read_flag_of(&db, "m-archive").await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1033,15 +1135,17 @@ mod tests {
|
||||
total_messages: 1,
|
||||
unread_messages: 1,
|
||||
});
|
||||
mock.add_message(mock_email("m-plain", "inbox", "Just a normal update"));
|
||||
mock.add_message(mock_placed("m-plain", "inbox", "Just a normal update"));
|
||||
|
||||
let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
|
||||
.await
|
||||
.expect("sync succeeds");
|
||||
assert_eq!(synced, 1);
|
||||
|
||||
let stored = db.get_message("m-plain").await.expect("message persisted");
|
||||
assert!(!stored.read, "non-matching message must stay untouched");
|
||||
assert!(
|
||||
!read_flag_of(&db, "m-plain").await,
|
||||
"non-matching message must stay untouched"
|
||||
);
|
||||
|
||||
let logs = db
|
||||
.list_filter_execution_logs(10)
|
||||
@@ -1092,7 +1196,9 @@ mod tests {
|
||||
total_messages: 1,
|
||||
unread_messages: 1,
|
||||
});
|
||||
mock.add_message(mock_email("m-repeat", "inbox", "Urgent: server down"));
|
||||
let repeat = mock_placed("m-repeat", "inbox", "Urgent: server down");
|
||||
let repeat_key = repeat.placement.key();
|
||||
mock.add_message(repeat);
|
||||
|
||||
// First sync: the message is genuinely new, so both rules must fire.
|
||||
let synced_first = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
|
||||
@@ -1100,11 +1206,7 @@ mod tests {
|
||||
.expect("first sync succeeds");
|
||||
assert_eq!(synced_first, 1);
|
||||
|
||||
let stored_after_first = db
|
||||
.get_message("m-repeat")
|
||||
.await
|
||||
.expect("message persisted after first sync");
|
||||
assert!(stored_after_first.read);
|
||||
assert!(read_flag_of(&db, "m-repeat").await);
|
||||
|
||||
let logs_after_first = db
|
||||
.list_filter_execution_logs(10)
|
||||
@@ -1121,7 +1223,7 @@ mod tests {
|
||||
// Manually flip the message back to unread, exactly as a user
|
||||
// reading and then un-reading it would -- proves the second sync
|
||||
// doesn't re-flip it back to read via a repeat MARK READ fire.
|
||||
db.set_message_read("m-repeat", false)
|
||||
db.set_placement_read(&repeat_key, false)
|
||||
.await
|
||||
.expect("manually mark unread");
|
||||
|
||||
@@ -1132,12 +1234,8 @@ mod tests {
|
||||
.expect("second sync succeeds");
|
||||
assert_eq!(synced_second, 1);
|
||||
|
||||
let stored_after_second = db
|
||||
.get_message("m-repeat")
|
||||
.await
|
||||
.expect("message persisted after second sync");
|
||||
assert!(
|
||||
!stored_after_second.read,
|
||||
!read_flag_of(&db, "m-repeat").await,
|
||||
"a re-sync of an already-seen message must not re-fire MARK READ"
|
||||
);
|
||||
|
||||
@@ -1164,10 +1262,10 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn sync_with_backend_fires_filters_only_for_new_messages_in_a_mixed_batch() {
|
||||
// A single fetched chunk containing a mix of an already-seen message
|
||||
// (persisted by a prior sync) and genuinely new ones must fire filter
|
||||
// actions ONLY for the new ones, via the batched
|
||||
// `existing_message_ids` classification -- not a per-message lookup.
|
||||
// A single fetched chunk containing a mix of an already-stored
|
||||
// occupancy (persisted by a prior sync) and genuinely new ones must
|
||||
// fire filter actions ONLY for the new ones, via the batched
|
||||
// `existing_placements` classification -- not a per-row lookup.
|
||||
let (db, _dir) = DatabaseEngine::connect_ephemeral()
|
||||
.await
|
||||
.expect("ephemeral db");
|
||||
@@ -1181,16 +1279,13 @@ mod tests {
|
||||
.expect("parse rule");
|
||||
let filter_engine = FilterEngine::new(vec![mark_read_rule]).expect("compile rule");
|
||||
|
||||
// Pre-seed one message directly, simulating it having landed in a
|
||||
// Pre-seed one occupancy directly, simulating it having landed in a
|
||||
// prior sync pass. It must NOT re-fire even though this pass's
|
||||
// backend still reports it (sync is not yet incremental).
|
||||
db.save_email(&mock_email(
|
||||
"m-already-seen",
|
||||
"inbox",
|
||||
"Urgent: known issue",
|
||||
))
|
||||
.await
|
||||
.expect("pre-seed existing message");
|
||||
let seeded = mock_placed("m-already-seen", "inbox", "Urgent: known issue");
|
||||
db.save_email_at(&seeded.email, seeded.source, &seeded.placement)
|
||||
.await
|
||||
.expect("pre-seed existing occupancy");
|
||||
|
||||
let mock = MockMailBackend::new();
|
||||
mock.add_folder(Folder {
|
||||
@@ -1200,9 +1295,13 @@ mod tests {
|
||||
unread_messages: 3,
|
||||
});
|
||||
// Mixed chunk: one already-present id, two genuinely new ids.
|
||||
mock.add_message(mock_email("m-already-seen", "inbox", "Urgent: known issue"));
|
||||
mock.add_message(mock_email("m-new-1", "inbox", "Urgent: brand new"));
|
||||
mock.add_message(mock_email("m-new-2", "inbox", "Urgent: also new"));
|
||||
mock.add_message(mock_placed(
|
||||
"m-already-seen",
|
||||
"inbox",
|
||||
"Urgent: known issue",
|
||||
));
|
||||
mock.add_message(mock_placed("m-new-1", "inbox", "Urgent: brand new"));
|
||||
mock.add_message(mock_placed("m-new-2", "inbox", "Urgent: also new"));
|
||||
|
||||
let synced = sync_with_backend(&db, &event_bus, &mock, &filter_engine, None)
|
||||
.await
|
||||
@@ -1224,16 +1323,12 @@ mod tests {
|
||||
assert!(fired_ids.contains("m-new-2"));
|
||||
assert!(!fired_ids.contains("m-already-seen"));
|
||||
|
||||
// The pre-seeded message must still be persisted (INSERT OR REPLACE
|
||||
// semantics are unaffected by the batched new/seen classification)
|
||||
// but its read flag must be untouched by this pass's MARK READ rule.
|
||||
let already_seen = db
|
||||
.get_message("m-already-seen")
|
||||
.await
|
||||
.expect("still persisted");
|
||||
// The pre-seeded occupancy must still be persisted (the upsert is
|
||||
// unaffected by the batched new/seen classification) but its read flag
|
||||
// must be untouched by this pass's MARK READ rule.
|
||||
assert!(
|
||||
!already_seen.read,
|
||||
"a message that already existed before this sync must not re-fire MARK READ"
|
||||
!read_flag_of(&db, "m-already-seen").await,
|
||||
"an occupancy that already existed before this sync must not re-fire MARK READ"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1256,7 +1351,7 @@ mod tests {
|
||||
total_messages: 1,
|
||||
unread_messages: 1,
|
||||
});
|
||||
mock.add_message(mock_email("m-inc-1", "inbox", "First"));
|
||||
mock.add_message(mock_placed("m-inc-1", "inbox", "First"));
|
||||
|
||||
let filter_engine = empty_filter_engine();
|
||||
|
||||
@@ -1301,4 +1396,224 @@ mod tests {
|
||||
vec![None, Some("uidnext-4242".to_string())],
|
||||
);
|
||||
}
|
||||
|
||||
/// A rule that matches the one message the shared-identity mock stages.
|
||||
fn rule_matching_the_shared_message() -> nuncio_filter::FilterRule {
|
||||
nuncio_filter::NsqlParser::parse_rule(
|
||||
"Archive Shared",
|
||||
1,
|
||||
"WHERE subject CONTAINS 'Shared' ACTION MOVE TO 'Archive'",
|
||||
)
|
||||
.expect("parse rule")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_message_arriving_in_a_second_folder_still_gets_evaluated() {
|
||||
// The regression this guards: with identity now folder-independent, a
|
||||
// message already stored from INBOX has a known message_key, so a
|
||||
// message-scoped "already present" test would classify its arrival in
|
||||
// Archive as old and skip filtering it entirely -- silently, with no
|
||||
// error and nothing in the log.
|
||||
let (db, _dir) = DatabaseEngine::connect_ephemeral()
|
||||
.await
|
||||
.expect("ephemeral db");
|
||||
let event_bus = EventBus::new();
|
||||
|
||||
let rule = rule_matching_the_shared_message();
|
||||
let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule");
|
||||
|
||||
// Already stored, from INBOX. The message key is therefore known
|
||||
// before this sync begins -- but nothing has ever filtered Archive.
|
||||
let in_inbox = MockMailBackend::shared_message_in("INBOX", "1");
|
||||
db.save_email_at(&in_inbox.email, in_inbox.source, &in_inbox.placement)
|
||||
.await
|
||||
.expect("pre-seed the INBOX occupancy");
|
||||
|
||||
let backend = MockMailBackend::with_same_message_in(&["Archive"]);
|
||||
let processed = fetch_and_persist(
|
||||
&db,
|
||||
&event_bus,
|
||||
&backend,
|
||||
&filter_engine,
|
||||
Some(MockMailBackend::SHARED_ACCOUNT_ID),
|
||||
)
|
||||
.await
|
||||
.expect("sync succeeds");
|
||||
|
||||
assert_eq!(processed, 1, "the Archive occupancy is processed");
|
||||
let placements = db
|
||||
.placements_of(shared_key())
|
||||
.await
|
||||
.expect("read the message's placements");
|
||||
assert_eq!(
|
||||
placements.len(),
|
||||
2,
|
||||
"one message, now occupying both mailboxes"
|
||||
);
|
||||
|
||||
assert!(
|
||||
db.has_filter_fired(&rule.id, shared_key())
|
||||
.await
|
||||
.expect("read the claim ledger"),
|
||||
"an arrival in a second folder must be evaluated, not skipped as already-seen"
|
||||
);
|
||||
let pending = db
|
||||
.list_pending_mutations(10)
|
||||
.await
|
||||
.expect("list pending mutations");
|
||||
assert_eq!(
|
||||
pending.len(),
|
||||
1,
|
||||
"with a real outbox mutation behind the fire, not just a ledger row"
|
||||
);
|
||||
assert_eq!(pending[0].message_id, shared_key());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_rule_fires_once_for_a_message_that_lands_in_two_folders() {
|
||||
// MOVE and FLAG converge under repetition; FORWARD and CALL WEBHOOK do
|
||||
// not. Two occupancies of one message must not page an on-call
|
||||
// engineer twice.
|
||||
let (db, _dir) = DatabaseEngine::connect_ephemeral()
|
||||
.await
|
||||
.expect("ephemeral db");
|
||||
let event_bus = EventBus::new();
|
||||
|
||||
let rule = rule_matching_the_shared_message();
|
||||
let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule");
|
||||
|
||||
let backend = MockMailBackend::with_same_message_in(&["INBOX", "Archive"]);
|
||||
let processed = fetch_and_persist(
|
||||
&db,
|
||||
&event_bus,
|
||||
&backend,
|
||||
&filter_engine,
|
||||
Some(MockMailBackend::SHARED_ACCOUNT_ID),
|
||||
)
|
||||
.await
|
||||
.expect("sync succeeds");
|
||||
|
||||
assert_eq!(processed, 2, "both occupancies are processed");
|
||||
let placements = db
|
||||
.placements_of(shared_key())
|
||||
.await
|
||||
.expect("read the message's placements");
|
||||
assert_eq!(placements.len(), 2);
|
||||
|
||||
assert_eq!(
|
||||
db.filter_fire_count().await.expect("read the claim ledger"),
|
||||
1,
|
||||
"one claim, therefore one set of actions"
|
||||
);
|
||||
assert!(db
|
||||
.has_filter_fired(&rule.id, shared_key())
|
||||
.await
|
||||
.expect("read the claim ledger"));
|
||||
|
||||
let pending = db
|
||||
.list_pending_mutations(10)
|
||||
.await
|
||||
.expect("list pending mutations");
|
||||
assert_eq!(
|
||||
pending.len(),
|
||||
1,
|
||||
"one outbox mutation for the message, not one per folder"
|
||||
);
|
||||
let logs = db
|
||||
.list_filter_execution_logs(10)
|
||||
.await
|
||||
.expect("list execution logs");
|
||||
assert_eq!(logs.len(), 1, "and one audit entry, not one per folder");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unverifiable_claim_skips_the_actions_rather_than_licensing_them() {
|
||||
// Fail closed. If the claim itself errors, the engine cannot know
|
||||
// whether this rule has already acted on this message -- and a lost
|
||||
// claim check on a FORWARD is a duplicate mail the recipient cannot
|
||||
// un-receive. Dropping the ledger table makes the claim error for real
|
||||
// while every other write still works, so a mutation appearing here
|
||||
// would be a genuine "acted without a claim", not a dead store.
|
||||
let (db, _dir) = DatabaseEngine::connect_ephemeral()
|
||||
.await
|
||||
.expect("ephemeral db");
|
||||
|
||||
let rule = rule_matching_the_shared_message();
|
||||
let filter_engine = FilterEngine::new(vec![rule.clone()]).expect("compile rule");
|
||||
|
||||
let placed = MockMailBackend::shared_message_in("INBOX", "1");
|
||||
db.save_email_at(&placed.email, placed.source, &placed.placement)
|
||||
.await
|
||||
.expect("store the occupancy");
|
||||
|
||||
sqlx::query("DROP TABLE filter_fired")
|
||||
.execute(db.pool())
|
||||
.await
|
||||
.expect("drop the claim ledger");
|
||||
|
||||
let applied =
|
||||
apply_filter_actions(&db, &filter_engine, &placed.email, &placed.placement).await;
|
||||
assert_eq!(applied, 0, "an unclaimed rule must apply nothing");
|
||||
|
||||
let pending = db
|
||||
.list_pending_mutations(10)
|
||||
.await
|
||||
.expect("list pending mutations");
|
||||
assert!(
|
||||
pending.is_empty(),
|
||||
"no outbox mutation may be enqueued off an unverifiable claim"
|
||||
);
|
||||
let logs = db
|
||||
.list_filter_execution_logs(10)
|
||||
.await
|
||||
.expect("list execution logs");
|
||||
assert!(
|
||||
logs.is_empty(),
|
||||
"and nothing may be recorded as having fired"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_account_less_pass_still_applies_server_confirmed_removals() {
|
||||
// A `PlacementKey` names its own account, so an explicitly-reported
|
||||
// removal needs no ambient account context. Gating the whole removals
|
||||
// block on `account_id.is_some()` would silently discard a VANISHED
|
||||
// report on a mock-driven `SyncAll`, leaving a ghost the server has
|
||||
// already told us is gone.
|
||||
let (db, _dir) = DatabaseEngine::connect_ephemeral()
|
||||
.await
|
||||
.expect("ephemeral db");
|
||||
let event_bus = EventBus::new();
|
||||
let filter_engine = empty_filter_engine();
|
||||
|
||||
let staged = mock_placed("m-vanished", "inbox", "Deleted elsewhere");
|
||||
db.save_email_at(&staged.email, staged.source, &staged.placement)
|
||||
.await
|
||||
.expect("store the occupancy a prior pass left behind");
|
||||
|
||||
let mock = MockMailBackend::new();
|
||||
mock.add_folder(Folder {
|
||||
id: "inbox".to_string(),
|
||||
name: "Inbox".to_string(),
|
||||
total_messages: 0,
|
||||
unread_messages: 0,
|
||||
});
|
||||
mock.set_removals(vec![staged.placement.key()]);
|
||||
|
||||
// `None` account: the pass has nowhere to key a checkpoint and cannot
|
||||
// run the `present` diff, but the removal is still actionable.
|
||||
let synced = fetch_and_persist(&db, &event_bus, &mock, &filter_engine, None)
|
||||
.await
|
||||
.expect("sync succeeds");
|
||||
assert_eq!(synced, 0, "nothing new arrived");
|
||||
|
||||
let placements = db
|
||||
.placements_of("m-vanished")
|
||||
.await
|
||||
.expect("read the message's placements");
|
||||
assert!(
|
||||
placements.is_empty(),
|
||||
"the server-confirmed removal must be applied without an account id"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
//! Boots a real `nunciod` daemon gRPC server (`nunciod::grpc::
|
||||
//! serve_on_listener`) on an ephemeral loopback port, backed by a real
|
||||
//! temp file-backed [`DatabaseEngine`] seeded directly via
|
||||
//! [`DatabaseEngine::save_email`] (standing in for messages that were synced
|
||||
//! [`DatabaseEngine::save_email_at`] (standing in for messages that were synced
|
||||
//! before the active rule set existed) and a live [`FilterEngine`] carrying
|
||||
//! a real persisted rule.
|
||||
//!
|
||||
@@ -25,7 +25,7 @@
|
||||
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
|
||||
use nuncio_core::model::Email;
|
||||
use nuncio_core::model::{Email, IdentitySource, Placement};
|
||||
use nuncio_core::EventBus;
|
||||
use nuncio_filter::{FilterEngine, NsqlParser};
|
||||
use nuncio_proto::v1::filters_client::FiltersClient;
|
||||
@@ -42,20 +42,52 @@ fn sample_email(id: &str, subject: &str) -> Email {
|
||||
Email {
|
||||
id: id.to_string(),
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
remote_id: id.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: subject.to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "triage-e2e@nuncio.mx".to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
read: false,
|
||||
body_plain: Some("body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The INBOX occupancy a seeded message sits in. `id` doubles as the UID, so
|
||||
/// distinct messages occupy distinct rows.
|
||||
fn sample_placement(id: &str) -> Placement {
|
||||
Placement {
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: id.to_string(),
|
||||
read: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Seed a message and its occupancy the way a sync pass would.
|
||||
async fn seed(db: &DatabaseEngine, id: &str, subject: &str) {
|
||||
db.save_email_at(
|
||||
&sample_email(id, subject),
|
||||
IdentitySource::Surrogate,
|
||||
&sample_placement(id),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("seed message '{id}': {e}"));
|
||||
}
|
||||
|
||||
/// The read flag of a seeded message's single occupancy.
|
||||
async fn read_flag_of(db: &DatabaseEngine, id: &str) -> bool {
|
||||
let placements = db.placements_of(id).await.expect("read placements");
|
||||
assert_eq!(
|
||||
placements.len(),
|
||||
1,
|
||||
"'{id}' should occupy exactly one folder"
|
||||
);
|
||||
placements[0].read
|
||||
}
|
||||
|
||||
/// Boots the real daemon gRPC server (bearer-token authenticated) on an
|
||||
/// ephemeral loopback port, backed by a real temp file-backed
|
||||
/// `DatabaseEngine` and the given live `FilterEngine`. Returns the server
|
||||
@@ -143,15 +175,9 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun
|
||||
// Seed messages directly into the store, standing in for mail that was
|
||||
// synced BEFORE these rules existed -- exactly the scenario `Triage` is
|
||||
// for. Two match (one per rule), one does not.
|
||||
db.save_email(&sample_email("m-urgent", "Urgent: server down"))
|
||||
.await
|
||||
.expect("seed urgent message");
|
||||
db.save_email(&sample_email("m-archive", "Please Archive Me"))
|
||||
.await
|
||||
.expect("seed archive message");
|
||||
db.save_email(&sample_email("m-plain", "Just saying hi"))
|
||||
.await
|
||||
.expect("seed non-matching message");
|
||||
seed(&db, "m-urgent", "Urgent: server down").await;
|
||||
seed(&db, "m-archive", "Please Archive Me").await;
|
||||
seed(&db, "m-plain", "Just saying hi").await;
|
||||
|
||||
let mut client = FiltersClient::connect(format!("http://{addr}"))
|
||||
.await
|
||||
@@ -188,8 +214,10 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun
|
||||
|
||||
// Real side effects, not a fabricated count: the MARK READ action
|
||||
// actually flipped the stored read flag...
|
||||
let urgent = db.get_message("m-urgent").await.expect("get urgent");
|
||||
assert!(urgent.read, "MARK READ must genuinely flip the read flag");
|
||||
assert!(
|
||||
read_flag_of(&db, "m-urgent").await,
|
||||
"MARK READ must genuinely flip the occupancy's read flag"
|
||||
);
|
||||
|
||||
// ...and the MOVE TO action actually queued a real outbox mutation.
|
||||
let pending = db
|
||||
@@ -203,8 +231,10 @@ async fn triage_applies_live_rule_retroactively_and_reports_real_cumulative_coun
|
||||
);
|
||||
assert_eq!(pending[0].message_id, "m-archive");
|
||||
|
||||
let plain = db.get_message("m-plain").await.expect("get plain");
|
||||
assert!(!plain.read, "a non-matching message must be left untouched");
|
||||
assert!(
|
||||
!read_flag_of(&db, "m-plain").await,
|
||||
"a non-matching message must be left untouched"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -219,12 +249,7 @@ async fn triage_streams_multiple_chunks_for_a_store_bigger_than_one_chunk() {
|
||||
// updates before the final `done: true` -- proof this is a real
|
||||
// multi-chunk stream, not one batch disguised as one.
|
||||
for i in 0..5 {
|
||||
db.save_email(&sample_email(
|
||||
&format!("m-{i:02}"),
|
||||
&format!("Message number {i}"),
|
||||
))
|
||||
.await
|
||||
.expect("seed message");
|
||||
seed(&db, &format!("m-{i:02}"), &format!("Message number {i}")).await;
|
||||
}
|
||||
|
||||
let mut client = FiltersClient::connect(format!("http://{addr}"))
|
||||
|
||||
@@ -27,10 +27,10 @@ use std::sync::{Arc, Mutex};
|
||||
|
||||
use nuncio_cal::{CalDavAccountConfig, CalDavClient, CalendarBackend};
|
||||
use nuncio_contacts::{CardDavAccountConfig, CardDavClient, ContactsBackend};
|
||||
use nuncio_core::model::{Email, Folder};
|
||||
use nuncio_core::model::{Email, Folder, IdentitySource, Placement};
|
||||
use nuncio_core::EventBus;
|
||||
use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher};
|
||||
use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender};
|
||||
use nuncio_mail::{JmapEngine, MailBackend, MockMailBackend, MockMessageSender, PlacedMessage};
|
||||
use nuncio_proto::v1::{
|
||||
AccountConfig, AddAccountRequest, GetStatusRequest, MarkReadRequest, SendMessageRequest,
|
||||
SyncRequest, TlsMode,
|
||||
@@ -139,17 +139,35 @@ fn sentinel_inbound_email() -> Email {
|
||||
Email {
|
||||
id: "msg-canary-1".to_string(),
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
remote_id: "msg-canary-1".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: SENTINEL_SUBJECT.to_string(),
|
||||
sender: SENTINEL_SENDER.to_string(),
|
||||
recipient: SENTINEL_RECIPIENT.to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
read: false,
|
||||
body_plain: Some(SENTINEL_BODY.to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The INBOX occupancy the sentinel message is found in.
|
||||
fn sentinel_placement() -> Placement {
|
||||
Placement {
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: "msg-canary-1".to_string(),
|
||||
read: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The sentinel message as a backend would surface it.
|
||||
fn sentinel_placed_message() -> PlacedMessage {
|
||||
PlacedMessage {
|
||||
email: sentinel_inbound_email(),
|
||||
source: IdentitySource::Surrogate,
|
||||
placement: sentinel_placement(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,7 +230,7 @@ async fn drive_daemon() {
|
||||
total_messages: 1,
|
||||
unread_messages: 1,
|
||||
});
|
||||
mock_backend.add_message(sentinel_inbound_email());
|
||||
mock_backend.add_message(sentinel_placed_message());
|
||||
let mock_sender = MockMessageSender::new();
|
||||
let overrides = MailEngineOverrides {
|
||||
mail_backend: Some(Arc::new(mock_backend)),
|
||||
@@ -307,7 +325,12 @@ async fn drive_filter_and_webhook() {
|
||||
)
|
||||
.expect("parse rule");
|
||||
let engine = FilterEngine::new(vec![rule]).expect("compile rule");
|
||||
let matches = engine.evaluate(&sentinel_inbound_email());
|
||||
let sentinel = sentinel_inbound_email();
|
||||
let placement = sentinel_placement();
|
||||
let matches = engine.evaluate(nuncio_filter::PlacedEmail {
|
||||
email: &sentinel,
|
||||
placement: &placement,
|
||||
});
|
||||
assert_eq!(matches.len(), 1, "rule must fire on the sentinel message");
|
||||
|
||||
let mock_server = MockServer::start().await;
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
|
||||
use async_trait::async_trait;
|
||||
use nuncio_core::model::Email;
|
||||
use nuncio_core::model::{Email, IdentitySource, Placement};
|
||||
use nuncio_core::EventBus;
|
||||
use nuncio_filter::{FilterEngine, NsqlParser, ValidationOptions, WebhookDispatcher, WebhookError};
|
||||
use nuncio_mail::{
|
||||
@@ -116,17 +116,28 @@ fn sample_email(subject: &str) -> Email {
|
||||
Email {
|
||||
id: "surrogate-inbox-42".to_string(),
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
folder_id: FOLDER_ID.to_string(),
|
||||
remote_id: "42".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: subject.to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: "owner@nuncio.mx".to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
read: false,
|
||||
body_plain: Some("original body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The mailbox occupancy the seeded message sits in. The outbox recovers its
|
||||
/// remote addressing from here now -- the UID and the UIDVALIDITY scope it was
|
||||
/// captured under are properties of the occupancy, not of the message.
|
||||
fn sample_placement(remote_id: &str) -> Placement {
|
||||
Placement {
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
folder_id: FOLDER_ID.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: remote_id.to_string(),
|
||||
read: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,11 +153,14 @@ async fn seed_and_enqueue(db: &DatabaseEngine, rule_nsql: &str, subject: &str) {
|
||||
.expect("save folder checkpoint");
|
||||
|
||||
let email = sample_email(subject);
|
||||
db.save_email(&email).await.expect("save email");
|
||||
let placement = sample_placement("42");
|
||||
db.save_email_at(&email, IdentitySource::Surrogate, &placement)
|
||||
.await
|
||||
.expect("save email");
|
||||
|
||||
let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule");
|
||||
let engine = FilterEngine::new(vec![rule]).expect("compile rule");
|
||||
apply_filter_actions(db, &engine, &email).await;
|
||||
apply_filter_actions(db, &engine, &email, &placement).await;
|
||||
|
||||
let pending = db.list_pending_mutations(10).await.expect("list pending");
|
||||
assert_eq!(
|
||||
@@ -173,14 +187,16 @@ async fn seed_and_enqueue_distinct(
|
||||
.await
|
||||
.expect("save folder checkpoint");
|
||||
|
||||
// `remote_id`/`uid_validity` (not just `id`) are part of the store's
|
||||
// `messages` identity UNIQUE index -- leaving them at `sample_email`'s
|
||||
// shared default would make this INSERT OR REPLACE the OTHER seeded
|
||||
// message sharing that same remote identity out from under it.
|
||||
// The occupancy needs its own UID as well as its own message key: the
|
||||
// placements primary key is (account, folder, uidvalidity, uid), so
|
||||
// leaving `remote_id` at the shared default would repoint the OTHER
|
||||
// seeded message's occupancy at this message.
|
||||
let mut email = sample_email(subject);
|
||||
email.id = message_id.to_string();
|
||||
email.remote_id = message_id.to_string();
|
||||
db.save_email(&email).await.expect("save email");
|
||||
let placement = sample_placement(message_id);
|
||||
db.save_email_at(&email, IdentitySource::Surrogate, &placement)
|
||||
.await
|
||||
.expect("save email");
|
||||
|
||||
let before = db
|
||||
.list_pending_mutations(100)
|
||||
@@ -190,7 +206,7 @@ async fn seed_and_enqueue_distinct(
|
||||
|
||||
let rule = NsqlParser::parse_rule("Outbox Rule", 1, rule_nsql).expect("parse rule");
|
||||
let engine = FilterEngine::new(vec![rule]).expect("compile rule");
|
||||
apply_filter_actions(db, &engine, &email).await;
|
||||
apply_filter_actions(db, &engine, &email, &placement).await;
|
||||
|
||||
let after = db
|
||||
.list_pending_mutations(100)
|
||||
|
||||
@@ -31,10 +31,10 @@
|
||||
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
|
||||
use nuncio_core::model::{Email, Folder};
|
||||
use nuncio_core::model::{Email, Folder, IdentitySource, Placement};
|
||||
use nuncio_core::EventBus;
|
||||
use nuncio_filter::FilterEngine;
|
||||
use nuncio_mail::{MockMailBackend, MockMessageSender};
|
||||
use nuncio_mail::{MockMailBackend, MockMessageSender, PlacedMessage};
|
||||
use nuncio_proto::v1::event::Kind;
|
||||
use nuncio_proto::v1::{
|
||||
AccountConfig, AddAccountRequest, GetMessageRequest, ListAccountsRequest, ListMessagesRequest,
|
||||
@@ -73,21 +73,31 @@ fn sample_account_config() -> AccountConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn mock_inbound_email(id: &str, subject: &str, body: &str) -> Email {
|
||||
Email {
|
||||
id: id.to_string(),
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
remote_id: id.to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
subject: subject.to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: ACCOUNT_EMAIL.to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
read: false,
|
||||
body_plain: Some(body.to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
/// One inbound message as a backend surfaces it: identity, the tier the key
|
||||
/// came from, and the INBOX occupancy this pass found it in.
|
||||
fn mock_inbound_email(id: &str, subject: &str, body: &str) -> PlacedMessage {
|
||||
PlacedMessage {
|
||||
email: Email {
|
||||
id: id.to_string(),
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
subject: subject.to_string(),
|
||||
sender: "alice@nuncio.mx".to_string(),
|
||||
recipient: ACCOUNT_EMAIL.to_string(),
|
||||
received_at: 1_700_000_000,
|
||||
body_plain: Some(body.to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
source: IdentitySource::Surrogate,
|
||||
placement: Placement {
|
||||
account_id: ACCOUNT_ID.to_string(),
|
||||
folder_id: "inbox".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
remote_id: id.to_string(),
|
||||
read: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+15
-5
@@ -104,7 +104,7 @@ part of the original M1–M7 numbering:
|
||||
`tls_mode_from_db` silent `ImplicitTls` fallback (#271), the
|
||||
`NaturalLanguageScheduler` that returns wrong times (#258), plus CLI and audit
|
||||
cleanups.
|
||||
- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 44 open). Six tracks
|
||||
- **WS-A … WS-F — the pre-freeze reshape** (22 closed / 56 open). Six tracks
|
||||
that must land before `nuncio.v1` can be frozen: **WS-A** contract hardening
|
||||
(typed errors, `Timestamp`/`Duration` unification, keyset pagination, id
|
||||
conventions, enum hygiene, account transport `oneof`); **WS-B** mail model and
|
||||
@@ -228,19 +228,29 @@ The engine becomes production-solid.
|
||||
### M6.5, WS and OBS — the work between M6 and M7
|
||||
|
||||
Opened after M5, when the reshape needed before an honest `v1` freeze proved
|
||||
larger than M6. Status as of 2026-08-07:
|
||||
larger than M6. Status as of 2026-08-12:
|
||||
|
||||
| Track | Closed / Open | Scope |
|
||||
| :--- | :---: | :--- |
|
||||
| **M6.5** Feature completeness & de-fabrication | 3 / 8 | Remove the remaining fabricated or silently-wrong paths (Contacts sync wiring #255, DAV XML namespaces #257, TLS-mode fallback #271, NL scheduler #258) |
|
||||
| **WS-A** Contract hardening (pre-freeze reshape) | 6 / 3 | Typed errors, `Timestamp`/`Duration`, keyset pagination, id conventions, enum hygiene, account transport `oneof` — **gates M7** |
|
||||
| **WS-B** Mail model & mutations | 1 / 9 | Message identity, drafts, folder management, attachments on receive |
|
||||
| **WS-C** Sync, push & lifecycle | 4 / 4 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown |
|
||||
| **WS-B** Mail model & mutations | 1 / 14 | Message identity, drafts, folder management, attachments on receive |
|
||||
| **WS-C** Sync, push & lifecycle | 4 / 9 | Autonomous scheduler, IMAP IDLE, resumable event stream, graceful shutdown |
|
||||
| **WS-D** Security to 9/10 | 4 / 8 | Whole-DB SQLCipher (#299), UDS / named-pipe IPC (#305), signed updates (#300), HTML sanitization (#302) |
|
||||
| **WS-E** Calendar/Contacts write & Search | 2 / 10 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service |
|
||||
| **WS-E** Calendar/Contacts write & Search | 2 / 12 | DAV write-back with conditional PUT/DELETE, recurrence editing, structured Search service |
|
||||
| **WS-F** Ops, usability & maintainability | 5 / 10 | CLI polish, `System.Backup`, and the `grpc.rs` (#333) / `DatabaseEngine` (#334) decompositions |
|
||||
| **OBS-1 … OBS-9** Observability | 9 / 0 | **COMPLETE.** Tracing subscriber, RPC spans + request-id, domain/lifecycle coverage, protocol instrumentation, `GetStatus`, CLI verbosity, redaction canary |
|
||||
|
||||
WS-B and WS-C grew when **ADR 0002** decided
|
||||
the convergent multi-engine sync model and broke it into #386–#397. Those are
|
||||
in flight, not closed: the message/placement identity split, Message-ID and
|
||||
content-hash capture, `user_version` gating, the raw IMAP command layer, the
|
||||
four-rung change enumeration ladder, and three-state verified mutations are all
|
||||
implemented and green on a branch, and the counts above move when it merges.
|
||||
Still open behind them: the mutation RPCs and durable conflict surface (#395),
|
||||
DAV href/ETag identity and `sync-collection` (#392, #393), the scheduler and
|
||||
rate limiting (#396), and the ≥3-engine convergence harness (#397).
|
||||
|
||||
The OBS stories (#357–#365) were merged without a milestone assignment; the
|
||||
epic is complete regardless, and `docs/LOGGING.md` is its reference.
|
||||
|
||||
|
||||
@@ -280,11 +280,13 @@
|
||||
<p class="cbody">Opened after M5, when the reshape needed before an honest <code>v1</code> freeze proved larger
|
||||
than M6 alone. <b>M6.5</b> de-fabrication (3/11) removes the remaining silently-wrong paths: Contacts sync
|
||||
wiring (#255), namespace-aware DAV XML (#257), the TLS-mode fallback (#271), the NL scheduler (#258).
|
||||
<b>WS-A … WS-F</b> (22/66) is the pre-freeze reshape: contract hardening (WS-A, <b>gates M7</b>), mail model
|
||||
<b>WS-A … WS-F</b> (22/78) is the pre-freeze reshape: contract hardening (WS-A, <b>gates M7</b>), mail model
|
||||
& mutations (WS-B), sync/push/lifecycle (WS-C), security to 9/10 (WS-D — SQLCipher #299, UDS &
|
||||
named-pipe IPC #305, signed updates #300), calendar/contacts write-back & search (WS-E), and ops,
|
||||
usability & maintainability (WS-F — including the <code>grpc.rs</code> #333 and <code>DatabaseEngine</code>
|
||||
#334 decompositions). <b>OBS-1 … OBS-9 is complete</b> (9/9, #357–#365): layered tracing subscriber, per-RPC
|
||||
#334 decompositions). WS-B and WS-C grew when <b>ADR 0002</b> decided the convergent multi-engine sync
|
||||
model and broke it into #386–#397; the message/placement identity split, change enumeration and verified
|
||||
mutations are implemented and green on a branch, not yet merged. <b>OBS-1 … OBS-9 is complete</b> (9/9, #357–#365): layered tracing subscriber, per-RPC
|
||||
spans with request-id correlation, domain and lifecycle coverage, protocol instrumentation, a real
|
||||
<code>System.GetStatus</code>, CLI <code>-v</code>/<code>-vv</code> with typed errors, and a
|
||||
<code>Redacted<T></code> policy with a no-secrets-in-logs canary.</p>
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user