feat(mail): capture Message-ID and a content hash at ingest #398
No files matched your search
@@ -195,6 +195,8 @@ mod tests {
|
||||
body_plain: Some("Hello world body text".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
Email {
|
||||
id: "msg-101".to_string(),
|
||||
@@ -210,6 +212,8 @@ mod tests {
|
||||
body_plain: Some("Second body text".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
@@ -104,6 +104,26 @@ pub struct Email {
|
||||
pub body_html: Option<String>,
|
||||
/// List of attached files.
|
||||
pub attachments: Vec<Attachment>,
|
||||
/// The message's RFC 5322 `Message-ID`, normalized by
|
||||
/// [`Email::normalize_message_id`], or `None` when the message carried
|
||||
/// none (the header is `SHOULD`, not `MUST`) or the transport did not
|
||||
/// surface it.
|
||||
///
|
||||
/// Captured but not yet used for identity. It is a *hint* toward a
|
||||
/// folder-independent identity, never an identity on its own: the value is
|
||||
/// public (it is echoed in the `References` of every reply) and forgeable,
|
||||
/// so keying storage on it alone would let a crafted message address, and
|
||||
/// overwrite, one the user already trusts.
|
||||
pub message_id: Option<String>,
|
||||
/// Hex SHA-256 over the full RFC822 octets the message was parsed from, or
|
||||
/// `None` when only headers or an envelope were available.
|
||||
///
|
||||
/// Pairs with [`Email::message_id`]: the same `Message-ID` carrying
|
||||
/// different content is two messages, not one. That case is ordinary --
|
||||
/// a mailing-list copy and a direct copy of the same mail share a
|
||||
/// `Message-ID` while differing in headers and footer, and a draft keeps
|
||||
/// one `Message-ID` across every save.
|
||||
pub content_hash: Option<String>,
|
||||
}
|
||||
|
||||
impl Email {
|
||||
@@ -131,6 +151,51 @@ impl Email {
|
||||
}
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
|
||||
/// Canonicalize a raw `Message-ID` header value.
|
||||
///
|
||||
/// Every engine syncing an account must derive byte-identical values from
|
||||
/// the same header, or two caches that agree in substance will disagree in
|
||||
/// bytes and no comparison between them can be trusted. The rules are
|
||||
/// therefore fixed and deliberately small:
|
||||
///
|
||||
/// 1. Remove **all** ASCII whitespace. A `msg-id` cannot legally contain
|
||||
/// any, so this unfolds a wrapped header without needing to know how the
|
||||
/// transport folded it -- and transports differ.
|
||||
/// 2. Strip one surrounding pair of angle brackets, which servers include
|
||||
/// or omit inconsistently between `ENVELOPE` and a raw header.
|
||||
/// 3. Preserve case. Only the domain half is case-insensitive per RFC 5322
|
||||
/// §3.6.4; lowercasing the whole value would merge ids that the
|
||||
/// originating server considers distinct, and the same server returns
|
||||
/// the same bytes to every engine anyway.
|
||||
///
|
||||
/// Returns `None` for an absent or empty value so a missing header stays
|
||||
/// distinguishable from a present-but-empty one.
|
||||
pub fn normalize_message_id(raw: &str) -> Option<String> {
|
||||
let stripped: String = raw.chars().filter(|c| !c.is_ascii_whitespace()).collect();
|
||||
let stripped = stripped
|
||||
.strip_prefix('<')
|
||||
.and_then(|s| s.strip_suffix('>'))
|
||||
.unwrap_or(&stripped);
|
||||
if stripped.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(stripped.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Hex SHA-256 over a message's full RFC822 octets, for the
|
||||
/// `content_hash` field.
|
||||
///
|
||||
/// Deterministic for identical bytes, which is what makes it usable as the
|
||||
/// tiebreaker against a duplicated `Message-ID`. It is *not* stable across
|
||||
/// servers that re-serialize a message, so it distinguishes content within
|
||||
/// an account rather than identifying a message globally.
|
||||
pub fn content_hash_of(raw_bytes: &[u8]) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(raw_bytes);
|
||||
hex::encode(hasher.finalize())
|
||||
}
|
||||
}
|
||||
|
||||
/// Mailbox folder entity.
|
||||
@@ -247,4 +312,55 @@ mod tests {
|
||||
Email::surrogate_id("ab", "c", "1", "1"),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_message_id_unfolds_and_unwraps_to_one_canonical_form() {
|
||||
// Every spelling a transport might hand us for the same id must
|
||||
// collapse to identical bytes, or two engines holding the same message
|
||||
// cannot be shown to agree.
|
||||
let canonical = "abc123@mail.example.com";
|
||||
for raw in [
|
||||
"<abc123@mail.example.com>",
|
||||
"abc123@mail.example.com",
|
||||
" <abc123@mail.example.com> ",
|
||||
"<abc123@mail.\r\n example.com>",
|
||||
"<abc123@mail.\t example.com>",
|
||||
] {
|
||||
assert_eq!(
|
||||
Email::normalize_message_id(raw).as_deref(),
|
||||
Some(canonical),
|
||||
"input {raw:?} did not canonicalize"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_message_id_preserves_case_and_rejects_empty() {
|
||||
// Only the domain half is case-insensitive per RFC 5322 3.6.4;
|
||||
// lowercasing wholesale would merge ids the origin considers distinct.
|
||||
assert_eq!(
|
||||
Email::normalize_message_id("<AbC@Example.COM>").as_deref(),
|
||||
Some("AbC@Example.COM")
|
||||
);
|
||||
assert_eq!(Email::normalize_message_id(""), None);
|
||||
assert_eq!(Email::normalize_message_id("<>"), None);
|
||||
assert_eq!(Email::normalize_message_id(" "), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_hash_distinguishes_identical_message_ids_with_different_bodies() {
|
||||
// The case the hash exists for: a list copy and a direct copy share a
|
||||
// Message-ID but are not the same message.
|
||||
let direct = b"Message-ID: <shared@example.com>\r\n\r\nbody";
|
||||
let via_list = b"Message-ID: <shared@example.com>\r\nList-Id: l\r\n\r\nbody\n--\nfooter";
|
||||
assert_ne!(
|
||||
Email::content_hash_of(direct),
|
||||
Email::content_hash_of(via_list)
|
||||
);
|
||||
assert_eq!(
|
||||
Email::content_hash_of(direct),
|
||||
Email::content_hash_of(direct),
|
||||
"the hash must be deterministic for identical octets"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -495,6 +495,8 @@ mod tests {
|
||||
body_plain: Some("Hello".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -518,6 +520,8 @@ mod tests {
|
||||
body_plain: Some("Hello".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
|
||||
let results = engine.evaluate(&email);
|
||||
|
||||
@@ -931,6 +931,14 @@ impl ImapEngine {
|
||||
|
||||
let received_at = fetch_data.internal_date().map_or(0, |dt| dt.timestamp());
|
||||
|
||||
// No body was returned, so there are no octets to hash -- but the
|
||||
// ENVELOPE carries `message-id` (RFC 3501 section 7.4.2), so identity
|
||||
// capture does not have to be skipped on this path.
|
||||
let message_id = fetch_data
|
||||
.envelope()
|
||||
.and_then(|env| env.message_id.as_ref())
|
||||
.and_then(|raw| Email::normalize_message_id(&String::from_utf8_lossy(raw)));
|
||||
|
||||
Ok(Email {
|
||||
id: email_id,
|
||||
account_id: self.account_id.clone(),
|
||||
@@ -945,6 +953,8 @@ impl ImapEngine {
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id,
|
||||
content_hash: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -982,6 +992,12 @@ impl ImapEngine {
|
||||
email.body_plain = None;
|
||||
email.body_html = None;
|
||||
email.attachments = Vec::new();
|
||||
// `parse_mime` hashed the header section it was handed, but
|
||||
// `content_hash` is defined over a message's FULL octets.
|
||||
// Keeping that value would publish a hash that silently means
|
||||
// something else, so drop it; the `Message-ID` it recovered
|
||||
// from those headers is still correct and is kept.
|
||||
email.content_hash = None;
|
||||
return email;
|
||||
}
|
||||
}
|
||||
@@ -1004,6 +1020,8 @@ impl ImapEngine {
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -92,6 +92,10 @@ pub struct JmapEmail {
|
||||
pub received_at: Option<i64>,
|
||||
pub is_unread: Option<bool>,
|
||||
pub body_snippet: Option<String>,
|
||||
/// RFC 8621 models `messageId` as a list, because a message may carry more
|
||||
/// than one `Message-ID` header. The first entry is the one used.
|
||||
#[serde(default)]
|
||||
pub message_id: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
/// JMAP email address object.
|
||||
@@ -184,7 +188,7 @@ impl JmapEngine {
|
||||
{
|
||||
"accountId": account_id,
|
||||
"ids": ids,
|
||||
"properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet"]
|
||||
"properties": ["id", "subject", "from", "to", "receivedAt", "isUnread", "bodySnippet", "messageId"]
|
||||
},
|
||||
"c1"
|
||||
]
|
||||
@@ -418,6 +422,15 @@ impl JmapEngine {
|
||||
body_plain: item.body_snippet,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: item
|
||||
.message_id
|
||||
.as_ref()
|
||||
.and_then(|ids| ids.first())
|
||||
.and_then(|raw| Email::normalize_message_id(raw)),
|
||||
// `Email/get` returns a `bodySnippet`, not the message's
|
||||
// octets, so there is nothing to hash here without an
|
||||
// extra download per message.
|
||||
content_hash: None,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
@@ -705,6 +718,35 @@ mod tests {
|
||||
assert_eq!(new_state, "s-2");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_email_get_response_captures_normalized_message_id() {
|
||||
// RFC 8621 models `messageId` as a list; the first entry is taken and
|
||||
// normalized to the same canonical form the IMAP paths produce, so two
|
||||
// engines syncing one account via different protocols agree.
|
||||
let raw = r#"{
|
||||
"methodResponses": [
|
||||
["Email/get", {"state": "s-1", "list": [
|
||||
{"id": "m-1", "subject": "Hi", "messageId": ["<Abc.123@mail.nuncio.mx>"]},
|
||||
{"id": "m-2", "subject": "No id"}
|
||||
]}, "c1"]
|
||||
]
|
||||
}"#;
|
||||
let engine = JmapEngine::new("acct-1");
|
||||
let (emails, _state) = engine.parse_email_get_response(raw).expect("parse get");
|
||||
|
||||
assert_eq!(
|
||||
emails[0].message_id.as_deref(),
|
||||
Some("Abc.123@mail.nuncio.mx")
|
||||
);
|
||||
assert_eq!(
|
||||
emails[1].message_id, None,
|
||||
"an absent messageId must stay None, not become empty"
|
||||
);
|
||||
// `Email/get` returns a snippet, never the octets, so there is nothing
|
||||
// honest to hash on this path.
|
||||
assert_eq!(emails[0].content_hash, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_invalid_jmap_json_fails() {
|
||||
let engine = JmapEngine::new("acct-1");
|
||||
|
||||
@@ -254,6 +254,8 @@ mod tests {
|
||||
body_plain: Some("Mock body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
mock.add_message(email.clone());
|
||||
|
||||
|
||||
@@ -95,6 +95,9 @@ impl MimeParserAdapter {
|
||||
|
||||
let received_at = msg.date().map_or(0, |d| d.to_timestamp());
|
||||
|
||||
let message_id = msg.message_id().and_then(Email::normalize_message_id);
|
||||
let content_hash = Some(Email::content_hash_of(raw_bytes));
|
||||
|
||||
let body_plain = msg.body_text(0).map(|b| b.to_string());
|
||||
let body_html = msg.body_html(0).map(|b| b.to_string());
|
||||
|
||||
@@ -141,6 +144,8 @@ impl MimeParserAdapter {
|
||||
body_plain,
|
||||
body_html,
|
||||
attachments,
|
||||
message_id,
|
||||
content_hash,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -170,6 +175,39 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_captures_normalized_message_id_and_content_hash() {
|
||||
let raw = b"From: Alice <alice@nuncio.mx>\r\n\
|
||||
To: Bob <bob@nuncio.mx>\r\n\
|
||||
Message-ID: <Abc.123@mail.nuncio.mx>\r\n\
|
||||
Subject: Test\r\n\
|
||||
\r\n\
|
||||
body";
|
||||
|
||||
let email =
|
||||
MimeParserAdapter::parse_mime("msg-1", "acct-1", "inbox", raw).expect("parse succeeds");
|
||||
assert_eq!(email.message_id.as_deref(), Some("Abc.123@mail.nuncio.mx"));
|
||||
assert_eq!(
|
||||
email.content_hash,
|
||||
Some(nuncio_core::model::Email::content_hash_of(raw)),
|
||||
"the hash must cover the octets actually parsed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_reports_no_message_id_when_the_header_is_absent() {
|
||||
// RFC 5322 3.6.4 makes Message-ID a SHOULD. An absent header must read
|
||||
// as "none present", never as an invented or empty value.
|
||||
let raw = b"From: Alice <alice@nuncio.mx>\r\n\
|
||||
Subject: No id\r\n\
|
||||
\r\n\
|
||||
body";
|
||||
let email =
|
||||
MimeParserAdapter::parse_mime("msg-2", "acct-1", "inbox", raw).expect("parse succeeds");
|
||||
assert_eq!(email.message_id, None);
|
||||
assert!(email.content_hash.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_invalid_mime_returns_error() {
|
||||
let raw = b"";
|
||||
|
||||
@@ -368,6 +368,8 @@ mod tests {
|
||||
body_plain: Some("Plaintext status update".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -512,7 +512,13 @@ impl DatabaseEngine {
|
||||
received_at INTEGER NOT NULL,
|
||||
read_flag INTEGER NOT NULL DEFAULT 0,
|
||||
body_plain TEXT,
|
||||
body_html TEXT
|
||||
body_html TEXT,
|
||||
-- Captured at ingest, not yet read. Both are nullable because
|
||||
-- neither is always available: `Message-ID` is SHOULD, not
|
||||
-- MUST (RFC 5322 3.6.4), and a headers-only or envelope-only
|
||||
-- fetch has no full octets to hash.
|
||||
message_id TEXT,
|
||||
content_hash TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS calendar_events (
|
||||
@@ -751,11 +757,53 @@ impl DatabaseEngine {
|
||||
self.ensure_accounts_tls_mode_columns().await?;
|
||||
self.ensure_accounts_dav_columns().await?;
|
||||
self.ensure_messages_identity_columns().await?;
|
||||
self.ensure_messages_provenance_columns().await?;
|
||||
self.backfill_message_fts().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Additive migration adding the `message_id` / `content_hash` provenance
|
||||
/// columns to a pre-existing `messages` table.
|
||||
///
|
||||
/// Both are nullable and there is **no backfill**, deliberately. The values
|
||||
/// cannot be reconstructed for rows already stored: the raw octets are
|
||||
/// discarded after parsing and the `Message-ID` header was never persisted,
|
||||
/// so the only ways to populate history would be to re-download every
|
||||
/// message or to invent a value. Legacy rows therefore stay `NULL`, which
|
||||
/// reads as "not captured" rather than as "this message has no
|
||||
/// `Message-ID`" -- a distinction any future consumer has to respect.
|
||||
///
|
||||
/// A fresh database gets both columns from `CREATE TABLE IF NOT EXISTS`
|
||||
/// above, so the `PRAGMA table_info` check finds them present and this is a
|
||||
/// no-op. SQLite has no `ADD COLUMN IF NOT EXISTS`, hence the explicit
|
||||
/// check, which makes this safe to run on every daemon startup.
|
||||
async fn ensure_messages_provenance_columns(&self) -> Result<(), DatabaseError> {
|
||||
let existing_columns: Vec<String> = sqlx::query("PRAGMA table_info(messages)")
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
.map_err(DatabaseError::Query)?
|
||||
.iter()
|
||||
.map(|row| row.get::<String, _>("name"))
|
||||
.collect();
|
||||
|
||||
for column in ["message_id", "content_hash"] {
|
||||
if existing_columns.iter().any(|c| c == column) {
|
||||
continue;
|
||||
}
|
||||
sqlx::query(&format!("ALTER TABLE messages ADD COLUMN {column} TEXT"))
|
||||
.execute(&self.pool)
|
||||
.await
|
||||
.map_err(DatabaseError::Query)?;
|
||||
tracing::info!(
|
||||
column,
|
||||
"migrated messages table: added message provenance column"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Additive, backfill-safe migration that adds the `remote_id` /
|
||||
/// `uid_validity` message-identity columns to a pre-existing `messages`
|
||||
/// table that predated the opaque surrogate id, and installs the
|
||||
@@ -1295,8 +1343,8 @@ impl DatabaseEngine {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT OR REPLACE INTO messages
|
||||
(id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
(id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag, body_plain, body_html, message_id, content_hash)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
"#,
|
||||
)
|
||||
.bind(&email.id)
|
||||
@@ -1311,6 +1359,8 @@ impl DatabaseEngine {
|
||||
.bind(if email.read { 1i64 } else { 0i64 })
|
||||
.bind(&enc_plain)
|
||||
.bind(&enc_html)
|
||||
.bind(&email.message_id)
|
||||
.bind(&email.content_hash)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(DatabaseError::Query)?;
|
||||
@@ -1361,9 +1411,11 @@ impl DatabaseEngine {
|
||||
Option<String>,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity
|
||||
SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash
|
||||
FROM messages
|
||||
WHERE folder_id = ?
|
||||
ORDER BY received_at DESC
|
||||
@@ -1391,6 +1443,8 @@ impl DatabaseEngine {
|
||||
body_html,
|
||||
remote_id,
|
||||
uid_validity,
|
||||
message_id,
|
||||
content_hash,
|
||||
)| {
|
||||
let dec_plain = body_plain
|
||||
.map(|p| {
|
||||
@@ -1424,6 +1478,8 @@ impl DatabaseEngine {
|
||||
body_plain: dec_plain,
|
||||
body_html: dec_html,
|
||||
attachments: Vec::new(),
|
||||
message_id,
|
||||
content_hash,
|
||||
})
|
||||
},
|
||||
)
|
||||
@@ -1445,7 +1501,7 @@ impl DatabaseEngine {
|
||||
) -> Result<(Vec<nuncio_core::model::Email>, Option<(i64, String)>), DatabaseError> {
|
||||
let fetch = page_size.saturating_add(1);
|
||||
let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new(
|
||||
"SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages WHERE folder_id = ",
|
||||
"SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages WHERE folder_id = ",
|
||||
);
|
||||
builder.push_bind(folder_id.to_string());
|
||||
if let Some((ts, id)) = &after {
|
||||
@@ -1474,6 +1530,8 @@ impl DatabaseEngine {
|
||||
Option<String>,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
)>()
|
||||
.fetch_all(&self.pool)
|
||||
.await
|
||||
@@ -1510,6 +1568,8 @@ impl DatabaseEngine {
|
||||
body_plain: dec_plain,
|
||||
body_html: dec_html,
|
||||
attachments: Vec::new(),
|
||||
message_id: r.12,
|
||||
content_hash: r.13,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, DatabaseError>>()?;
|
||||
@@ -1545,9 +1605,11 @@ impl DatabaseEngine {
|
||||
Option<String>,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
) = sqlx::query_as(
|
||||
r#"
|
||||
SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity
|
||||
SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash
|
||||
FROM messages
|
||||
WHERE id = ?
|
||||
"#,
|
||||
@@ -1582,6 +1644,8 @@ impl DatabaseEngine {
|
||||
body_plain: dec_plain,
|
||||
body_html: dec_html,
|
||||
attachments: Vec::new(),
|
||||
message_id: row.12,
|
||||
content_hash: row.13,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -2435,7 +2499,7 @@ impl DatabaseEngine {
|
||||
limit: usize,
|
||||
) -> Result<Vec<nuncio_core::model::Email>, DatabaseError> {
|
||||
let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new(
|
||||
"SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages "
|
||||
"SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages "
|
||||
);
|
||||
if !last_id.is_empty() {
|
||||
builder.push("WHERE id > ");
|
||||
@@ -2457,6 +2521,8 @@ impl DatabaseEngine {
|
||||
Option<String>,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
)>();
|
||||
|
||||
let rows = query
|
||||
@@ -2492,6 +2558,8 @@ impl DatabaseEngine {
|
||||
body_plain: dec_plain,
|
||||
body_html: dec_html,
|
||||
attachments: Vec::new(),
|
||||
message_id: r.12,
|
||||
content_hash: r.13,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, DatabaseError>>()
|
||||
@@ -2509,7 +2577,7 @@ impl DatabaseEngine {
|
||||
folder_id: Option<&str>,
|
||||
) -> Result<Vec<nuncio_core::model::Email>, DatabaseError> {
|
||||
let mut builder: sqlx::QueryBuilder<'_, sqlx::Sqlite> = sqlx::QueryBuilder::new(
|
||||
"SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity FROM messages"
|
||||
"SELECT id, account_id, folder_id, subject, sender, recipient, received_at, read_flag, body_plain, body_html, remote_id, uid_validity, message_id, content_hash FROM messages"
|
||||
);
|
||||
|
||||
let mut has_filter = false;
|
||||
@@ -2541,6 +2609,8 @@ impl DatabaseEngine {
|
||||
Option<String>,
|
||||
String,
|
||||
String,
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
)>();
|
||||
|
||||
let rows = query
|
||||
@@ -2576,6 +2646,8 @@ impl DatabaseEngine {
|
||||
body_plain: dec_plain,
|
||||
body_html: dec_html,
|
||||
attachments: Vec::new(),
|
||||
message_id: r.12,
|
||||
content_hash: r.13,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, DatabaseError>>()
|
||||
@@ -3084,6 +3156,8 @@ mod tests {
|
||||
body_plain: Some("Plaintext content".to_string()),
|
||||
body_html: Some("<p>HTML content</p>".to_string()),
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
|
||||
engine
|
||||
@@ -3133,6 +3207,8 @@ mod tests {
|
||||
body_plain: Some("Sensitive plaintext body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
engine
|
||||
.save_email(&email)
|
||||
@@ -3437,6 +3513,8 @@ mod tests {
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
engine.save_email(&email).await.expect("save email");
|
||||
|
||||
@@ -4129,6 +4207,8 @@ mod tests {
|
||||
body_plain: Some("Hello".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
engine.save_email(&email).await.unwrap();
|
||||
}
|
||||
@@ -4256,6 +4336,8 @@ mod tests {
|
||||
body_plain: Some("body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4368,6 +4450,73 @@ mod tests {
|
||||
.expect("re-running an already-complete migration is a no-op");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn message_provenance_round_trips_through_every_read_path() {
|
||||
let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
|
||||
|
||||
let mut email = synced_email("acct-1", "INBOX", "42", "7");
|
||||
email.message_id = Some("abc123@mail.nuncio.mx".to_string());
|
||||
email.content_hash = Some(nuncio_core::model::Email::content_hash_of(b"raw octets"));
|
||||
engine.save_email(&email).await.expect("save");
|
||||
|
||||
let by_id = engine.get_message(&email.id).await.expect("get_message");
|
||||
assert_eq!(by_id.message_id, email.message_id);
|
||||
assert_eq!(by_id.content_hash, email.content_hash);
|
||||
|
||||
let listed = engine.list_messages("INBOX", 10).await.expect("list");
|
||||
let found = listed
|
||||
.iter()
|
||||
.find(|m| m.id == email.id)
|
||||
.expect("the saved message is listed");
|
||||
assert_eq!(found.message_id, email.message_id);
|
||||
assert_eq!(found.content_hash, email.content_hash);
|
||||
|
||||
// A message with no Message-ID stays None rather than becoming "".
|
||||
let plain = synced_email("acct-1", "INBOX", "42", "8");
|
||||
assert_eq!(plain.message_id, None);
|
||||
engine.save_email(&plain).await.expect("save");
|
||||
let fetched = engine.get_message(&plain.id).await.expect("get_message");
|
||||
assert_eq!(fetched.message_id, None);
|
||||
assert_eq!(fetched.content_hash, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn provenance_migration_adds_nullable_columns_without_inventing_history() {
|
||||
let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
|
||||
|
||||
// Reproduce a pre-migration database: the columns cannot be dropped
|
||||
// from a live table cheaply, so assert the property that matters --
|
||||
// a row written before capture existed reads back as "not captured".
|
||||
sqlx::query(
|
||||
"INSERT INTO messages \
|
||||
(id, account_id, folder_id, remote_id, uid_validity, subject, sender, recipient, received_at, read_flag) \
|
||||
VALUES ('legacy-1', 'acct-1', 'INBOX', '9', '42', 'subj', 'a@nuncio.mx', 'b@nuncio.mx', 0, 0)",
|
||||
)
|
||||
.execute(engine.pool())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
engine
|
||||
.ensure_messages_provenance_columns()
|
||||
.await
|
||||
.expect("migration is idempotent on an already-migrated database");
|
||||
|
||||
let legacy = engine.get_message("legacy-1").await.expect("get_message");
|
||||
assert_eq!(
|
||||
legacy.message_id, None,
|
||||
"a legacy row must read as not-captured, never as a fabricated id"
|
||||
);
|
||||
assert_eq!(legacy.content_hash, None);
|
||||
|
||||
// Running it twice more must stay a no-op.
|
||||
for _ in 0..2 {
|
||||
engine
|
||||
.ensure_messages_provenance_columns()
|
||||
.await
|
||||
.expect("re-running is a no-op");
|
||||
}
|
||||
}
|
||||
|
||||
fn export_test_email(id: &str, account_id: &str, folder_id: &str) -> nuncio_core::model::Email {
|
||||
nuncio_core::model::Email {
|
||||
id: id.to_string(),
|
||||
@@ -4383,6 +4532,8 @@ mod tests {
|
||||
body_plain: Some(format!("Body {id}")),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -266,6 +266,8 @@ mod tests {
|
||||
body_plain: Some(body_plain.to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1989,6 +1989,8 @@ fn synthetic_preview_email(id: &str) -> nuncio_core::model::Email {
|
||||
body_plain: Some("Sample body text".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3303,6 +3305,8 @@ mod tests {
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
remote_id: "remote-1".to_string(),
|
||||
uid_validity: "1".to_string(),
|
||||
};
|
||||
@@ -4498,6 +4502,8 @@ mod tests {
|
||||
body_plain: Some(body.to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5552,6 +5558,8 @@ mod tests {
|
||||
body_plain: Some("body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
};
|
||||
|
||||
// Before creating anything, the live engine matches nothing.
|
||||
@@ -5663,6 +5671,8 @@ mod tests {
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
})
|
||||
.is_empty());
|
||||
}
|
||||
@@ -5813,6 +5823,8 @@ mod tests {
|
||||
body_plain: Some("Please review the attached deck".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
})
|
||||
.await
|
||||
.expect("seed message");
|
||||
@@ -5987,6 +5999,8 @@ mod tests {
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
});
|
||||
assert_eq!(matches.len(), 1);
|
||||
assert_eq!(matches[0].0.id, rule_id);
|
||||
@@ -6168,6 +6182,8 @@ mod tests {
|
||||
body_plain: None,
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
});
|
||||
assert_eq!(matches.len(), 1);
|
||||
}
|
||||
|
||||
@@ -601,6 +601,8 @@ mod tests {
|
||||
body_plain: Some("secret outbox body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -550,6 +550,8 @@ mod tests {
|
||||
body_plain: Some(format!("body for {id}")),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -53,6 +53,8 @@ fn sample_email(id: &str, subject: &str) -> Email {
|
||||
body_plain: Some("body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -150,6 +150,8 @@ fn sentinel_inbound_email() -> Email {
|
||||
body_plain: Some(SENTINEL_BODY.to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -127,6 +127,8 @@ fn sample_email(subject: &str) -> Email {
|
||||
body_plain: Some("original body".to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -88,6 +88,8 @@ fn mock_inbound_email(id: &str, subject: &str, body: &str) -> Email {
|
||||
body_plain: Some(body.to_string()),
|
||||
body_html: None,
|
||||
attachments: Vec::new(),
|
||||
message_id: None,
|
||||
content_hash: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user