feat(sync): incremental IMAP sync with progress + per-item FETCH timeout #231

Merged
KofTwentyTwo merged 4 commits from story/m5-s2-incremental-sync into dev 2026-07-30 02:33:56 +00:00
8 changed files with 866 additions and 31 deletions

No files matched your search

+13
View File
@@ -93,6 +93,19 @@ pub enum CoreEvent {
/// Action description.
action_taken: String,
},
/// Progress notification emitted during an inbound mail sync, so a
/// long-running sync of a large mailbox is observably progressing rather
/// than indistinguishable from a hang. Emitted per folder as it completes.
SyncProgress {
/// Account being synced, if the sync is scoped to one.
account_id: Option<String>,
/// Folder whose messages were just fetched and persisted.
folder_id: String,
/// Messages fetched from this folder on this pass.
fetched: usize,
/// Cumulative messages fetched across the sync so far.
total: usize,
},
/// Progress notification during bulk filter execution over a batch of emails.
BatchFilterProgress {
/// Messages processed so far in the batch.
+3
View File
@@ -31,3 +31,6 @@ reqwest.workspace = true
[dev-dependencies]
tempfile.workspace = true
wiremock.workspace = true
# `test-util` (not in tokio's `full`) enables paused-clock tests so the
# bounded per-item FETCH timeout can be asserted without a real-time wait.
tokio = { workspace = true, features = ["test-util"] }
+533 -19
View File
@@ -23,9 +23,108 @@ use tokio_stream::StreamExt;
/// hang a connect -- and therefore a `TestAccountConnection` RPC -- forever.
const CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
/// Upper bound on how long a single `FETCH` response item may take to arrive
/// once the `UID FETCH` command has been acknowledged. Fetching full bodies
/// for a large mailbox in one command can be legitimately slow overall, so the
/// bound is per item rather than for the whole stream; but a single item that
/// never arrives (dead socket, server-side hang) must still surface as an
/// honest timeout rather than hanging the sync indefinitely.
const FETCH_ITEM_TIMEOUT: Duration = Duration::from_secs(60);
use crate::backend::MailBackend;
use crate::parser::{MailError, MimeParserAdapter};
/// Delimiter between the two components of a folder sync checkpoint. The
/// checkpoint state string is `"{uidvalidity}{DELIM}{uidnext}"` (e.g.
/// `"42:105"`): the mailbox's UIDVALIDITY at the time it was written, then the
/// UID boundary to resume from. Both components are decimal `u32`.
///
/// UIDVALIDITY MUST be carried because IMAP UIDs are only meaningful within a
/// stable UIDVALIDITY (RFC 3501 s2.3.1.1). If a mailbox is recreated and its
/// UIDVALIDITY changes, UIDs restart from low numbers, so resuming from a UID
/// boundary captured under the old UIDVALIDITY would issue `{old_uidnext}:*`
/// against renumbered low UIDs -- which per `n:*` semantics returns only the
/// single highest message and silently skips the rest. Comparing UIDVALIDITY
/// first turns that into a safe full re-fetch instead.
const CHECKPOINT_DELIM: char = ':';
/// The `UID FETCH` sequence-set to issue for a folder sync, resolved from the
/// stored checkpoint. Kept as a small typed value (rather than collapsing to a
/// bare range string) so the distinct fall-back reasons stay observable: all
/// of `Full`/`CorruptCheckpoint`/`UidValidityChanged` fetch `1:*`, but only the
/// latter two are worth flagging, and tests can assert on which branch fired.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FetchRange {
/// No prior checkpoint (first-ever sync of this folder): fetch everything.
Full,
/// A checkpoint was stored but could not be parsed (malformed, or a legacy
/// bare-UID checkpoint predating UIDVALIDITY tracking). Falls back to a
/// full fetch, surfaced (logged) rather than silently ignored.
CorruptCheckpoint,
/// The stored checkpoint's UIDVALIDITY no longer matches the mailbox's
/// current UIDVALIDITY (the mailbox was renumbered) or the server did not
/// report a current UIDVALIDITY to compare against. Stored UIDs are no
/// longer meaningful, so this falls back to a safe full re-fetch.
UidValidityChanged {
/// UIDVALIDITY recorded in the stored checkpoint.
stored: u32,
/// UIDVALIDITY the server reports now, if any.
current: Option<u32>,
},
/// A valid checkpoint whose UIDVALIDITY still matches: fetch only UIDs at
/// or above the stored boundary, i.e. messages that arrived since.
Incremental(u32),
}
impl FetchRange {
/// Render the IMAP `UID FETCH` sequence-set. Every non-incremental variant
/// fetches all UIDs (`1:*`); an incremental checkpoint `n` fetches `n:*`,
/// which the server bounds to UIDs `>= n` -- a genuinely narrower fetch.
fn sequence_set(self) -> String {
match self {
FetchRange::Full
| FetchRange::CorruptCheckpoint
| FetchRange::UidValidityChanged { .. } => "1:*".to_string(),
FetchRange::Incremental(uid) => format!("{}:*", uid),
}
}
}
/// Resolve the fetch range from a stored `since_state` checkpoint and the
/// mailbox's CURRENT UIDVALIDITY (as just reported by `SELECT`). `None` means
/// no prior sync (full fetch). A checkpoint in the `"{uidvalidity}:{uidnext}"`
/// format proceeds incrementally only when its UIDVALIDITY still matches the
/// server's current one; a mismatch (mailbox renumbered) or a missing current
/// UIDVALIDITY forces a safe full re-fetch. A malformed or legacy bare-number
/// checkpoint is treated as corrupt (full fetch, flagged), never misread as a
/// live UID boundary.
fn resolve_fetch_range(since_state: Option<&str>, current_uid_validity: Option<u32>) -> FetchRange {
let Some(raw) = since_state else {
return FetchRange::Full;
};
let parsed = raw
.split_once(CHECKPOINT_DELIM)
.and_then(|(validity, uid)| {
Some((
validity.trim().parse::<u32>().ok()?,
uid.trim().parse::<u32>().ok()?,
))
});
let Some((stored_validity, uid)) = parsed else {
return FetchRange::CorruptCheckpoint;
};
if uid < 1 {
return FetchRange::CorruptCheckpoint;
}
match current_uid_validity {
Some(current) if current == stored_validity => FetchRange::Incremental(uid),
current => FetchRange::UidValidityChanged {
stored: stored_validity,
current,
},
}
}
/// State of the dedicated IMAP IDLE socket listener.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IdleSocketState {
@@ -449,21 +548,53 @@ impl ImapEngine {
&self.socket_manager
}
/// Fetch and parse all messages in a folder over an active IMAP session.
/// Fetch and parse the messages in a folder over an active IMAP session,
/// returning the fetched messages and the new sync checkpoint.
///
/// `since_state` is the checkpoint returned by a previous sync of this
/// folder, encoded as `"{uidvalidity}:{uidnext}"` (see [`CHECKPOINT_DELIM`]).
/// The `UID FETCH` is narrowed to `{uidnext}:*` only when the checkpoint's
/// UIDVALIDITY still matches the mailbox's current UIDVALIDITY; a mismatch
/// (the mailbox was renumbered) or a malformed/legacy checkpoint forces a
/// safe full re-fetch instead of silently skipping renumbered mail. The
/// returned checkpoint is derived from real server state -- the mailbox's
/// UIDVALIDITY plus its `UIDNEXT` (or one past the highest UID actually
/// seen) after selecting the folder -- never from a synthetic counter.
pub async fn sync_folder_messages_with_session<S>(
&self,
folder_id: &str,
since_state: Option<&str>,
session: &mut async_imap::Session<S>,
) -> Result<Vec<Email>, MailError>
) -> Result<(Vec<Email>, String), MailError>
where
S: AsyncRead + AsyncWrite + Unpin + Send + Debug + 'static,
{
session.select(folder_id).await.map_err(|e| {
let mailbox = session.select(folder_id).await.map_err(|e| {
MailError::ImapError(format!("failed to select folder '{}': {}", folder_id, e))
})?;
let server_uid_next = mailbox.uid_next;
let server_uid_validity = mailbox.uid_validity;
let range = resolve_fetch_range(since_state, server_uid_validity);
match range {
FetchRange::CorruptCheckpoint => tracing::warn!(
folder_id,
checkpoint = since_state.unwrap_or_default(),
"ignoring unparseable folder sync checkpoint; falling back to a full fetch"
),
FetchRange::UidValidityChanged { stored, current } => tracing::warn!(
folder_id,
stored_uidvalidity = stored,
current_uidvalidity = current,
"mailbox UIDVALIDITY changed (renumbered); the stored UID checkpoint is no \
longer valid, so re-fetching in full to avoid silently skipping messages"
),
FetchRange::Full | FetchRange::Incremental(_) => {}
}
let query = build_fetch_command_query();
let mut fetch_stream = session.uid_fetch("1:*", query).await.map_err(|e| {
let sequence_set = range.sequence_set();
let mut fetch_stream = session.uid_fetch(&sequence_set, query).await.map_err(|e| {
MailError::ImapError(format!(
"UID FETCH failed for folder '{}': {}",
folder_id, e
@@ -471,11 +602,29 @@ impl ImapEngine {
})?;
let mut emails = Vec::new();
while let Some(fetch_res) = fetch_stream.next().await {
let mut max_uid_seen: u32 = 0;
loop {
// Bound EVERY individual item read: a slow-but-progressing large
// fetch is fine, but a single item that never arrives must surface
// as an honest `FetchStalled` instead of hanging the whole sync.
let next_item = tokio::time::timeout(FETCH_ITEM_TIMEOUT, fetch_stream.next())
.await
.map_err(|_| {
MailError::FetchStalled(format!(
"no FETCH response for folder '{}' within {}s ({} message(s) read so far)",
folder_id,
FETCH_ITEM_TIMEOUT.as_secs(),
emails.len()
))
})?;
let Some(fetch_res) = next_item else {
break;
};
let fetch_data = fetch_res
.map_err(|e| MailError::ImapError(format!("failed reading fetch item: {}", e)))?;
let uid_num = fetch_data.uid.unwrap_or(0);
max_uid_seen = max_uid_seen.max(uid_num);
let email_id = format!("imap-uid-{}", uid_num);
let is_read = fetch_data
@@ -559,14 +708,44 @@ impl ImapEngine {
emails.push(email);
}
Ok(emails)
// Derive the next checkpoint from real server state as
// `"{uidvalidity}:{uidnext}"`. The UID boundary is the mailbox's
// authoritative UIDNEXT, else one past the highest UID actually seen.
// A checkpoint is only meaningful when both a UIDVALIDITY and a UID
// boundary are known; if either is missing (a server that omitted
// UIDVALIDITY, or an empty fetch that revealed no UIDNEXT/UID), keep
// the prior checkpoint rather than writing a validity-less one that the
// next sync could misread. As a last resort (no prior checkpoint
// either) emit an intentionally unparseable marker so the next sync
// safely falls back to a full fetch.
let boundary_uid = server_uid_next
.or_else(|| (max_uid_seen > 0).then_some(max_uid_seen.saturating_add(1)));
let new_checkpoint = match (server_uid_validity, boundary_uid) {
(Some(validity), Some(uid)) => format!("{}{}{}", validity, CHECKPOINT_DELIM, uid),
_ => since_state
.map(str::to_string)
.unwrap_or_else(|| "full-resync-required".to_string()),
};
Ok((emails, new_checkpoint))
}
/// Execute `UID FETCH 1:* (FLAGS INTERNALDATE RFC822.SIZE ENVELOPE BODY.PEEK[])` and parse returned messages into domain `Email` entities.
pub async fn sync_folder_messages(&self, folder_id: &str) -> Result<Vec<Email>, MailError> {
/// Fetch a folder's messages over a freshly authenticated session,
/// honoring `since_state` for an incremental fetch and returning the new
/// sync checkpoint alongside the messages. See
/// [`Self::sync_folder_messages_with_session`] for the checkpoint model.
pub async fn sync_folder_messages(
&self,
folder_id: &str,
since_state: Option<&str>,
) -> Result<(Vec<Email>, String), MailError> {
let (username, password) = match (&self.username, &self.password) {
(Some(u), Some(p)) => (u.as_str(), p.as_str()),
_ => {
// Credential-less fixture path used by unit tests that never
// reach a real server. The checkpoint (one past the fixture's
// UID) is internally consistent with the fixture rather than
// derived from `emails.len()`.
let mock_emails = vec![Email {
id: "imap-uid-100".to_string(),
account_id: self.account_id.clone(),
@@ -580,7 +759,7 @@ impl ImapEngine {
body_html: None,
attachments: Vec::new(),
}];
return Ok(mock_emails);
return Ok((mock_emails, "101".to_string()));
}
};
@@ -588,11 +767,11 @@ impl ImapEngine {
.socket_manager
.connect_session(username, password)
.await?;
let emails = self
.sync_folder_messages_with_session(folder_id, &mut session)
.await?;
let result = self
.sync_folder_messages_with_session(folder_id, since_state, &mut session)
.await;
let _ = session.logout().await;
Ok(emails)
result
}
/// Listen for real-time IMAP IDLE notification events on an active IMAP session.
@@ -742,11 +921,9 @@ impl MailBackend for ImapEngine {
async fn sync_messages(
&self,
folder_id: &str,
_since_state: Option<&str>,
since_state: Option<&str>,
) -> Result<(Vec<Email>, String), MailError> {
let emails = self.sync_folder_messages(folder_id).await?;
let modseq = format!("imap-modseq-{}", emails.len());
Ok((emails, modseq))
self.sync_folder_messages(folder_id, since_state).await
}
async fn send_email(&self, _email: &Email) -> Result<(), MailError> {
@@ -781,8 +958,9 @@ mod tests {
assert_eq!(folders.len(), 2);
assert_eq!(folders[0].id, "INBOX");
let (emails, modseq) = engine.sync_messages("INBOX", None).await?;
assert_eq!(modseq, "imap-modseq-1");
let (emails, checkpoint) = engine.sync_messages("INBOX", None).await?;
// Credential-less fixture: checkpoint is one past the fixture UID.
assert_eq!(checkpoint, "101");
assert_eq!(emails.len(), 1);
assert_eq!(emails[0].id, "imap-uid-100");
@@ -790,6 +968,342 @@ mod tests {
Ok(())
}
#[test]
fn resolve_fetch_range_distinguishes_none_valid_corrupt_and_renumbered_checkpoints() {
// First-ever sync: full history (current UIDVALIDITY irrelevant).
assert_eq!(resolve_fetch_range(None, Some(1)), FetchRange::Full);
assert_eq!(resolve_fetch_range(None, None).sequence_set(), "1:*");
// Valid checkpoint whose UIDVALIDITY still matches: narrowed fetch.
assert_eq!(
resolve_fetch_range(Some("1:105"), Some(1)),
FetchRange::Incremental(105)
);
assert_eq!(
resolve_fetch_range(Some("1:105"), Some(1)).sequence_set(),
"105:*"
);
assert_eq!(
resolve_fetch_range(Some(" 7 : 42 "), Some(7)),
FetchRange::Incremental(42)
);
// UIDVALIDITY changed (mailbox renumbered) or the server reported none
// to compare against: MUST fall back to a full fetch, distinctly, so a
// renumber can never silently skip the low new UIDs.
assert_eq!(
resolve_fetch_range(Some("1:105"), Some(2)),
FetchRange::UidValidityChanged {
stored: 1,
current: Some(2)
}
);
assert_eq!(
resolve_fetch_range(Some("1:105"), Some(2)).sequence_set(),
"1:*"
);
assert_eq!(
resolve_fetch_range(Some("1:105"), None),
FetchRange::UidValidityChanged {
stored: 1,
current: None
}
);
// Malformed or legacy bare-number (pre-UIDVALIDITY) checkpoints are
// corrupt -> full fetch, never misread as a live UID boundary.
assert_eq!(
resolve_fetch_range(Some(""), Some(1)),
FetchRange::CorruptCheckpoint
);
assert_eq!(
resolve_fetch_range(Some("105"), Some(1)),
FetchRange::CorruptCheckpoint,
"a legacy bare-number checkpoint must not be read as a UID boundary"
);
assert_eq!(
resolve_fetch_range(Some("1:0"), Some(1)),
FetchRange::CorruptCheckpoint
);
assert_eq!(
resolve_fetch_range(Some("not:auid"), Some(1)),
FetchRange::CorruptCheckpoint
);
assert_eq!(
resolve_fetch_range(Some("not-a-uid"), Some(1)).sequence_set(),
"1:*"
);
}
/// Read a single CRLF-terminated line from a scripted-server stream,
/// returning `None` at EOF (client hung up). Used only by the in-memory
/// duplex tests below; the real fetch path never parses raw lines.
async fn read_scripted_line<R: AsyncRead + Unpin>(reader: &mut R) -> Option<String> {
let mut buf = Vec::new();
let mut byte = [0u8; 1];
loop {
match reader.read(&mut byte).await {
Ok(0) => {
return if buf.is_empty() {
None
} else {
Some(String::from_utf8_lossy(&buf).into_owned())
}
}
Ok(_) => {
if byte[0] == b'\n' {
return Some(String::from_utf8_lossy(&buf).into_owned());
}
if byte[0] != b'\r' {
buf.push(byte[0]);
}
}
Err(_) => return None,
}
}
}
#[tokio::test]
async fn incremental_sync_narrows_the_fetch_range_on_the_second_pass() {
// Drives a real `async_imap::Session` over an in-memory duplex pair
// whose server side is a scripted IMAP responder. Proves the SECOND
// sync issues a GENUINELY narrower `UID FETCH` derived from the first
// sync's returned checkpoint -- not a full re-fetch.
let (client_io, server_io) = tokio::io::duplex(8192);
let fetch_ranges = Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let server_ranges = fetch_ranges.clone();
let server = tokio::spawn(async move {
let mut io = server_io;
while let Some(line) = read_scripted_line(&mut io).await {
let tag = line.split_whitespace().next().unwrap_or("").to_string();
let upper = line.to_ascii_uppercase();
if upper.contains("UID FETCH") {
if let Ok(mut g) = server_ranges.lock() {
g.push(line.clone());
}
let _ = io
.write_all(format!("{tag} OK FETCH completed\r\n").as_bytes())
.await;
} else if upper.contains("LOGIN") {
let _ = io
.write_all(format!("{tag} OK Logged in\r\n").as_bytes())
.await;
} else if upper.contains("SELECT") {
let resp = format!(
"* FLAGS (\\Seen)\r\n* 3 EXISTS\r\n* 0 RECENT\r\n\
* OK [UIDVALIDITY 1] ok\r\n* OK [UIDNEXT 105] ok\r\n\
{tag} OK [READ-WRITE] SELECT done\r\n"
);
let _ = io.write_all(resp.as_bytes()).await;
} else if upper.contains("LOGOUT") {
let _ = io
.write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes())
.await;
break;
}
}
});
let client = async_imap::Client::new(client_io);
let mut session = client
.login("user", "pass")
.await
.map_err(|(e, _)| e)
.expect("scripted login succeeds");
let engine = ImapEngine::new("acct-1", "example.test", 993);
// First sync: no checkpoint -> full history; returns
// "{uidvalidity}:{uidnext}" from real server state.
let (first, checkpoint) = engine
.sync_folder_messages_with_session("INBOX", None, &mut session)
.await
.expect("first sync succeeds");
assert!(first.is_empty());
assert_eq!(checkpoint, "1:105");
// Second sync: feed back the checkpoint; UIDVALIDITY still matches
// (1), so the fetch is narrowed.
let (_second, checkpoint2) = engine
.sync_folder_messages_with_session("INBOX", Some(&checkpoint), &mut session)
.await
.expect("second sync succeeds");
assert_eq!(checkpoint2, "1:105");
drop(session);
let _ = server.await;
let ranges = fetch_ranges.lock().expect("lock captured ranges");
assert_eq!(ranges.len(), 2, "expected exactly two UID FETCH commands");
assert!(
ranges[0].contains("1:*"),
"first sync must fetch full history, got: {}",
ranges[0]
);
assert!(
ranges[1].contains("105:*"),
"second sync must be narrowed to the checkpoint, got: {}",
ranges[1]
);
assert!(
!ranges[1].contains("1:*"),
"second sync must NOT re-fetch full history, got: {}",
ranges[1]
);
}
#[tokio::test]
async fn a_uidvalidity_change_forces_a_full_resync_instead_of_skipping_mail() {
// A mailbox renumber changes UIDVALIDITY and restarts UIDs from low
// numbers. Resuming from the old UID boundary (`105:*`) would return
// only the single highest message and silently drop the rest, so a
// UIDVALIDITY mismatch MUST force a full `1:*` re-fetch. The scripted
// server reports UIDVALIDITY 1 on the first SELECT and 2 on the second.
let (client_io, server_io) = tokio::io::duplex(8192);
let fetch_ranges = Arc::new(std::sync::Mutex::new(Vec::<String>::new()));
let server_ranges = fetch_ranges.clone();
let server = tokio::spawn(async move {
let mut io = server_io;
let mut select_count = 0u32;
while let Some(line) = read_scripted_line(&mut io).await {
let tag = line.split_whitespace().next().unwrap_or("").to_string();
let upper = line.to_ascii_uppercase();
if upper.contains("UID FETCH") {
if let Ok(mut g) = server_ranges.lock() {
g.push(line.clone());
}
let _ = io
.write_all(format!("{tag} OK FETCH completed\r\n").as_bytes())
.await;
} else if upper.contains("LOGIN") {
let _ = io
.write_all(format!("{tag} OK Logged in\r\n").as_bytes())
.await;
} else if upper.contains("SELECT") {
select_count += 1;
// First SELECT: UIDVALIDITY 1. Second: a DIFFERENT
// UIDVALIDITY (2), i.e. the mailbox was renumbered.
let uidvalidity = if select_count == 1 { 1 } else { 2 };
let resp = format!(
"* FLAGS (\\Seen)\r\n* 3 EXISTS\r\n* 0 RECENT\r\n\
* OK [UIDVALIDITY {uidvalidity}] ok\r\n* OK [UIDNEXT 105] ok\r\n\
{tag} OK [READ-WRITE] SELECT done\r\n"
);
let _ = io.write_all(resp.as_bytes()).await;
} else if upper.contains("LOGOUT") {
let _ = io
.write_all(format!("* BYE\r\n{tag} OK LOGOUT\r\n").as_bytes())
.await;
break;
}
}
});
let client = async_imap::Client::new(client_io);
let mut session = client
.login("user", "pass")
.await
.map_err(|(e, _)| e)
.expect("scripted login succeeds");
let engine = ImapEngine::new("acct-1", "example.test", 993);
// First sync under UIDVALIDITY 1 stores "1:105".
let (_first, checkpoint) = engine
.sync_folder_messages_with_session("INBOX", None, &mut session)
.await
.expect("first sync succeeds");
assert_eq!(checkpoint, "1:105");
// Second sync: server now reports UIDVALIDITY 2. The stored checkpoint
// is stale, so the fetch MUST be a full `1:*`, and the checkpoint is
// rewritten under the new UIDVALIDITY.
let (_second, checkpoint2) = engine
.sync_folder_messages_with_session("INBOX", Some(&checkpoint), &mut session)
.await
.expect("second sync succeeds");
assert_eq!(
checkpoint2, "2:105",
"checkpoint must be rewritten under the new UIDVALIDITY"
);
drop(session);
let _ = server.await;
let ranges = fetch_ranges.lock().expect("lock captured ranges");
assert_eq!(ranges.len(), 2, "expected exactly two UID FETCH commands");
assert!(
ranges[0].contains("1:*"),
"first sync fetches full history, got: {}",
ranges[0]
);
assert!(
ranges[1].contains("1:*") && !ranges[1].contains("105:*"),
"a UIDVALIDITY change MUST force a full re-fetch (1:*), never 105:*, got: {}",
ranges[1]
);
}
#[tokio::test(start_paused = true)]
async fn a_never_responding_fetch_surfaces_fetchstalled_without_hanging() {
// The scripted server answers LOGIN and SELECT but then goes silent on
// the FETCH. With the tokio clock paused, the bounded per-item timeout
// fast-forwards, so the stall is turned into an honest `FetchStalled`
// error in negligible WALL-CLOCK time -- proving it never hangs.
let (client_io, server_io) = tokio::io::duplex(8192);
let server = tokio::spawn(async move {
let mut io = server_io;
while let Some(line) = read_scripted_line(&mut io).await {
let tag = line.split_whitespace().next().unwrap_or("").to_string();
let upper = line.to_ascii_uppercase();
if upper.contains("UID FETCH") {
// Deliberately never respond: a mid-stream server stall.
std::future::pending::<()>().await;
} else if upper.contains("LOGIN") {
let _ = io
.write_all(format!("{tag} OK Logged in\r\n").as_bytes())
.await;
} else if upper.contains("SELECT") {
let resp = format!(
"* 1 EXISTS\r\n* OK [UIDVALIDITY 1] ok\r\n* OK [UIDNEXT 42] ok\r\n\
{tag} OK [READ-WRITE] SELECT done\r\n"
);
let _ = io.write_all(resp.as_bytes()).await;
}
}
});
let client = async_imap::Client::new(client_io);
let mut session = client
.login("user", "pass")
.await
.map_err(|(e, _)| e)
.expect("scripted login succeeds");
let engine = ImapEngine::new("acct-1", "example.test", 993);
// Real wall clock, not the (paused) tokio clock: proves negligible
// elapsed time despite the 60s virtual timeout expiring.
let started = std::time::Instant::now();
let err = engine
.sync_folder_messages_with_session("INBOX", None, &mut session)
.await
.expect_err("a never-responding FETCH must surface an error");
assert!(
matches!(err, MailError::FetchStalled(_)),
"expected FetchStalled, got: {err:?}"
);
assert!(
started.elapsed() < Duration::from_secs(5),
"stalled fetch must not hang in wall-clock time, took {:?}",
started.elapsed()
);
server.abort();
}
#[tokio::test]
async fn with_tls_mode_records_mode_and_resolves_default_ports() {
// A `0` port resolves to the conventional default for each mode.
+51 -3
View File
@@ -14,12 +14,41 @@ pub struct MockMailBackend {
messages: Arc<Mutex<Vec<Email>>>,
sent_messages: Arc<Mutex<Vec<Email>>>,
should_fail: Arc<Mutex<bool>>,
/// Every `since_state` argument this mock's `sync_messages` was called
/// with, in call order, so tests can prove a caller threads the returned
/// checkpoint back into the next sync (real incrementality).
since_state_calls: Arc<Mutex<Vec<Option<String>>>>,
/// The checkpoint token this mock returns from `sync_messages`.
returned_state: Arc<Mutex<String>>,
}
impl MockMailBackend {
/// Create a new `MockMailBackend` with empty storage.
pub fn new() -> Self {
Self::default()
let backend = Self::default();
if let Ok(mut state) = backend.returned_state.lock() {
*state = "mock-state-token-100".to_string();
}
backend
}
/// Override the checkpoint token this mock returns from `sync_messages`,
/// so a test can control the exact state a caller must persist and thread
/// back on the next sync.
pub fn set_returned_state(&self, state: &str) {
if let Ok(mut guard) = self.returned_state.lock() {
*guard = state.to_string();
}
}
/// Every `since_state` argument passed to `sync_messages`, in call order.
/// A caller that genuinely resumes from a checkpoint will show `None` on
/// the first call and the previously-returned state on the next.
pub fn since_state_calls(&self) -> Vec<Option<String>> {
self.since_state_calls
.lock()
.map(|guard| guard.clone())
.unwrap_or_default()
}
/// Configure the mock to simulate network failure errors.
@@ -74,8 +103,14 @@ impl MailBackend for MockMailBackend {
async fn sync_messages(
&self,
folder_id: &str,
_since_state: Option<&str>,
since_state: Option<&str>,
) -> Result<(Vec<Email>, String), MailError> {
// Record the checkpoint arg before any early return, so even a failing
// call is visible to a test asserting on how the caller resumes.
if let Ok(mut calls) = self.since_state_calls.lock() {
calls.push(since_state.map(str::to_string));
}
let should_fail = self
.should_fail
.lock()
@@ -85,6 +120,19 @@ impl MailBackend for MockMailBackend {
"simulated network failure".to_string(),
));
}
let returned_state = self
.returned_state
.lock()
.map_err(|e| MailError::ParseFailed(e.to_string()))?
.clone();
// Simulate a since_state-aware backend: with a prior checkpoint there
// is nothing new to hand back, so the incremental fetch is genuinely
// narrower (empty) rather than a full re-report of every message.
if since_state.is_some() {
return Ok((Vec::new(), returned_state));
}
let messages = self
.messages
.lock()
@@ -94,7 +142,7 @@ impl MailBackend for MockMailBackend {
.filter(|m| m.folder_id == folder_id)
.cloned()
.collect();
Ok((matches, "mock-state-token-100".to_string()))
Ok((matches, returned_state))
}
async fn send_email(&self, email: &Email) -> Result<(), MailError> {
+8
View File
@@ -32,6 +32,14 @@ pub enum MailError {
#[error("IMAP error: {0}")]
ImapError(String),
/// A single FETCH response item exceeded its bounded read timeout without
/// the server responding. Distinct from a generic [`MailError::ImapError`]
/// so callers can tell an honest mid-stream stall (dead socket, server-side
/// hang) apart from a protocol/parse failure instead of the sync hanging
/// forever.
#[error("IMAP FETCH stalled: {0}")]
FetchStalled(String),
/// Network or IO error.
#[error("network I/O error: {0}")]
NetworkError(String),
+113
View File
@@ -649,6 +649,19 @@ impl DatabaseEngine {
INSERT INTO contacts_fts(id, display_name, organization, emails_json)
VALUES (new.id, new.display_name, COALESCE(new.organization, ''), new.emails_json);
END;
-- Per-folder inbound-sync checkpoint. `state` is an opaque,
-- protocol-defined resumption token (an IMAP UID boundary such as
-- UIDNEXT, a JMAP state string) that a backend hands back after a
-- sync and expects to receive on the next sync so it can fetch only
-- what changed since. Keyed by (account, folder) because the token
-- is meaningful only within a single mailbox on a single account.
CREATE TABLE IF NOT EXISTS folder_sync_state (
account_id TEXT NOT NULL,
folder_id TEXT NOT NULL,
state TEXT NOT NULL,
PRIMARY KEY (account_id, folder_id)
);
"#,
)
.execute(&self.pool)
@@ -915,6 +928,51 @@ impl DatabaseEngine {
.collect())
}
/// Fetch the persisted inbound-sync checkpoint for a folder, or `None` if
/// this account/folder pair has never completed a sync. A `None` result is
/// the signal to the backend that it must perform a full first-time fetch.
pub async fn get_folder_sync_state(
&self,
account_id: &str,
folder_id: &str,
) -> Result<Option<String>, DatabaseError> {
let row: Option<(String,)> = sqlx::query_as(
"SELECT state FROM folder_sync_state WHERE account_id = ? AND folder_id = ?",
)
.bind(account_id)
.bind(folder_id)
.fetch_optional(&self.pool)
.await
.map_err(DatabaseError::Query)?;
Ok(row.map(|(state,)| state))
}
/// Persist the inbound-sync checkpoint a backend returned for a folder,
/// overwriting any prior value (a checkpoint is a running high-water mark,
/// not history). Written only after a folder's messages have been fetched
/// and persisted, so the stored checkpoint can never advance past work that
/// actually landed in the store.
pub async fn save_folder_sync_state(
&self,
account_id: &str,
folder_id: &str,
state: &str,
) -> Result<(), DatabaseError> {
sqlx::query(
r#"
INSERT OR REPLACE INTO folder_sync_state (account_id, folder_id, state)
VALUES (?, ?, ?)
"#,
)
.bind(account_id)
.bind(folder_id)
.bind(state)
.execute(&self.pool)
.await
.map_err(DatabaseError::Query)?;
Ok(())
}
/// Save an [`nuncio_core::model::Email`] to SQLite (INSERT OR REPLACE).
///
/// The message body is encrypted (AES-256-GCM) before being written to the `messages`
@@ -2696,6 +2754,61 @@ mod tests {
.expect("second delete is a no-op");
}
#[tokio::test]
async fn folder_sync_state_round_trip_is_scoped_and_overwrites() {
let (engine, _dir) = DatabaseEngine::connect_ephemeral().await.unwrap();
// Absent before any sync completes.
assert_eq!(
engine
.get_folder_sync_state("acct-1", "INBOX")
.await
.expect("get succeeds"),
None
);
engine
.save_folder_sync_state("acct-1", "INBOX", "105")
.await
.expect("save succeeds");
assert_eq!(
engine
.get_folder_sync_state("acct-1", "INBOX")
.await
.expect("get succeeds"),
Some("105".to_string())
);
// A later checkpoint overwrites (high-water mark, not history).
engine
.save_folder_sync_state("acct-1", "INBOX", "220")
.await
.expect("save succeeds");
assert_eq!(
engine
.get_folder_sync_state("acct-1", "INBOX")
.await
.expect("get succeeds"),
Some("220".to_string())
);
// Keyed by (account, folder): a different folder/account is unaffected.
assert_eq!(
engine
.get_folder_sync_state("acct-1", "Sent")
.await
.expect("get succeeds"),
None
);
assert_eq!(
engine
.get_folder_sync_state("acct-2", "INBOX")
.await
.expect("get succeeds"),
None
);
}
/// Proves the additive `smtp_host`/`smtp_port`
/// migration is backfill-safe. Simulates a database created BEFORE this
/// feature existed (an `accounts` table with no `smtp_host`/`smtp_port`
+11 -5
View File
@@ -87,15 +87,17 @@ pub enum GrpcServeError {
}
/// Maps a `nuncio_core::CoreEvent` domain event onto its wire-format
/// `nuncio.v1.Event` representation, faithfully carrying every variant's
/// `nuncio.v1.Event` representation, carrying every wire-visible variant's
/// fields into the matching `oneof` case (see `proto/nuncio/v1/nuncio.proto`
/// for the wire contract).
/// for the wire contract). Returns `None` for events that are internal-only
/// and have no case in the versioned contract, so they are dropped from the
/// gRPC subscription stream rather than forced onto the wire.
///
/// `usize` fields (`processed`, `total`, `matched`, `salvaged_rules_count`)
/// are narrowed to `u64` for the wire; these are in-process counters that
/// cannot realistically approach `u64::MAX`, and protobuf has no native
/// `usize` type.
fn map_core_event(event: CoreEvent) -> Event {
fn map_core_event(event: CoreEvent) -> Option<Event> {
let kind = match event {
CoreEvent::SyncStarted { account_id } => Kind::SyncStarted(SyncStarted { account_id }),
CoreEvent::SyncCompleted { account_id } => {
@@ -140,8 +142,12 @@ fn map_core_event(event: CoreEvent) -> Event {
}),
CoreEvent::Error { message } => Kind::Error(EventError { message }),
CoreEvent::ShuttingDown => Kind::ShuttingDown(ShuttingDown {}),
// Internal sync-progress signal: intentionally not forwarded to gRPC
// subscribers, as it has no case in the versioned wire contract. It is
// observed in-process (e.g. by the daemon's own subscribers) only.
CoreEvent::SyncProgress { .. } => return None,
};
Event { kind: Some(kind) }
Some(Event { kind: Some(kind) })
}
/// `nuncio.v1.System` gRPC service implementation backed by the daemon's live
@@ -191,7 +197,7 @@ impl System for SystemGrpcService {
) -> Result<Response<Self::SubscribeStream>, Status> {
let receiver = self.event_bus.subscribe_events();
let mapped = BroadcastStream::new(receiver).filter_map(|item| match item {
Ok(core_event) => Some(Ok(map_core_event(core_event))),
Ok(core_event) => map_core_event(core_event).map(Ok),
Err(BroadcastStreamRecvError::Lagged(skipped)) => {
tracing::warn!(
"gRPC Subscribe stream lagged behind the daemon event bus; skipped {} \
+134 -4
View File
@@ -18,7 +18,7 @@
//! only places that resolve a real account's credentials and construct a
//! real backend; they then delegate the actual fetch/persist/event-emission
//! work to [`sync_with_backend`].
use nuncio_core::{AccountConfig, AccountProtocol, CoreCommand, EventBus};
use nuncio_core::{AccountConfig, AccountProtocol, CoreCommand, CoreEvent, EventBus};
use nuncio_filter::{FilterEngine, OutboxManager, RuleAction};
use nuncio_mail::{ImapEngine, JmapEngine, MailBackend, MailError};
use nuncio_store::db::DatabaseError;
@@ -235,13 +235,27 @@ async fn is_new_message(db: &nuncio_store::db::DatabaseEngine, message_id: &str)
/// [`MockMailBackend`].
async fn fetch_and_persist(
db: &nuncio_store::db::DatabaseEngine,
event_bus: &EventBus,
backend: &dyn MailBackend,
filter_engine: &FilterEngine,
account_id: Option<&str>,
) -> Result<usize, SyncError> {
let folders = backend.sync_folders().await?;
let mut synced = 0usize;
for folder in folders {
let (emails, _state) = backend.sync_messages(&folder.id, None).await?;
// Resume from the folder's stored checkpoint so the backend can fetch
// only what changed since the last sync. A checkpoint is keyed by
// (account, folder), so a run with no account context (a mock-driven
// `SyncAll` with `account_id == None`) has nowhere to key it and always
// performs a full fetch.
let last_state = match account_id {
Some(acct) => db.get_folder_sync_state(acct, &folder.id).await?,
None => None,
};
let (emails, new_state) = backend
.sync_messages(&folder.id, last_state.as_deref())
.await?;
let fetched = emails.len();
for email in emails {
let is_new = is_new_message(db, &email.id).await;
db.save_email(&email).await?;
@@ -250,6 +264,19 @@ async fn fetch_and_persist(
}
synced += 1;
}
// Persist the returned checkpoint only AFTER this folder's messages
// have landed, so the stored high-water mark can never advance past
// work that actually reached the store.
if let Some(acct) = account_id {
db.save_folder_sync_state(acct, &folder.id, &new_state)
.await?;
}
event_bus.publish_event(CoreEvent::SyncProgress {
account_id: account_id.map(str::to_string),
folder_id: folder.id.clone(),
fetched,
total: synced,
});
}
Ok(synced)
}
@@ -282,7 +309,8 @@ pub async fn sync_with_backend(
None => event_bus.process_command(CoreCommand::SyncAll),
}
let result = fetch_and_persist(db, backend, filter_engine).await;
let result =
fetch_and_persist(db, event_bus, backend, filter_engine, account_id.as_deref()).await;
if let Err(e) = &result {
event_bus.process_command(CoreCommand::ReportError {
@@ -394,7 +422,15 @@ pub async fn run_all_accounts_sync(
match secrets.get_secret(&config.keyring_secret_key) {
Ok(password) => {
let backend = build_mail_backend(&config, &password);
match fetch_and_persist(db, backend.as_ref(), filter_engine).await {
match fetch_and_persist(
db,
event_bus,
backend.as_ref(),
filter_engine,
Some(&config.id),
)
.await
{
Ok(count) => total_synced += count,
Err(e) => {
event_bus.process_command(CoreCommand::ReportError {
@@ -619,6 +655,16 @@ mod tests {
account_id: Some("acct-mock-1".to_string())
}
);
// A per-folder progress event is published between start and complete.
assert_eq!(
events.recv().await.expect("progress event"),
CoreEvent::SyncProgress {
account_id: Some("acct-mock-1".to_string()),
folder_id: "inbox".to_string(),
fetched: 2,
total: 2,
}
);
assert_eq!(
events.recv().await.expect("complete event"),
CoreEvent::SyncCompleted {
@@ -654,6 +700,16 @@ mod tests {
events.recv().await.expect("start event"),
CoreEvent::SyncStarted { account_id: None }
);
// The folder is empty, but a progress event still marks it done.
assert_eq!(
events.recv().await.expect("progress event"),
CoreEvent::SyncProgress {
account_id: None,
folder_id: "inbox".to_string(),
fetched: 0,
total: 0,
}
);
assert_eq!(
events.recv().await.expect("complete event"),
CoreEvent::SyncCompleted { account_id: None }
@@ -791,6 +847,15 @@ mod tests {
account_id: Some("acct-jmap-real-1".to_string())
}
);
assert_eq!(
events.recv().await.expect("progress event"),
CoreEvent::SyncProgress {
account_id: Some("acct-jmap-real-1".to_string()),
folder_id: "jmap-inbox".to_string(),
fetched: 1,
total: 1,
}
);
assert_eq!(
events.recv().await.expect("complete event"),
CoreEvent::SyncCompleted {
@@ -1202,4 +1267,69 @@ mod tests {
"no new outbox mutation should be enqueued on a repeat sync"
);
}
#[tokio::test]
async fn sync_with_backend_persists_and_threads_the_folder_checkpoint_across_syncs() {
// The regression this story exists to kill: the returned checkpoint
// must be persisted and fed back into the NEXT sync, so the second
// sync is genuinely incremental rather than a full re-fetch. Proven by
// capturing the exact `since_state` the backend received on each call.
let (db, _dir) = DatabaseEngine::connect_ephemeral()
.await
.expect("ephemeral db");
let event_bus = EventBus::new();
let mock = MockMailBackend::new();
mock.set_returned_state("uidnext-4242");
mock.add_folder(Folder {
id: "inbox".to_string(),
name: "Inbox".to_string(),
total_messages: 1,
unread_messages: 1,
});
mock.add_message(mock_email("m-inc-1", "inbox", "First"));
let filter_engine = empty_filter_engine();
// First sync: no checkpoint exists yet, so the backend is called with
// `None` and does a full fetch (one message).
let first = sync_with_backend(
&db,
&event_bus,
&mock,
&filter_engine,
Some("acct-inc-1".to_string()),
)
.await
.expect("first sync succeeds");
assert_eq!(first, 1);
// The returned checkpoint must now be persisted for (account, folder).
assert_eq!(
db.get_folder_sync_state("acct-inc-1", "inbox")
.await
.expect("read checkpoint"),
Some("uidnext-4242".to_string())
);
// Second sync: the daemon must feed the stored checkpoint back in. The
// mock returns nothing new for a non-None checkpoint -> narrower fetch.
let second = sync_with_backend(
&db,
&event_bus,
&mock,
&filter_engine,
Some("acct-inc-1".to_string()),
)
.await
.expect("second sync succeeds");
assert_eq!(second, 0, "an incremental second sync fetches nothing new");
// The proof of genuine incrementality: the backend saw `None` first,
// then the exact checkpoint the first sync returned.
assert_eq!(
mock.since_state_calls(),
vec![None, Some("uidnext-4242".to_string())],
);
}
}